css_lexer/
kindset.rs

1use crate::Kind;
2
3/// Match a token against one or more [Kinds][Kind].
4///
5/// Each [Kind] represents the token "type". [KindSet] is a bitmask of all possible [Kinds][Kind]. This is useful for
6/// efficiently comparing a token to see if it matches N token [Kinds][Kind].
7///
8/// # Example
9///
10/// ```
11/// use css_lexer::*;
12/// let mut lexer = Lexer::new(&EmptyAtomSet::ATOMS, "width: 1px");
13/// // The first token is either an AtKeyword, Ident or Function:
14/// assert_eq!(lexer.advance(), KindSet::new(&[Kind::AtKeyword, Kind::Ident, Kind::Function]));
15/// ```
16#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
17pub struct KindSet(u64);
18
19impl KindSet {
20	/// A [KindSet] that matches no [Kinds][Kind].
21	pub const NONE: KindSet = KindSet::new(&[]);
22
23	/// A [KindSet] that matches all trivia; [Kind::Whitespace] and [Kind::Comment].
24	pub const TRIVIA: KindSet = KindSet::new(&[Kind::Whitespace, Kind::Comment]);
25
26	/// A [KindSet] that matches just Whitespace. This is the same as [Kind::Whitespace] but can be useful to apply to
27	/// functions that expect a [KindSet] rather than [Kind].
28	pub const WHITESPACE: KindSet = KindSet::new(&[Kind::Whitespace]);
29
30	/// A [KindSet] that matches just Whitespace. This is the same as [Kind::Comment] but can be useful to apply to
31	/// functions that expect a [KindSet] rather than [Kind].
32	pub const COMMENTS: KindSet = KindSet::new(&[Kind::Comment]);
33
34	/// A [KindSet] that matches just Whitespace or Semicolons.
35	pub const WHITESPACE_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::Whitespace, Kind::Semicolon]);
36
37	/// A [KindSet] that matches either [Kind::RightCurly] or [Kind::Semicolon]. This is useful for matching
38	/// stop-tokens, for example checking the end of a declaration.
39	pub const RIGHT_CURLY_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::RightCurly, Kind::Semicolon]);
40
41	/// A [KindSet] that matches [Kind::RightCurly], [Kind::Semicolon] or [Kind::RightParen]. This is useful for
42	/// matching stop-tokens for a declaration.
43	pub const RIGHT_CURLY_SEMICOLON_OR_RIGHT_PAREN: KindSet =
44		KindSet::new(&[Kind::RightCurly, Kind::Semicolon, Kind::RightParen]);
45
46	/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::Semicolon]. This is useful for matching
47	/// stop-tokens, for example checking the end of an at-rule prelude.
48	pub const LEFT_CURLY_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::LeftCurly, Kind::Semicolon]);
49
50	/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::RightParen] or [Kind::Semicolon]. This is useful for
51	/// matching stop-tokens, for example checking the end of a function.
52	pub const LEFT_CURLY_RIGHT_PAREN_OR_SEMICOLON: KindSet =
53		KindSet::new(&[Kind::LeftCurly, Kind::RightParen, Kind::Semicolon]);
54
55	/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::RightParen] or [Kind::Comma] or [Kind::Semicolon].
56	/// This is useful for matching stop-tokens, for example checking the end of a function or Selector.
57	pub const LEFT_CURLY_RIGHT_PAREN_COMMA_OR_SEMICOLON: KindSet =
58		KindSet::new(&[Kind::LeftCurly, Kind::RightParen, Kind::Comma, Kind::Semicolon]);
59
60	/// A [KindSet] that matches either [Kind::Ident], [Kind::AtKeyword], [Kind::Function], [Kind::Hash].
61	pub const IDENT_LIKE: KindSet = KindSet::new(&[Kind::Ident, Kind::AtKeyword, Kind::Function, Kind::Hash]);
62
63	/// A [KindSet] that matches token kinds which can carry atom string content: [Kind::Ident], [Kind::Function],
64	/// [Kind::AtKeyword], and [Kind::Dimension]. These are the only kinds for which
65	/// [Token::atom_bits][crate::Token::atom_bits] may return a non-zero value.
66	pub const ATOM_LIKE: KindSet = KindSet::new(&[Kind::Ident, Kind::Function, Kind::AtKeyword, Kind::Dimension]);
67
68	/// A [KindSet] that matches the kinds which name something, such as [Kind::Ident] or
69	/// [Kind::String].
70	pub const NAMED: KindSet = KindSet::new(&[
71		Kind::Ident,
72		Kind::Function,
73		Kind::AtKeyword,
74		Kind::Hash,
75		Kind::String,
76		Kind::Url,
77		Kind::Comment,
78		Kind::UnicodeRange,
79		Kind::Dimension,
80		Kind::BadIdent,
81		Kind::BadFunction,
82		Kind::BadAtKeyword,
83		Kind::BadHash,
84		Kind::BadString,
85		Kind::BadUrl,
86		Kind::BadComment,
87		Kind::BadDimension,
88	]);
89
90	/// A [KindSet] that matches any single character token, such as [Kind::Delim] or [Kind::Colon] - [Kind::RightCurly].
91	pub const DELIM_LIKE: KindSet = KindSet::new(&[
92		Kind::Delim,
93		Kind::Colon,
94		Kind::Semicolon,
95		Kind::Comma,
96		Kind::LeftSquare,
97		Kind::RightSquare,
98		Kind::LeftParen,
99		Kind::RightParen,
100		Kind::LeftCurly,
101		Kind::RightCurly,
102	]);
103
104	/// A [KindSet] that matches either [Kind::LeftCurly], [Kind::LeftParen], [Kind::LeftSquare].
105	pub const PAIRWISE_START: KindSet = KindSet::new(&[Kind::LeftCurly, Kind::LeftParen, Kind::LeftSquare]);
106	///
107	/// A [KindSet] that matches either [Kind::RightCurly], [Kind::RightParen], [Kind::RightSquare].
108	pub const PAIRWISE_END: KindSet = KindSet::new(&[Kind::RightCurly, Kind::RightParen, Kind::RightSquare]);
109
110	/// A [KindSet] that matches _any_ token.
111	pub const ANY: KindSet = KindSet(u64::MAX);
112
113	/// Creates a new [KindSet] with the combination of all given [Kinds][Kind].
114	///
115	/// This function is marked `const` to allow creation of const [KindSets][KindSet].
116	pub const fn new(kinds: &[Kind]) -> Self {
117		let mut u = 0;
118		let mut i = 0;
119		let len = kinds.len();
120		while i < len {
121			u |= 1 << (kinds[i] as u8 & 0b111111);
122			i += 1;
123		}
124		Self(u)
125	}
126
127	/// Returns a new [KindSet] with the addition of the supplied [Kind].
128	///
129	/// This function is marked `const` to allow creation of const [KindSets][KindSet].
130	pub const fn add(&self, kind: Kind) -> Self {
131		Self(self.0 | (1 << (kind as u8 & 0b111111)))
132	}
133
134	/// Returns a new [KindSet] combined with the other [KindSet].
135	///
136	/// This function is marked `const` to allow creation of const [KindSets][KindSet].
137	pub const fn combine(&self, ks: KindSet) -> Self {
138		Self(self.0 | ks.0)
139	}
140
141	/// Returns a new [KindSet] without the supplied [Kind].
142	///
143	/// This function is marked `const` to allow creation of const [KindSets][KindSet].
144	pub const fn remove(&self, kind: Kind) -> Self {
145		Self(self.0 ^ (1 << (kind as u8 & 0b111111)))
146	}
147
148	/// Check if a [KindSet] contains the subpplied [Kind].
149	pub const fn contains(&self, kind: Kind) -> bool {
150		self.0 & (1 << (kind as u8 & 0b111111)) != 0
151	}
152
153	pub(crate) const fn contains_bits(&self, kind_bits: u8) -> bool {
154		self.0 & (1 << (kind_bits & 0b111111)) != 0
155	}
156}
157
158#[test]
159fn test_kindset_contains() {
160	let set = KindSet::new(&[Kind::Eof, Kind::Whitespace, Kind::Comment]);
161	assert!(set.contains(Kind::Eof));
162	assert!(set.contains(Kind::Whitespace));
163	assert!(set.contains(Kind::Comment));
164	assert!(!set.contains(Kind::String));
165	assert!(!set.contains(Kind::Url));
166
167	let set = KindSet::new(&[Kind::LeftCurly, Kind::LeftSquare, Kind::LeftParen]);
168
169	assert!(set.contains(Kind::LeftCurly));
170	assert!(!set.contains(Kind::RightCurly));
171	assert!(set.contains(Kind::LeftSquare));
172	assert!(!set.contains(Kind::RightSquare));
173	assert!(set.contains(Kind::LeftParen));
174	assert!(!set.contains(Kind::RightParen));
175	assert!(!set.contains(Kind::Ident));
176
177	assert!(KindSet::COMMENTS.contains(Kind::Comment));
178	assert!(!KindSet::COMMENTS.contains(Kind::Delim));
179}
180
181#[test]
182fn test_kindset_add_remove() {
183	let k_ident = KindSet::new(&[Kind::Ident]);
184	let k_ident_eof = k_ident.add(Kind::Eof);
185	assert!(k_ident.contains(Kind::Ident));
186	assert!(k_ident_eof.contains(Kind::Ident));
187	assert!(k_ident_eof.contains(Kind::Eof));
188	assert!(!k_ident_eof.remove(Kind::Eof).contains(Kind::Eof));
189}
css_lexer/kindset.rs

css_lexer/
kindset.rs
