css_lexer/
kindset.rs

1use crate::Kind;
2
3/// Match a token against one or more [Kinds][Kind].
4///
5/// Each [Kind] represents the token "type". [KindSet] is a bitmask of all possible [Kinds][Kind]. This is useful for
6/// efficiently comparing a token to see if it matches N token [Kinds][Kind].
7///
8/// # Example
9///
10/// ```
11/// use css_lexer::*;
12/// let mut lexer = Lexer::new(&EmptyAtomSet::ATOMS, "width: 1px");
13/// // The first token is either an AtKeyword, Ident or Function:
14/// assert_eq!(lexer.advance(), KindSet::new(&[Kind::AtKeyword, Kind::Ident, Kind::Function]));
15/// ```
16#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
17pub struct KindSet(u64);
18
19impl KindSet {
20	/// A [KindSet] that matches no [Kinds][Kind].
21	pub const NONE: KindSet = KindSet::new(&[]);
22
23	/// A [KindSet] that matches all trivia; [Kind::Whitespace] and [Kind::Comment].
24	pub const TRIVIA: KindSet = KindSet::new(&[Kind::Whitespace, Kind::Comment]);
25
26	/// A [KindSet] that matches just Whitespace. This is the same as [Kind::Whitespace] but can be useful to apply to
27	/// functions that expect a [KindSet] rather than [Kind].
28	pub const WHITESPACE: KindSet = KindSet::new(&[Kind::Whitespace]);
29
30	/// A [KindSet] that matches just Whitespace. This is the same as [Kind::Comment] but can be useful to apply to
31	/// functions that expect a [KindSet] rather than [Kind].
32	pub const COMMENTS: KindSet = KindSet::new(&[Kind::Comment]);
33
34	/// A [KindSet] that matches just Whitespace or Semicolons.
35	pub const WHITESPACE_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::Whitespace, Kind::Semicolon]);
36
37	/// A [KindSet] that matches either [Kind::RightCurly] or [Kind::Semicolon]. This is useful for matching
38	/// stop-tokens, for example checking the end of a declaration.
39	pub const RIGHT_CURLY_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::RightCurly, Kind::Semicolon]);
40
41	/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::Semicolon]. This is useful for matching
42	/// stop-tokens, for example checking the end of an at-rule prelude.
43	pub const LEFT_CURLY_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::LeftCurly, Kind::Semicolon]);
44
45	/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::RightParen] or [Kind::Semicolon]. This is useful for
46	/// matching stop-tokens, for example checking the end of a function.
47	pub const LEFT_CURLY_RIGHT_PAREN_OR_SEMICOLON: KindSet =
48		KindSet::new(&[Kind::LeftCurly, Kind::RightParen, Kind::Semicolon]);
49
50	/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::RightParen] or [Kind::Comma] or [Kind::Semicolon].
51	/// This is useful for matching stop-tokens, for example checking the end of a function or Selector.
52	pub const LEFT_CURLY_RIGHT_PAREN_COMMA_OR_SEMICOLON: KindSet =
53		KindSet::new(&[Kind::LeftCurly, Kind::RightParen, Kind::Comma, Kind::Semicolon]);
54
55	/// A [KindSet] that matches either [Kind::Ident], [Kind::AtKeyword], [Kind::Function], [Kind::Hash].
56	pub const IDENT_LIKE: KindSet = KindSet::new(&[Kind::Ident, Kind::AtKeyword, Kind::Function, Kind::Hash]);
57
58	/// A [KindSet] that matches token kinds which can carry atom string content: [Kind::Ident], [Kind::Function],
59	/// [Kind::AtKeyword], and [Kind::Dimension]. These are the only kinds for which
60	/// [Token::atom_bits][crate::Token::atom_bits] may return a non-zero value.
61	pub const ATOM_LIKE: KindSet = KindSet::new(&[Kind::Ident, Kind::Function, Kind::AtKeyword, Kind::Dimension]);
62
63	/// A [KindSet] that matches any single character token, such as [Kind::Delim] or [Kind::Colon] - [Kind::RightCurly].
64	pub const DELIM_LIKE: KindSet = KindSet::new(&[
65		Kind::Delim,
66		Kind::Colon,
67		Kind::Semicolon,
68		Kind::Comma,
69		Kind::LeftSquare,
70		Kind::RightSquare,
71		Kind::LeftParen,
72		Kind::RightParen,
73		Kind::LeftCurly,
74		Kind::RightCurly,
75	]);
76
77	/// A [KindSet] that matches either [Kind::LeftCurly], [Kind::LeftParen], [Kind::LeftSquare].
78	pub const PAIRWISE_START: KindSet = KindSet::new(&[Kind::LeftCurly, Kind::LeftParen, Kind::LeftSquare]);
79	///
80	/// A [KindSet] that matches either [Kind::RightCurly], [Kind::RightParen], [Kind::RightSquare].
81	pub const PAIRWISE_END: KindSet = KindSet::new(&[Kind::RightCurly, Kind::RightParen, Kind::RightSquare]);
82
83	/// A [KindSet] that matches _any_ token.
84	pub const ANY: KindSet = KindSet(u64::MAX);
85
86	/// Creates a new [KindSet] with the combination of all given [Kinds][Kind].
87	///
88	/// This function is marked `const` to allow creation of const [KindSets][KindSet].
89	pub const fn new(kinds: &[Kind]) -> Self {
90		let mut u = 0;
91		let mut i = 0;
92		let len = kinds.len();
93		while i < len {
94			u |= 1 << (kinds[i] as u8 & 0b111111);
95			i += 1;
96		}
97		Self(u)
98	}
99
100	/// Returns a new [KindSet] with the addition of the supplied [Kind].
101	///
102	/// This function is marked `const` to allow creation of const [KindSets][KindSet].
103	pub const fn add(&self, kind: Kind) -> Self {
104		Self(self.0 | (1 << (kind as u8 & 0b111111)))
105	}
106
107	/// Returns a new [KindSet] combined with the other [KindSet].
108	///
109	/// This function is marked `const` to allow creation of const [KindSets][KindSet].
110	pub const fn combine(&self, ks: KindSet) -> Self {
111		Self(self.0 | ks.0)
112	}
113
114	/// Returns a new [KindSet] without the supplied [Kind].
115	///
116	/// This function is marked `const` to allow creation of const [KindSets][KindSet].
117	pub const fn remove(&self, kind: Kind) -> Self {
118		Self(self.0 ^ (1 << (kind as u8 & 0b111111)))
119	}
120
121	/// Check if a [KindSet] contains the subpplied [Kind].
122	pub fn contains(&self, kind: Kind) -> bool {
123		self.0 & (1 << (kind as u8 & 0b111111)) != 0
124	}
125
126	pub(crate) const fn contains_bits(&self, kind_bits: u8) -> bool {
127		self.0 & (1 << (kind_bits & 0b111111)) != 0
128	}
129}
130
131#[test]
132fn test_kindset_contains() {
133	let set = KindSet::new(&[Kind::Eof, Kind::Whitespace, Kind::Comment]);
134	assert!(set.contains(Kind::Eof));
135	assert!(set.contains(Kind::Whitespace));
136	assert!(set.contains(Kind::Comment));
137	assert!(!set.contains(Kind::String));
138	assert!(!set.contains(Kind::Url));
139
140	let set = KindSet::new(&[Kind::LeftCurly, Kind::LeftSquare, Kind::LeftParen]);
141
142	assert!(set.contains(Kind::LeftCurly));
143	assert!(!set.contains(Kind::RightCurly));
144	assert!(set.contains(Kind::LeftSquare));
145	assert!(!set.contains(Kind::RightSquare));
146	assert!(set.contains(Kind::LeftParen));
147	assert!(!set.contains(Kind::RightParen));
148	assert!(!set.contains(Kind::Ident));
149
150	assert!(KindSet::COMMENTS.contains(Kind::Comment));
151	assert!(!KindSet::COMMENTS.contains(Kind::Delim));
152}
153
154#[test]
155fn test_kindset_add_remove() {
156	let k_ident = KindSet::new(&[Kind::Ident]);
157	let k_ident_eof = k_ident.add(Kind::Eof);
158	assert!(k_ident.contains(Kind::Ident));
159	assert!(k_ident_eof.contains(Kind::Ident));
160	assert!(k_ident_eof.contains(Kind::Eof));
161	assert!(!k_ident_eof.remove(Kind::Eof).contains(Kind::Eof));
162}
css_lexer/kindset.rs

css_lexer/
kindset.rs
