css_lexer/kind.rs
1use core::fmt;
2
3use crate::KindSet;
4
5/// Kind represents the token "Type", categorised mostly by the token types within the CSS Syntax spec.
6///
7/// Importantly, `Kind` is represented as `u8` and must only use the 5 low bits, because the upper 3 bits get used to
8/// house details about each kind, that a token would be interested in learning about.
9///
10/// Maintaining parity with the spec makes it easier to reason about logic around the parser, despite it being possible to
11/// group a bunch of these tokens into a single "delimiter" token. These Delim kinds, however, set the upper bit which
12/// means they cannot be inserted directly into a token. Instead a token.
13#[repr(u8)]
14#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
15pub enum Kind {
16 // Trivias (mask as 0b0_00XX)
17 /// Represents the [<eof-token>][1] defined in CSS. While CSS stipulates that this token is never produced by a
18 /// tokenizer, this [Lexer][crate::Lexer] _will_ produce [<eof-token>s][1] if the underlying source has been
19 /// fully consumed.
20 ///
21 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-eof-token
22 Eof = 0b0000,
23
24 /// Represents the [<whitespace-token>][1] defined in CSS.
25 ///
26 /// ```md
27 /// <newline>
28 /// │├──╮─ "\n" ───╭──┤│
29 /// ├─ "\r\n" ─┤
30 /// ├─ "\r" ───┤
31 /// ╰─ "\f" ───╯
32 ///
33 /// <whitespace>
34 /// │├──╮─ " " ───────╭──┤│
35 /// ├─ "\t" ──────┤
36 /// ╰─ <newline> ─╯
37 ///
38 /// <whitespace-token>
39 /// │├─╭─ <whitespace> ─╮─┤│
40 /// ╰────────────────╯
41 /// ```
42 ///
43 /// While CSS stipulates that this token represents collapsed whitespace, it is possible for [Lexer][crate::Lexer]
44 /// to produce multiple consecutive [Kind::Whitespace] tokens if the
45 /// [Feature::SeparateWhitespace][crate::Feature::SeparateWhitespace] runtime feature is enabled. In this case,
46 /// `<whitespace-token>` becomes:
47 ///
48 /// ```md
49 /// <whitespace-token>
50 /// │├──╮─╭─ " " ───────╮─╭──┤│
51 /// │ ╰─────────────╯ │
52 /// ├─╭─ "\t" ──────╮─┤
53 /// │ ╰─────────────╯ │
54 /// ╰─╭─ <newline> ─╮─╯
55 /// ╰─────────────╯
56 /// ```
57 ///
58 /// [1]: https://drafts.csswg.org/css-syntax/#whitespace-token-diagram
59 #[default]
60 Whitespace = 0b0001,
61
62 /// Represents the [<comment>][1] defined in CSS. While CSS stipulates comment tokens are not produced during
63 /// tokenization, they are for this [Lexer][crate::Lexer] as they're needed in order to preserve them.
64 ///
65 /// ```md
66 /// <comment>
67 /// ╭──────────────────────────────────────────╮
68 /// │├─ "/*" ─╯-╭─ (anything but "*" followed by "/") ─╮─╰─ "*/" ─┤│
69 /// ╰──────────────────────────────────────╯
70 /// ```
71 ///
72 /// It is possible for [Lexer][crate::Lexer] to produce [Kind::Whitespace] tokens that begin `//` if the
73 /// [Feature::SingleLineComments][crate::Feature::SingleLineComments] runtime feature is enabled. In this mode,
74 /// `<comment>` becomes:
75 ///
76 /// ```md
77 /// <comment>
78 /// ╭──────────────────────────────────────────╮
79 /// │├──╮─ "/*" ─╯-╭─ (anything but "*" followed by "/") ─╮─╰─ "*/" ─╭─┤│
80 /// │ ╰──────────────────────────────────────╯ │
81 /// │ ╭───────────────────────────╮ │
82 /// ╰─ "//" ───────╯-╭─ (anything but "\n") ─╮─╰─ "\n" ──────────╯
83 /// ╰───────────────────────╯
84 /// ```
85 ///
86 /// [1]: https://drafts.csswg.org/css-syntax/#comment-diagram
87 Comment = 0b0010,
88
89 /// Represents both the [<cdc-token>][1] and [<cdo-token>][2]s defined in CSS. While CSS separates these tokens,
90 /// they're only useful representations at the top-level stylesheet, anywhere else they represent a parse error, and
91 /// it's a little pointless to define two tokens types for what amounts to a parse error.
92 ///
93 /// ```md
94 /// <cdo-token>
95 /// │├─ "<!--" ─┤│
96 ///
97 /// <cdc-token>
98 /// │├─ "-->" ─┤│
99 ///
100 /// <cdc-or-cdo-token> (Not part of the CSS specification)
101 /// │├──╮─ <cdo-token> ─╭──┤│
102 /// ╰─ <crc-token> ─╯
103 /// ```
104 ///
105 /// [1]: https://drafts.csswg.org/css-syntax/#CDC-token-diagram
106 /// [2]: https://drafts.csswg.org/css-syntax/#CDO-token-diagram
107 CdcOrCdo = 0b0011,
108
109 // Numerics (mask as 0b0_010X)
110 /// Represents the [<number-token>][1].
111 ///
112 /// ```md
113 ///
114 /// <number-token>
115 /// ╭─ "+" ─╮
116 /// │├─├───────┤───╭─ [digit] ─╮─ "." ─╭─ [digit] ─╮──╭───╮──────────────────────────────────╭──┤│
117 /// ╰─ "-" ─╯ │ ╰───────────╯ ╰───────────╯ │ │ ╭─ "+" ─╮ │
118 /// ├───────── ╭─ [digit] ─╮─────────────┤ ├─ "e" ─╭─├───────┤──╭─ [digit] ─╮─╯
119 /// │ ╰───────────╯ │ ╰─ "E" ─╯ ╰─ "-" ─╯ ╰───────────╯
120 /// ╰──── "." ─╭─ [digit] ─╮─────────────╯
121 /// ╰───────────╯
122 /// ```
123 ///
124 /// [1]: https://drafts.csswg.org/css-syntax/#number-token-diagram
125 Number = 0b0100,
126
127 /// Represents the [<dimension-token>][1].
128 ///
129 /// Here we deviate from the spec slightly, which has both [<dimension-token>][1] and [<percentage-token>][2].
130 /// `<percentage-token>` represents a dimension with a `%` symbol, but having this as a separate token results in more
131 /// work in the parser for little gain in the Lexer. So instead this lexer does not have a `<percentage-token>` and
132 /// instead folds the grammar for it inside of `<dimension-token>`.
133 ///
134 /// ```md
135 ///
136 /// <newline>
137 /// │├──╮─ "\n" ───╭──┤│
138 /// ├─ "\r\n" ─┤
139 /// ├─ "\r" ───┤
140 /// ╰─ "\f" ───╯
141 ///
142 /// <whitespace>
143 /// │├──╮─ " " ───────╭──┤│
144 /// ├─ "\t" ──────┤
145 /// ╰─ <newline> ─╯
146 ///
147 /// <hexdigit>
148 /// │├─ [ 0-9, A-F, a-f ] ─┤│
149 ///
150 ///
151 /// <escape>
152 /// │├─ "\" ─╮───── [not <newline> or <hexdigit>] ───╭─┤│
153 /// ╰─╭── <hexdigit> ─╮──╮────────────────╭─╯
154 /// ╰─ (1-6 times) ─╯ ╰─ <whitespace> ─╯
155 ///
156 /// <ident-token>
157 /// ╭───────────────── "--" ─────────────────────╮ ╭───────────────────────────────────────────╮
158 /// │├─╯─╮───────╭─╮─ [a-z, A-Z, "_", non-ASCII] ─╭─╰──╯─╭─╮─ [a-z, A-Z, 0-9, "_", non-ASCII] ─╭─╮─╰──┤│
159 /// ╰─ "-" ─╯ ╰──────── <escape> ────────────╯ │ ╰──────────── <escape> ─────────────╯ │
160 /// ╰───────────────────────────────────────╯
161 ///
162 /// <number-token>
163 /// ╭─ "+" ─╮
164 /// │├─├───────┤─╮─╭─ [digit] ─╮─ "." ─╭─ [digit] ─╮──╭───╮──────────────────────────────────╭──┤│
165 /// ╰─ "-" ─╯ │ ╰───────────╯ ╰───────────╯ │ │ ╭─ "+" ─╮ │
166 /// ├───────── ╭─ [digit] ─╮─────────────┤ ├─ "e" ─╭─├───────┤──╭─ [digit] ─╮─╯
167 /// │ ╰───────────╯ │ ╰─ "E" ─╯ ╰─ "-" ─╯ ╰───────────╯
168 /// ╰──── "." ─╭─ [digit] ─╮─────────────╯
169 /// ╰───────────╯
170 ///
171 /// <dimension-token>
172 /// │├─ <number-token> ─ <ident-token> ─┤│
173 ///
174 /// ```
175 ///
176 /// ```md
177 ///
178 /// <dimension-token> // Refined for this lexer, not true to the standard.
179 /// │├─ <number-token> ─╮─ <ident-token> ─╭──┤│
180 /// ╰────── "%" ──────╯
181 /// ```
182 ///
183 /// [1]: https://drafts.csswg.org/css-syntax/#dimension-token-diagram
184 /// [2]: https://drafts.csswg.org/css-syntax/#percentage-token-diagram
185 Dimension = 0b0101,
186
187 // Errors (mask as 0b1_XXXX)
188 /// Represents the [<bad-string-token>][1]. This token is a failure to fully lex the [<string-token>][2].
189 ///
190 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-bad-string-token
191 /// [2]: https://drafts.csswg.org/css-syntax/#typedef-string-token
192 BadString = 0b1_1100,
193
194 /// Represents the [<bad-url-token>][1]. This token is a failure to fully lex the [<url-token>][2].
195 ///
196 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-bad-url-token
197 /// [2]: https://drafts.csswg.org/css-syntax/#typedef-url-token
198 BadUrl = 0b1_1101,
199
200 /// These kind are non-standard Bad kinds and never emitted by the Lexer, but can be used by Parsers to denote a
201 /// token that are either:
202 /// - a Token that was unexpected in this position.
203 /// - a Token that was inserted to recover the parser to a known state.
204 BadWhitespace = 0b1_0001,
205 BadComment = 0b1_0010,
206 BadCdcOrCdo = 0b1_0011,
207 BadNumber = 0b1_0100,
208 BadDimension = 0b1_0101,
209 BadIdent = 0b1_1000,
210 BadFunction = 0b1_1001,
211 BadAtKeyword = 0b1_1010,
212 BadHash = 0b1_1011,
213 BadDelim = 0b1_1111,
214
215 // Variable length Ident-like Tokens (mask: 0b0_1XXX)
216 /// Represents the [<ident-token>][1].
217 ///
218 /// ```md
219 ///
220 /// <newline>
221 /// │├──╮─ "\n" ───╭──┤│
222 /// ├─ "\r\n" ─┤
223 /// ├─ "\r" ───┤
224 /// ╰─ "\f" ───╯
225 ///
226 /// <whitespace>
227 /// │├──╮─ " " ─────╭──┤│
228 /// ├─ "\t" ────┤
229 /// ╰─ newline ─╯
230 ///
231 /// <hexdigit>
232 /// │├─ [ 0-9, A-F, a-f ] ─┤│
233 ///
234 ///
235 /// <escape>
236 /// │├─ "\" ─╮───── [not <newline> or <hexdigit>] ───╭─┤│
237 /// ╰─╭── <hexdigit> ─╮──╮────────────────╭─╯
238 /// ╰─ (1-6 times) ─╯ ╰─ <whitespace> ─╯
239 ///
240 /// <ident-token>
241 /// ╭───────────────── "--" ─────────────────────╮ ╭───────────────────────────────────────────╮
242 /// │├─╯─╮───────╭─╮─ [a-z, A-Z, "_", non-ASCII] ─╭─╰──╯─╭─╮─ [a-z, A-Z, 0-9, "_", non-ASCII] ─╭─╮─╰──┤│
243 /// ╰─ "-" ─╯ ╰──────── <escape> ────────────╯ │ ╰──────────── <escape> ─────────────╯ │
244 /// ╰───────────────────────────────────────╯
245 ///
246 /// ```
247 ///
248 /// [1]: https://drafts.csswg.org/css-syntax/#ident-token-diagram
249 Ident = 0b1000,
250
251 /// Represents the [<function-token>][1].
252 ///
253 /// ```md
254 ///
255 /// <newline>
256 /// │├──╮─ "\n" ───╭──┤│
257 /// ├─ "\r\n" ─┤
258 /// ├─ "\r" ───┤
259 /// ╰─ "\f" ───╯
260 ///
261 /// <whitespace>
262 /// │├──╮─ " " ───────╭──┤│
263 /// ├─ "\t" ──────┤
264 /// ╰─ <newline> ─╯
265 ///
266 /// <hexdigit>
267 /// │├─ [ 0-9, A-F, a-f ] ─┤│
268 ///
269 ///
270 /// <escape>
271 /// │├─ "\" ─╮───── [not <newline> or <hexdigit>] ───╭─┤│
272 /// ╰─╭── <hexdigit> ─╮──╮────────────────╭─╯
273 /// ╰─ (1-6 times) ─╯ ╰─ <whitespace> ─╯
274 ///
275 /// <ident-token>
276 /// ╭───────────────── "--" ─────────────────────╮ ╭───────────────────────────────────────────╮
277 /// │├─╯─╮───────╭─╮─ [a-z, A-Z, "_", non-ASCII] ─╭─╰──╯─╭─╮─ [a-z, A-Z, 0-9, "_", non-ASCII] ─╭─╮─╰──┤│
278 /// ╰─ "-" ─╯ ╰──────── <escape> ────────────╯ │ ╰──────────── <escape> ─────────────╯ │
279 /// ╰───────────────────────────────────────╯
280 ///
281 /// <function-token>
282 /// │├─ <ident-token> ─ "(" ─┤│
283 ///
284 /// ```
285 ///
286 /// [1]: https://drafts.csswg.org/css-syntax/#function-token-diagram
287 Function = 0b1001,
288
289 /// Represents the [<at-keyword-token>][1].
290 ///
291 /// ```md
292 ///
293 /// <newline>
294 /// │├──╮─ "\n" ───╭──┤│
295 /// ├─ "\r\n" ─┤
296 /// ├─ "\r" ───┤
297 /// ╰─ "\f" ───╯
298 ///
299 /// <whitespace>
300 /// │├──╮─ " " ───────╭──┤│
301 /// ├─ "\t" ──────┤
302 /// ╰─ <newline> ─╯
303 ///
304 /// <hexdigit>
305 /// │├─ [ 0-9, A-F, a-f ] ─┤│
306 ///
307 ///
308 /// <escape>
309 /// │├─ "\" ─╮───── [not <newline> or <hexdigit>] ───╭─┤│
310 /// ╰─╭── <hexdigit> ─╮──╮────────────────╭─╯
311 /// ╰─ (1-6 times) ─╯ ╰─ <whitespace> ─╯
312 ///
313 /// <ident-token>
314 /// ╭───────────────── "--" ─────────────────────╮ ╭───────────────────────────────────────────╮
315 /// │├─╯─╮───────╭─╮─ [a-z, A-Z, "_", non-ASCII] ─╭─╰──╯─╭─╮─ [a-z, A-Z, 0-9, "_", non-ASCII] ─╭─╮─╰──┤│
316 /// ╰─ "-" ─╯ ╰──────── <escape> ────────────╯ │ ╰──────────── <escape> ─────────────╯ │
317 /// ╰───────────────────────────────────────╯
318 ///
319 /// <at-keyword-token>
320 /// │├─ "@" ─ <ident-token> ─┤│
321 ///
322 /// ```
323 ///
324 /// [1]: https://drafts.csswg.org/css-syntax/#hash-token-diagram
325 AtKeyword = 0b1010,
326
327 /// Represents the [<hash-token>][1].
328 ///
329 /// ```md
330 ///
331 /// <newline>
332 /// │├──╮─ "\n" ───╭──┤│
333 /// ├─ "\r\n" ─┤
334 /// ├─ "\r" ───┤
335 /// ╰─ "\f" ───╯
336 ///
337 /// <whitespace>
338 /// │├──╮─ " " ───────╭──┤│
339 /// ├─ "\t" ──────┤
340 /// ╰─ <newline> ─╯
341 ///
342 /// <hexdigit>
343 /// │├─ [ 0-9, A-F, a-f ] ─┤│
344 ///
345 ///
346 /// <escape>
347 /// │├─ "\" ─╮───── [not <newline> or <hexdigit>] ───╭─┤│
348 /// ╰─╭── <hexdigit> ─╮──╮────────────────╭─╯
349 /// ╰─ (1-6 times) ─╯ ╰─ <whitespace> ─╯
350 ///
351 /// <hash-token>
352 /// │├─ "#" ──╭─╮─ [a-z, A-Z, 0-9, "_", "-", non-ASCII] ─╭─╮─┤│
353 /// │ ╰─────────────── <escape> ───────────────╯ │
354 /// ╰────────────────────────────────────────────╯
355 ///
356 /// ```
357 ///
358 /// [1]: https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram
359 Hash = 0b1011,
360
361 /// Represents the [<string-token>][1].
362 ///
363 /// ```md
364 ///
365 /// <newline>
366 /// │├──╮─ "\n" ───╭──┤│
367 /// ├─ "\r\n" ─┤
368 /// ├─ "\r" ───┤
369 /// ╰─ "\f" ───╯
370 ///
371 /// <escape>
372 /// │├─ "\" ─╮───── [not <newline> or <hexdigit>] ───╭─┤│
373 /// ╰─╭── <hexdigit> ─╮──╮────────────────╭─╯
374 /// ╰─ (1-6 times) ─╯ ╰─ <whitespace> ─╯
375 ///
376 /// <string-token>
377 /// ╭───────────────────────────────────╮
378 /// │├─╮─ """ ─╯─╭─╮─ [not """, "\", newline] ─╭─╮─╰── """ ─╭─┤│
379 /// │ │ ├──────── <escape> ─────────┤ │ │
380 /// │ │ ╰───── "\" ─ <newline> ─────╯ │ │
381 /// │ ╰───────────────────────────────╯ │
382 /// │ ╭───────────────────────────────────╮ │
383 /// ╰─ "'" ─╯─╭─╮─ [not """, "\", newline] ─╭─╮─╰── "'" ─╯
384 /// │ ├──────── <escape> ─────────┤ │
385 /// │ ╰───── "\" ─ <newline> ─────╯ │
386 /// ╰───────────────────────────────╯
387 ///
388 /// ```
389 ///
390 /// [1]: https://drafts.csswg.org/css-syntax/#string-token-diagram
391 String = 0b1100,
392
393 /// Represents the [<url-token>][1].
394 ///
395 /// ```md
396 ///
397 /// <newline>
398 /// │├──╮─ "\n" ───╭──┤│
399 /// ├─ "\r\n" ─┤
400 /// ├─ "\r" ───┤
401 /// ╰─ "\f" ───╯
402 ///
403 /// <whitespace>
404 /// │├──╮─ " " ───────╭──┤│
405 /// ├─ "\t" ──────┤
406 /// ╰─ <newline> ─╯
407 ///
408 /// <whitespace-token>
409 /// │├─╭─ <whitespace> ─╮─┤│
410 /// ╰────────────────╯
411 ///
412 /// <ws*>
413 /// ╭──────────────────────────╮
414 /// │├─╯─╭─ <whitespace-token> ─╮─╰─┤│
415 /// ╰──────────────────────╯
416 ///
417 /// <hexdigit>
418 /// │├─ [ 0-9, A-F, a-f ] ─┤│
419 ///
420 ///
421 /// <escape>
422 /// │├─ "\" ─╮───── [not <newline> or <hexdigit>] ───╭─┤│
423 /// ╰─╭── <hexdigit> ─╮──╮────────────────╭─╯
424 /// ╰─ (1-6 times) ─╯ ╰─ <whitespace> ─╯
425 ///
426 /// <ident-token>
427 /// ╭───────────────── "--" ─────────────────────╮ ╭───────────────────────────────────────────╮
428 /// │├─╯─╮───────╭─╮─ [a-z, A-Z, "_", non-ASCII] ─╭─╰──╯─╭─╮─ [a-z, A-Z, 0-9, "_", non-ASCII] ─╭─╮─╰──┤│
429 /// ╰─ "-" ─╯ ╰──────── <escape> ────────────╯ │ ╰──────────── <escape> ─────────────╯ │
430 /// ╰───────────────────────────────────────╯
431 ///
432 /// <url-token>
433 /// ╭───────────────────────────────────────────────────────────────────╮
434 /// │├─ <ident-token "url"> ─ "(" ─ <ws*> ─╯─╭─╮─ [not """ "'" "(" ")" "\" <whitespace> or non-printable] ─╭─╮─╰─ <ws*> ─ ")" ─┤│
435 /// │ ╰──────────────────────── <escape> ─────────────────────────╯ │
436 /// ╰───────────────────────────────────────────────────────────────╯
437 ///
438 /// ```
439 ///
440 /// [1]: https://drafts.csswg.org/css-syntax/#url-token-diagram
441 Url = 0b1101,
442
443 /// Represents the [<unicode-range-token>][1]. This token is only produced when the
444 /// [Feature::UnicodeRange][crate::Feature::UnicodeRange] feature is enabled.
445 ///
446 /// ```md
447 ///
448 /// <hexdigit>
449 /// │├─ [ 0-9, A-F, a-f ] ─┤│
450 ///
451 /// <unicode-range-token>
452 /// │├─╮─ 'U' ─╭─ '+' ─╭──────────────────╭── <hexdigit> ─╮──────────────────╭─┤│
453 /// ╰─ 'u' ─╯ │ ╰─ (1-6 times) ─╯ │
454 /// │ ╭───────────────────╮ │
455 /// ├─╯─╭── <hexdigit> ─╮─╰─╭───────────── ? ───────────╮─┤
456 /// │ ╰─ (1-5 times) ─╯ ╰─ (1 to (6 digits) times) ─╯ │
457 /// │ │
458 /// ╰────╭── <hexdigit> ─╮── '-' ──╭── <hexdigit> ─╮──────╯
459 /// ╰─ (1-5 times) ─╯ ╰─ (1-5 times) ─╯
460 /// ```
461 ///
462 /// [1]: https://drafts.csswg.org/css-syntax/#unicode-range-token-diagram
463 UnicodeRange = 0b1110,
464 /// Represents the [<delim-token>][1]. The `<delim-token>` has a value composed of a single code point.
465 ///
466 /// ```md
467 /// <delim-token>
468 /// │├─ [codepoint] ─┤│
469 /// ```
470 ///
471 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-delim-token
472 Delim = 0b1111,
473
474 // Single character Tokens (mask 0b11_XXXX)
475 /// Represents the [<colon-token>][1].
476 ///
477 /// ```md
478 /// <colon-token>
479 /// │├─ ":" ─┤│
480 /// ```
481 ///
482 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-colon-token
483 Colon = 0b10_0001,
484
485 /// Represents the [<semicolon-token>][1].
486 ///
487 /// ```md
488 /// <semicolon-token>
489 /// │├─ ";" ─┤│
490 /// ```
491 ///
492 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-semicolon-token
493 Semicolon = 0b10_0010,
494
495 /// Represents the [<comma-token>][1].
496 ///
497 /// ```md
498 /// <comma-token>
499 /// │├─ "," ─┤│
500 /// ```
501 ///
502 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-comma-token
503 Comma = 0b10_0011,
504
505 /// Represents the [<\[-token>][1].
506 ///
507 /// ```md
508 /// <[-token>
509 /// │├─ "[" ─┤│
510 /// ```
511 ///
512 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-open-square
513 LeftSquare = 0b10_0100,
514
515 /// Represents the [<\]-token>][1].
516 ///
517 /// ```md
518 /// <]-token>
519 /// │├─ "]" ─┤│
520 /// ```
521 ///
522 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-close-square
523 RightSquare = 0b10_0101,
524
525 /// Represents the [<(-token>][1].
526 ///
527 /// ```md
528 /// <(-token>
529 /// │├─ "(" ─┤│
530 /// ```
531 ///
532 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-open-paren
533 LeftParen = 0b10_0110,
534
535 /// Represents the [<)-token>][1].
536 ///
537 /// ```md
538 /// <)-token>
539 /// │├─ ")" ─┤│
540 /// ```
541 ///
542 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-close-paren
543 RightParen = 0b10_0111,
544
545 /// Represents the [<{-token>][1].
546 ///
547 /// ```md
548 /// <{-token>
549 /// │├─ "{" ─┤│
550 /// ```
551 ///
552 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-open-curly
553 LeftCurly = 0b10_1000,
554
555 /// Represents the [<}-token>][1].
556 ///
557 /// ```md
558 /// <}-token>
559 /// │├─ "}" ─┤│
560 /// ```
561 ///
562 /// [1]: https://drafts.csswg.org/css-syntax/#typedef-close-curly
563 RightCurly = 0b10_1001,
564
565 /// These kind are non-standard Bad kinds and never emitted by the Lexer, but can be used by Parsers to denote a
566 /// token that are either:
567 /// - a Token that was unexpected in this position.
568 /// - a Token that was inserted to recover the parser to a known state.
569 BadColon = 0b11_0001,
570 BadSemicolon = 0b11_0010,
571 BadComma = 0b11_0011,
572 BadLeftSquare = 0b11_0100,
573 BadRightSquare = 0b11_0101,
574 BadLeftParen = 0b11_0110,
575 BadRightParen = 0b11_0111,
576 BadLeftCurly = 0b11_1000,
577 BadRightCurly = 0b11_1001,
578}
579
580impl Kind {
581 pub(crate) const fn from_bits(bits: u8) -> Self {
582 match bits {
583 0b0001 => Self::Whitespace,
584 0b0010 => Self::Comment,
585 0b0011 => Self::CdcOrCdo,
586 0b0100 => Self::Number,
587 0b0101 => Self::Dimension,
588 // 0b0110 => Reserved
589 // 0b0111 => Reserved
590 0b1000 => Self::Ident,
591 0b1001 => Self::Function,
592 0b1010 => Self::AtKeyword,
593 0b1011 => Self::Hash,
594 0b1100 => Self::String,
595 0b1101 => Self::Url,
596 0b1110 => Self::UnicodeRange,
597 0b1111 => Self::Delim,
598
599 // Error tokens are represented in 5 bits.
600 0b1_0001 => Self::BadWhitespace,
601 0b1_0010 => Self::BadComment,
602 0b1_0011 => Self::BadCdcOrCdo,
603 0b1_0100 => Self::BadNumber,
604 0b1_0101 => Self::BadDimension,
605 // 0b1_0110 => Self::Reserved,
606 // 0b1_0111 => Self::Reserved,
607 0b1_1000 => Self::BadIdent,
608 0b1_1001 => Self::BadFunction,
609 0b1_1010 => Self::BadAtKeyword,
610 0b1_1011 => Self::BadHash,
611 0b1_1100 => Self::BadString,
612 0b1_1101 => Self::BadUrl,
613 0b1_1110 => Self::UnicodeRange,
614 0b1_1111 => Self::BadDelim,
615
616 // Single character delimiters are represented in 6 bits
617 0b10_0001 => Self::Colon,
618 0b10_0010 => Self::Semicolon,
619 0b10_0011 => Self::Comma,
620 0b10_0100 => Self::LeftSquare,
621 0b10_0101 => Self::RightSquare,
622 0b10_0110 => Self::LeftParen,
623 0b10_0111 => Self::RightParen,
624 0b10_1000 => Self::LeftCurly,
625 0b10_1001 => Self::RightCurly,
626
627 0b11_0001 => Self::BadColon,
628 0b11_0010 => Self::BadSemicolon,
629 0b11_0011 => Self::BadComma,
630 0b11_0100 => Self::BadLeftSquare,
631 0b11_0101 => Self::BadRightSquare,
632 0b11_0110 => Self::BadLeftParen,
633 0b11_0111 => Self::BadRightParen,
634 0b11_1000 => Self::BadLeftCurly,
635 0b11_1001 => Self::BadRightCurly,
636 _ => Self::Eof,
637 }
638 }
639
640 #[doc(hidden)]
641 pub const fn as_str(&self) -> &'static str {
642 match *self {
643 Kind::Eof => "Eof",
644 Kind::Whitespace => "Whitespace",
645 Kind::Comment => "Comment",
646 Kind::CdcOrCdo => "CdcOrCdo",
647 Kind::Number => "Number",
648 Kind::Dimension => "Dimension",
649 Kind::Ident => "Ident",
650 Kind::Function => "Function",
651 Kind::AtKeyword => "AtKeyword",
652 Kind::Hash => "Hash",
653 Kind::String => "String",
654 Kind::Url => "Url",
655 Kind::UnicodeRange => "UnicodeRange",
656 Kind::Delim => "Delim",
657
658 Kind::BadWhitespace => "BadWhitespace",
659 Kind::BadComment => "BadComment",
660 Kind::BadCdcOrCdo => "BadCdcOrCdo",
661 Kind::BadNumber => "BadNumber",
662 Kind::BadDimension => "BadDimension",
663 Kind::BadIdent => "BadIdent",
664 Kind::BadFunction => "BadFunction",
665 Kind::BadAtKeyword => "BadAtKeyword",
666 Kind::BadHash => "BadHash",
667 Kind::BadString => "BadString",
668 Kind::BadUrl => "BadUrl",
669 Kind::BadDelim => "BadDelim",
670
671 Kind::Colon => "Colon",
672 Kind::Semicolon => "Semicolon",
673 Kind::Comma => "Comma",
674 Kind::LeftSquare => "LeftSquare",
675 Kind::RightSquare => "RightSquare",
676 Kind::LeftParen => "LeftParen",
677 Kind::RightParen => "RightParen",
678 Kind::LeftCurly => "LeftCurly",
679 Kind::RightCurly => "RightCurly",
680
681 Kind::BadColon => "BadColon",
682 Kind::BadSemicolon => "BadSemicolon",
683 Kind::BadComma => "BadComma",
684 Kind::BadLeftSquare => "BadLeftSquare",
685 Kind::BadRightSquare => "BadRightSquare",
686 Kind::BadLeftParen => "BadLeftParen",
687 Kind::BadRightParen => "BadRightParen",
688 Kind::BadLeftCurly => "BadLeftCurly",
689 Kind::BadRightCurly => "BadRightCurly",
690 }
691 }
692
693 pub const fn is_bad(&self) -> bool {
694 (*self as u8) & 0b11_0000 == 0b01_0000
695 }
696}
697
698impl fmt::Debug for Kind {
699 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
700 write!(f, "Kind::{}", self.as_str())
701 }
702}
703
704impl fmt::Display for Kind {
705 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
706 write!(f, "Kind::{}", self.as_str())
707 }
708}
709
710impl PartialEq<KindSet> for Kind {
711 fn eq(&self, other: &KindSet) -> bool {
712 other.contains_bits(*self as u8)
713 }
714}
715
716#[test]
717fn test_from_bits() {
718 assert_eq!(Kind::from_bits(Kind::Eof as u8), Kind::Eof);
719 assert_eq!(Kind::from_bits(Kind::Whitespace as u8), Kind::Whitespace);
720 assert_eq!(Kind::from_bits(Kind::Comment as u8), Kind::Comment);
721 assert_eq!(Kind::from_bits(Kind::CdcOrCdo as u8), Kind::CdcOrCdo);
722 assert_eq!(Kind::from_bits(Kind::Number as u8), Kind::Number);
723 assert_eq!(Kind::from_bits(Kind::Dimension as u8), Kind::Dimension);
724 assert_eq!(Kind::from_bits(Kind::Ident as u8), Kind::Ident);
725 assert_eq!(Kind::from_bits(Kind::Function as u8), Kind::Function);
726 assert_eq!(Kind::from_bits(Kind::AtKeyword as u8), Kind::AtKeyword);
727 assert_eq!(Kind::from_bits(Kind::Hash as u8), Kind::Hash);
728 assert_eq!(Kind::from_bits(Kind::String as u8), Kind::String);
729 assert_eq!(Kind::from_bits(Kind::Url as u8), Kind::Url);
730 assert_eq!(Kind::from_bits(Kind::UnicodeRange as u8), Kind::UnicodeRange);
731 assert_eq!(Kind::from_bits(Kind::Delim as u8), Kind::Delim);
732
733 assert_eq!(Kind::from_bits(Kind::BadWhitespace as u8), Kind::BadWhitespace);
734 assert_eq!(Kind::from_bits(Kind::BadComment as u8), Kind::BadComment);
735 assert_eq!(Kind::from_bits(Kind::BadCdcOrCdo as u8), Kind::BadCdcOrCdo);
736 assert_eq!(Kind::from_bits(Kind::BadNumber as u8), Kind::BadNumber);
737 assert_eq!(Kind::from_bits(Kind::BadDimension as u8), Kind::BadDimension);
738 assert_eq!(Kind::from_bits(Kind::BadIdent as u8), Kind::BadIdent);
739 assert_eq!(Kind::from_bits(Kind::BadFunction as u8), Kind::BadFunction);
740 assert_eq!(Kind::from_bits(Kind::BadAtKeyword as u8), Kind::BadAtKeyword);
741 assert_eq!(Kind::from_bits(Kind::BadHash as u8), Kind::BadHash);
742 assert_eq!(Kind::from_bits(Kind::BadString as u8), Kind::BadString);
743 assert_eq!(Kind::from_bits(Kind::BadUrl as u8), Kind::BadUrl);
744 assert_eq!(Kind::from_bits(Kind::BadDelim as u8), Kind::BadDelim);
745
746 assert_eq!(Kind::from_bits(Kind::Colon as u8), Kind::Colon);
747 assert_eq!(Kind::from_bits(Kind::Semicolon as u8), Kind::Semicolon);
748 assert_eq!(Kind::from_bits(Kind::Comma as u8), Kind::Comma);
749 assert_eq!(Kind::from_bits(Kind::LeftSquare as u8), Kind::LeftSquare);
750 assert_eq!(Kind::from_bits(Kind::RightSquare as u8), Kind::RightSquare);
751 assert_eq!(Kind::from_bits(Kind::LeftParen as u8), Kind::LeftParen);
752 assert_eq!(Kind::from_bits(Kind::RightParen as u8), Kind::RightParen);
753 assert_eq!(Kind::from_bits(Kind::LeftCurly as u8), Kind::LeftCurly);
754 assert_eq!(Kind::from_bits(Kind::RightCurly as u8), Kind::RightCurly);
755
756 assert_eq!(Kind::from_bits(Kind::BadColon as u8), Kind::BadColon);
757 assert_eq!(Kind::from_bits(Kind::BadSemicolon as u8), Kind::BadSemicolon);
758 assert_eq!(Kind::from_bits(Kind::BadComma as u8), Kind::BadComma);
759 assert_eq!(Kind::from_bits(Kind::BadLeftSquare as u8), Kind::BadLeftSquare);
760 assert_eq!(Kind::from_bits(Kind::BadRightSquare as u8), Kind::BadRightSquare);
761 assert_eq!(Kind::from_bits(Kind::BadLeftParen as u8), Kind::BadLeftParen);
762 assert_eq!(Kind::from_bits(Kind::BadRightParen as u8), Kind::BadRightParen);
763 assert_eq!(Kind::from_bits(Kind::BadLeftCurly as u8), Kind::BadLeftCurly);
764 assert_eq!(Kind::from_bits(Kind::BadRightCurly as u8), Kind::BadRightCurly);
765}
766
767#[test]
768fn size_test() {
769 assert_eq!(::std::mem::size_of::<Kind>(), 1);
770}