1pub use css_lexer::AtomSet;
2use derive_atom_set::AtomSet as DeriveAtomSet;
3
4#[allow(non_camel_case_types)]
15#[derive(DeriveAtomSet, Debug, Default, Copy, Clone, PartialEq, Eq)]
16pub enum EncodingLabel {
17 #[default]
18 Unknown,
19
20 _866,
21 #[atom("ansi_x3.4-1968")]
22 Ansi_u_x3_d_4_1968,
23 Arabic,
24 Ascii,
25 Asmo_708,
26 Big5,
27 Big5_hkscs,
28 Chinese,
29 Cn_big5,
30 Cp1250,
31 Cp1251,
32 Cp1252,
33 Cp1253,
34 Cp1254,
35 Cp1255,
36 Cp1256,
37 Cp1257,
38 Cp1258,
39 Cp819,
40 Cp866,
41 Csbig5,
42 Cseuckr,
43 Cseucpkdfmtjapanese,
44 Csgb2312,
45 Csibm866,
46 Csiso2022jp,
47 Csiso2022kr,
48 Csiso58gb231280,
49 Csiso88596e,
50 Csiso88596i,
51 Csiso88598e,
52 Csiso88598i,
53 Csisolatin1,
54 Csisolatin2,
55 Csisolatin3,
56 Csisolatin4,
57 Csisolatin5,
58 Csisolatin6,
59 Csisolatin9,
60 Csisolatinarabic,
61 Csisolatincyrillic,
62 Csisolatingreek,
63 Csisolatinhebrew,
64 Cskoi8r,
65 Csksc56011987,
66 Csmacintosh,
67 Csshiftjis,
68 Csunicode,
69 Cyrillic,
70 Dos_874,
71 Ecma_114,
72 Ecma_118,
73 #[atom("elot_928")]
74 Elot_u_928,
75 Euc_jp,
76 Euc_kr,
77 Gb18030,
78 Gb2312,
79 #[atom("gb_2312")]
80 Gb_u_2312,
81 #[atom("gb_2312-80")]
82 Gb_u_2312_80,
83 Gbk,
84 Greek,
85 Greek8,
86 Hebrew,
87 Hz_gb_2312,
88 Ibm819,
89 Ibm866,
90 Iso_10646_ucs_2,
91 Iso_2022_cn,
92 Iso_2022_cn_ext,
93 Iso_2022_jp,
94 Iso_2022_kr,
95 Iso_8859_1,
96 Iso_8859_10,
97 Iso_8859_11,
98 Iso_8859_13,
99 Iso_8859_14,
100 Iso_8859_15,
101 Iso_8859_16,
102 Iso_8859_2,
103 Iso_8859_3,
104 Iso_8859_4,
105 Iso_8859_5,
106 Iso_8859_6,
107 Iso_8859_6_e,
108 Iso_8859_6_i,
109 Iso_8859_7,
110 Iso_8859_8,
111 Iso_8859_8_e,
112 Iso_8859_8_i,
113 Iso_8859_9,
114 Iso_ir_100,
115 Iso_ir_101,
116 Iso_ir_109,
117 Iso_ir_110,
118 Iso_ir_126,
119 Iso_ir_127,
120 Iso_ir_138,
121 Iso_ir_144,
122 Iso_ir_148,
123 Iso_ir_149,
124 Iso_ir_157,
125 Iso_ir_58,
126 Iso8859_1,
127 Iso8859_10,
128 Iso8859_11,
129 Iso8859_13,
130 Iso8859_14,
131 Iso8859_15,
132 Iso8859_2,
133 Iso8859_3,
134 Iso8859_4,
135 Iso8859_5,
136 Iso8859_6,
137 Iso8859_7,
138 Iso8859_8,
139 Iso8859_9,
140 Iso88591,
141 Iso885910,
142 Iso885911,
143 Iso885913,
144 Iso885914,
145 Iso885915,
146 Iso88592,
147 Iso88593,
148 Iso88594,
149 Iso88595,
150 Iso88596,
151 Iso88597,
152 Iso88598,
153 Iso88599,
154 #[atom("iso_8859-1")]
155 Iso_u_8859_1,
156 #[atom("iso_8859-15")]
157 Iso_u_8859_15,
158 #[atom("iso_8859-1:1987")]
159 Iso_u_8859_1_c_1987,
160 #[atom("iso_8859-2")]
161 Iso_u_8859_2,
162 #[atom("iso_8859-2:1987")]
163 Iso_u_8859_2_c_1987,
164 #[atom("iso_8859-3")]
165 Iso_u_8859_3,
166 #[atom("iso_8859-3:1988")]
167 Iso_u_8859_3_c_1988,
168 #[atom("iso_8859-4")]
169 Iso_u_8859_4,
170 #[atom("iso_8859-4:1988")]
171 Iso_u_8859_4_c_1988,
172 #[atom("iso_8859-5")]
173 Iso_u_8859_5,
174 #[atom("iso_8859-5:1988")]
175 Iso_u_8859_5_c_1988,
176 #[atom("iso_8859-6")]
177 Iso_u_8859_6,
178 #[atom("iso_8859-6:1987")]
179 Iso_u_8859_6_c_1987,
180 #[atom("iso_8859-7")]
181 Iso_u_8859_7,
182 #[atom("iso_8859-7:1987")]
183 Iso_u_8859_7_c_1987,
184 #[atom("iso_8859-8")]
185 Iso_u_8859_8,
186 #[atom("iso_8859-8:1988")]
187 Iso_u_8859_8_c_1988,
188 #[atom("iso_8859-9")]
189 Iso_u_8859_9,
190 #[atom("iso_8859-9:1989")]
191 Iso_u_8859_9_c_1989,
192 Koi,
193 Koi8,
194 Koi8_r,
195 Koi8_ru,
196 Koi8_u,
197 #[atom("koi8_r")]
198 Koi8_u_r,
199 Korean,
200 #[atom("ks_c_5601-1987")]
201 Ks_u_c_u_5601_1987,
202 #[atom("ks_c_5601-1989")]
203 Ks_u_c_u_5601_1989,
204 Ksc5601,
205 #[atom("ksc_5601")]
206 Ksc_u_5601,
207 L1,
208 L2,
209 L3,
210 L4,
211 L5,
212 L6,
213 L9,
214 Latin1,
215 Latin2,
216 Latin3,
217 Latin4,
218 Latin5,
219 Latin6,
220 Logical,
221 Mac,
222 Macintosh,
223 Ms932,
224 #[atom("ms_kanji")]
225 Ms_u_kanji,
226 Replacement,
227 Shift_jis,
228 #[atom("shift_jis")]
229 Shift_u_jis,
230 Sjis,
231 #[atom("sun_eu_greek")]
232 Sun_u_eu_u_greek,
233 Tis_620,
234 Ucs_2,
235 Unicode,
236 Unicode_1_1_utf_8,
237 Unicode11utf8,
238 Unicode20utf8,
239 Unicodefeff,
240 Unicodefffe,
241 Us_ascii,
242 Utf_16,
243 Utf_16be,
244 Utf_16le,
245 Utf_8,
246 Utf8,
247 Visual,
248 Windows_1250,
249 Windows_1251,
250 Windows_1252,
251 Windows_1253,
252 Windows_1254,
253 Windows_1255,
254 Windows_1256,
255 Windows_1257,
256 Windows_1258,
257 Windows_31j,
258 Windows_874,
259 Windows_949,
260 X_cp1250,
261 X_cp1251,
262 X_cp1252,
263 X_cp1253,
264 X_cp1254,
265 X_cp1255,
266 X_cp1256,
267 X_cp1257,
268 X_cp1258,
269 X_euc_jp,
270 X_gbk,
271 X_mac_cyrillic,
272 X_mac_roman,
273 X_mac_ukrainian,
274 X_sjis,
275 X_unicode20utf8,
276 X_user_defined,
277 X_x_big5,
278}
279
280impl EncodingLabel {
281 const COMPACT: [Self; 229] = [
283 Self::Unknown,
284 Self::_866,
285 Self::L1,
286 Self::Arabic,
287 Self::L1,
288 Self::Arabic,
289 Self::Big5,
290 Self::Big5,
291 Self::Gbk,
292 Self::Big5,
293 Self::Cp1250,
294 Self::Cp1251,
295 Self::L1,
296 Self::Cp1253,
297 Self::L5,
298 Self::Cp1255,
299 Self::Cp1256,
300 Self::Cp1257,
301 Self::Cp1258,
302 Self::L1,
303 Self::_866,
304 Self::Big5,
305 Self::Euc_kr,
306 Self::Euc_jp,
307 Self::Gbk,
308 Self::_866,
309 Self::Csiso2022jp,
310 Self::Hz_gb_2312,
311 Self::Gbk,
312 Self::Arabic,
313 Self::Arabic,
314 Self::Hebrew,
315 Self::Logical,
316 Self::L1,
317 Self::L2,
318 Self::L3,
319 Self::L4,
320 Self::L5,
321 Self::L6,
322 Self::L9,
323 Self::Arabic,
324 Self::Cyrillic,
325 Self::Greek,
326 Self::Hebrew,
327 Self::Koi,
328 Self::Euc_kr,
329 Self::Mac,
330 Self::Sjis,
331 Self::Unknown,
332 Self::Cyrillic,
333 Self::Dos_874,
334 Self::Arabic,
335 Self::Greek,
336 Self::Greek,
337 Self::Euc_jp,
338 Self::Euc_kr,
339 Self::Gb18030,
340 Self::Gbk,
341 Self::Gbk,
342 Self::Gbk,
343 Self::Gbk,
344 Self::Greek,
345 Self::Greek,
346 Self::Hebrew,
347 Self::Hz_gb_2312,
348 Self::L1,
349 Self::_866,
350 Self::Unknown,
351 Self::Hz_gb_2312,
352 Self::Hz_gb_2312,
353 Self::Csiso2022jp,
354 Self::Hz_gb_2312,
355 Self::L1,
356 Self::L6,
357 Self::Dos_874,
358 Self::Iso885913,
359 Self::Iso885914,
360 Self::L9,
361 Self::Iso_8859_16,
362 Self::L2,
363 Self::L3,
364 Self::L4,
365 Self::Cyrillic,
366 Self::Arabic,
367 Self::Arabic,
368 Self::Arabic,
369 Self::Greek,
370 Self::Hebrew,
371 Self::Hebrew,
372 Self::Logical,
373 Self::L5,
374 Self::L1,
375 Self::L2,
376 Self::L3,
377 Self::L4,
378 Self::Greek,
379 Self::Arabic,
380 Self::Hebrew,
381 Self::Cyrillic,
382 Self::L5,
383 Self::Euc_kr,
384 Self::L6,
385 Self::Gbk,
386 Self::L1,
387 Self::L6,
388 Self::Dos_874,
389 Self::Iso885913,
390 Self::Iso885914,
391 Self::L9,
392 Self::L2,
393 Self::L3,
394 Self::L4,
395 Self::Cyrillic,
396 Self::Arabic,
397 Self::Greek,
398 Self::Hebrew,
399 Self::L5,
400 Self::L1,
401 Self::L6,
402 Self::Dos_874,
403 Self::Iso885913,
404 Self::Iso885914,
405 Self::L9,
406 Self::L2,
407 Self::L3,
408 Self::L4,
409 Self::Cyrillic,
410 Self::Arabic,
411 Self::Greek,
412 Self::Hebrew,
413 Self::L5,
414 Self::L1,
415 Self::L9,
416 Self::L1,
417 Self::L2,
418 Self::L2,
419 Self::L3,
420 Self::L3,
421 Self::L4,
422 Self::L4,
423 Self::Cyrillic,
424 Self::Cyrillic,
425 Self::Arabic,
426 Self::Arabic,
427 Self::Greek,
428 Self::Greek,
429 Self::Hebrew,
430 Self::Hebrew,
431 Self::L5,
432 Self::L5,
433 Self::Koi,
434 Self::Koi,
435 Self::Koi,
436 Self::Koi8_u,
437 Self::Koi8_u,
438 Self::Koi,
439 Self::Euc_kr,
440 Self::Euc_kr,
441 Self::Euc_kr,
442 Self::Euc_kr,
443 Self::Euc_kr,
444 Self::L1,
445 Self::L2,
446 Self::L3,
447 Self::L4,
448 Self::L5,
449 Self::L6,
450 Self::L9,
451 Self::L1,
452 Self::L2,
453 Self::L3,
454 Self::L4,
455 Self::L5,
456 Self::L6,
457 Self::Logical,
458 Self::Mac,
459 Self::Mac,
460 Self::Sjis,
461 Self::Sjis,
462 Self::Hz_gb_2312,
463 Self::Sjis,
464 Self::Sjis,
465 Self::Sjis,
466 Self::Greek,
467 Self::Dos_874,
468 Self::Unknown,
469 Self::Unknown,
470 Self::Unknown,
471 Self::Unknown,
472 Self::Unknown,
473 Self::Unknown,
474 Self::Unknown,
475 Self::L1,
476 Self::Unknown,
477 Self::Unknown,
478 Self::Unknown,
479 Self::Unknown,
480 Self::Unknown,
481 Self::Hebrew,
482 Self::Cp1250,
483 Self::Cp1251,
484 Self::L1,
485 Self::Cp1253,
486 Self::L5,
487 Self::Cp1255,
488 Self::Cp1256,
489 Self::Cp1257,
490 Self::Cp1258,
491 Self::Sjis,
492 Self::Dos_874,
493 Self::Euc_kr,
494 Self::Cp1250,
495 Self::Cp1251,
496 Self::L1,
497 Self::Cp1253,
498 Self::L5,
499 Self::Cp1255,
500 Self::Cp1256,
501 Self::Cp1257,
502 Self::Cp1258,
503 Self::Euc_jp,
504 Self::Gbk,
505 Self::X_mac_cyrillic,
506 Self::Mac,
507 Self::X_mac_cyrillic,
508 Self::Sjis,
509 Self::Unknown,
510 Self::X_user_defined,
511 Self::Big5,
512 ];
513
514 pub fn from_label(label: &str) -> Self {
525 Self::from_str(label.trim_matches(|c| matches!(c, '\t' | '\n' | '\x0C' | '\r' | ' ')))
526 }
527
528 pub fn compact(self) -> Self {
543 Self::COMPACT[self as usize]
544 }
545}
546
547#[cfg(test)]
548mod tests {
549 use super::{AtomSet, EncodingLabel};
550
551 const ALL_LABELS: [&str; 228] = [
553 "866",
554 "ansi_x3.4-1968",
555 "arabic",
556 "ascii",
557 "asmo-708",
558 "big5",
559 "big5-hkscs",
560 "chinese",
561 "cn-big5",
562 "cp1250",
563 "cp1251",
564 "cp1252",
565 "cp1253",
566 "cp1254",
567 "cp1255",
568 "cp1256",
569 "cp1257",
570 "cp1258",
571 "cp819",
572 "cp866",
573 "csbig5",
574 "cseuckr",
575 "cseucpkdfmtjapanese",
576 "csgb2312",
577 "csibm866",
578 "csiso2022jp",
579 "csiso2022kr",
580 "csiso58gb231280",
581 "csiso88596e",
582 "csiso88596i",
583 "csiso88598e",
584 "csiso88598i",
585 "csisolatin1",
586 "csisolatin2",
587 "csisolatin3",
588 "csisolatin4",
589 "csisolatin5",
590 "csisolatin6",
591 "csisolatin9",
592 "csisolatinarabic",
593 "csisolatincyrillic",
594 "csisolatingreek",
595 "csisolatinhebrew",
596 "cskoi8r",
597 "csksc56011987",
598 "csmacintosh",
599 "csshiftjis",
600 "csunicode",
601 "cyrillic",
602 "dos-874",
603 "ecma-114",
604 "ecma-118",
605 "elot_928",
606 "euc-jp",
607 "euc-kr",
608 "gb18030",
609 "gb2312",
610 "gb_2312",
611 "gb_2312-80",
612 "gbk",
613 "greek",
614 "greek8",
615 "hebrew",
616 "hz-gb-2312",
617 "ibm819",
618 "ibm866",
619 "iso-10646-ucs-2",
620 "iso-2022-cn",
621 "iso-2022-cn-ext",
622 "iso-2022-jp",
623 "iso-2022-kr",
624 "iso-8859-1",
625 "iso-8859-10",
626 "iso-8859-11",
627 "iso-8859-13",
628 "iso-8859-14",
629 "iso-8859-15",
630 "iso-8859-16",
631 "iso-8859-2",
632 "iso-8859-3",
633 "iso-8859-4",
634 "iso-8859-5",
635 "iso-8859-6",
636 "iso-8859-6-e",
637 "iso-8859-6-i",
638 "iso-8859-7",
639 "iso-8859-8",
640 "iso-8859-8-e",
641 "iso-8859-8-i",
642 "iso-8859-9",
643 "iso-ir-100",
644 "iso-ir-101",
645 "iso-ir-109",
646 "iso-ir-110",
647 "iso-ir-126",
648 "iso-ir-127",
649 "iso-ir-138",
650 "iso-ir-144",
651 "iso-ir-148",
652 "iso-ir-149",
653 "iso-ir-157",
654 "iso-ir-58",
655 "iso8859-1",
656 "iso8859-10",
657 "iso8859-11",
658 "iso8859-13",
659 "iso8859-14",
660 "iso8859-15",
661 "iso8859-2",
662 "iso8859-3",
663 "iso8859-4",
664 "iso8859-5",
665 "iso8859-6",
666 "iso8859-7",
667 "iso8859-8",
668 "iso8859-9",
669 "iso88591",
670 "iso885910",
671 "iso885911",
672 "iso885913",
673 "iso885914",
674 "iso885915",
675 "iso88592",
676 "iso88593",
677 "iso88594",
678 "iso88595",
679 "iso88596",
680 "iso88597",
681 "iso88598",
682 "iso88599",
683 "iso_8859-1",
684 "iso_8859-15",
685 "iso_8859-1:1987",
686 "iso_8859-2",
687 "iso_8859-2:1987",
688 "iso_8859-3",
689 "iso_8859-3:1988",
690 "iso_8859-4",
691 "iso_8859-4:1988",
692 "iso_8859-5",
693 "iso_8859-5:1988",
694 "iso_8859-6",
695 "iso_8859-6:1987",
696 "iso_8859-7",
697 "iso_8859-7:1987",
698 "iso_8859-8",
699 "iso_8859-8:1988",
700 "iso_8859-9",
701 "iso_8859-9:1989",
702 "koi",
703 "koi8",
704 "koi8-r",
705 "koi8-ru",
706 "koi8-u",
707 "koi8_r",
708 "korean",
709 "ks_c_5601-1987",
710 "ks_c_5601-1989",
711 "ksc5601",
712 "ksc_5601",
713 "l1",
714 "l2",
715 "l3",
716 "l4",
717 "l5",
718 "l6",
719 "l9",
720 "latin1",
721 "latin2",
722 "latin3",
723 "latin4",
724 "latin5",
725 "latin6",
726 "logical",
727 "mac",
728 "macintosh",
729 "ms932",
730 "ms_kanji",
731 "replacement",
732 "shift-jis",
733 "shift_jis",
734 "sjis",
735 "sun_eu_greek",
736 "tis-620",
737 "ucs-2",
738 "unicode",
739 "unicode-1-1-utf-8",
740 "unicode11utf8",
741 "unicode20utf8",
742 "unicodefeff",
743 "unicodefffe",
744 "us-ascii",
745 "utf-16",
746 "utf-16be",
747 "utf-16le",
748 "utf-8",
749 "utf8",
750 "visual",
751 "windows-1250",
752 "windows-1251",
753 "windows-1252",
754 "windows-1253",
755 "windows-1254",
756 "windows-1255",
757 "windows-1256",
758 "windows-1257",
759 "windows-1258",
760 "windows-31j",
761 "windows-874",
762 "windows-949",
763 "x-cp1250",
764 "x-cp1251",
765 "x-cp1252",
766 "x-cp1253",
767 "x-cp1254",
768 "x-cp1255",
769 "x-cp1256",
770 "x-cp1257",
771 "x-cp1258",
772 "x-euc-jp",
773 "x-gbk",
774 "x-mac-cyrillic",
775 "x-mac-roman",
776 "x-mac-ukrainian",
777 "x-sjis",
778 "x-unicode20utf8",
779 "x-user-defined",
780 "x-x-big5",
781 ];
782
783 #[test]
784 fn every_label_resolves_to_its_own_variant() {
785 for (i, label) in ALL_LABELS.iter().enumerate() {
786 assert_eq!(
787 EncodingLabel::from_label(label).as_bits() as usize,
788 i + 1,
789 "{label} did not resolve to its own variant",
790 );
791 }
792 }
793
794 #[test]
795 fn compact_is_idempotent() {
796 for label in EncodingLabel::COMPACT {
797 assert_eq!(label.compact(), label, "{} is not its own shortest label", label.to_str());
798 }
799 }
800
801 #[test]
802 fn compact_never_lengthens_a_label() {
803 for label in ALL_LABELS.iter().map(|l| EncodingLabel::from_label(l)) {
804 let compact = label.compact();
805 if compact != EncodingLabel::Unknown {
806 assert!(compact.len() <= label.len(), "{} is longer than {}", compact.to_str(), label.to_str());
807 }
808 }
809 }
810
811 #[test]
812 fn shortens_windows_1252_aliases() {
813 assert_eq!(EncodingLabel::from_label("ISO-8859-1").compact().to_str(), "l1");
814 assert_eq!(EncodingLabel::from_label("us-ascii").compact().to_str(), "l1");
815 assert_eq!(EncodingLabel::from_label("l1").compact().to_str(), "l1");
816 }
817
818 #[test]
819 fn separators_are_significant() {
820 assert_eq!(EncodingLabel::from_label("iso_8859-15").compact().to_str(), "l9");
821 assert_eq!(EncodingLabel::from_label("iso8859-15").compact().to_str(), "l9");
822 assert_eq!(EncodingLabel::from_label("iso-8859-15").compact().to_str(), "l9");
823 assert_eq!(EncodingLabel::from_label("iso-8859_15"), EncodingLabel::Unknown);
824 }
825
826 #[test]
827 fn drops_utf8_and_utf16_labels() {
828 assert_eq!(EncodingLabel::from_label("UTF-8").compact(), EncodingLabel::Unknown);
829 assert_eq!(EncodingLabel::from_label("utf-16").compact(), EncodingLabel::Unknown);
830 assert_eq!(EncodingLabel::from_label("utf-16be").compact(), EncodingLabel::Unknown);
831 assert_eq!(EncodingLabel::from_label("unicode-1-1-utf-8").compact(), EncodingLabel::Unknown);
832 }
833
834 #[test]
835 fn drops_unknown_labels() {
836 assert_eq!(EncodingLabel::from_label("not-a-charset"), EncodingLabel::Unknown);
837 assert_eq!(EncodingLabel::from_label(""), EncodingLabel::Unknown);
838 assert_eq!(EncodingLabel::from_label("a-label-far-longer-than-any-real-one"), EncodingLabel::Unknown);
839 }
840}