source_tools/
cursor.rs

1use crate::{SourceOffset, Span, ToSpan};
2
3/// A token that occupies bytes in source text.
4pub trait SourceToken: Copy {
5	/// Token category.
6	type Kind: Copy;
7
8	/// Token occupying no source bytes.
9	const EMPTY: Self;
10
11	/// Returns token category.
12	fn kind(self) -> Self::Kind;
13
14	/// Returns token length in bytes.
15	fn len(self) -> u32;
16
17	/// Returns token category name.
18	fn kind_name(self) -> &'static str;
19
20	/// Returns bytes before token value.
21	fn leading_len(self) -> u32 {
22		0
23	}
24
25	/// Returns bytes after token value.
26	fn trailing_len(self) -> u32 {
27		0
28	}
29
30	/// Returns whether token occupies no bytes.
31	fn is_empty(self) -> bool {
32		self.len() == 0
33	}
34}
35
36/// A token and its byte offset in source text.
37#[repr(C)]
38#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
39pub struct Cursor<T>(SourceOffset, T);
40
41impl<T> Cursor<T> {
42	/// Places `token` at `offset`.
43	#[inline(always)]
44	pub const fn new(offset: SourceOffset, token: T) -> Self {
45		Self(offset, token)
46	}
47
48	/// Places a synthetic token at [SourceOffset::DUMMY].
49	#[inline(always)]
50	pub const fn dummy(token: T) -> Self {
51		Self(SourceOffset::DUMMY, token)
52	}
53
54	/// Returns token by reference.
55	#[inline(always)]
56	pub const fn token_ref(&self) -> &T {
57		&self.1
58	}
59
60	/// Returns starting offset.
61	#[inline(always)]
62	pub const fn offset(&self) -> SourceOffset {
63		self.0
64	}
65
66	/// Replaces token while preserving offset.
67	#[inline(always)]
68	pub fn with_token<U>(self, token: U) -> Cursor<U> {
69		Cursor(self.0, token)
70	}
71
72	/// Maps token while preserving offset.
73	#[inline(always)]
74	pub fn map_token<U>(self, map: impl FnOnce(T) -> U) -> Cursor<U> {
75		Cursor(self.0, map(self.1))
76	}
77}
78
79impl<T: Copy> Cursor<T> {
80	/// Returns token.
81	#[inline(always)]
82	pub const fn token(&self) -> T {
83		self.1
84	}
85}
86
87impl<T: SourceToken> Cursor<T> {
88	/// Empty cursor at start of source text.
89	pub const EMPTY: Self = Self(SourceOffset::ZERO, T::EMPTY);
90
91	/// Returns token category.
92	#[inline(always)]
93	pub fn kind(&self) -> T::Kind {
94		self.1.kind()
95	}
96
97	/// Returns offset immediately after token.
98	#[inline(always)]
99	pub fn end_offset(&self) -> SourceOffset {
100		if self.0 == SourceOffset::DUMMY {
101			return self.0;
102		}
103		SourceOffset(self.0.0 + self.len())
104	}
105
106	/// Returns token length in bytes.
107	#[inline(always)]
108	pub fn len(&self) -> u32 {
109		self.1.len()
110	}
111
112	/// Returns whether token occupies no bytes.
113	#[inline(always)]
114	pub fn is_empty(&self) -> bool {
115		self.1.is_empty()
116	}
117
118	/// Returns token source span.
119	#[inline(always)]
120	pub fn span(&self) -> Span {
121		Span::new(self.0, self.end_offset())
122	}
123
124	/// Returns source text covered by token.
125	#[inline(always)]
126	pub fn str_slice<'a>(&self, source: &'a str) -> &'a str {
127		let start = self.0.0 as usize;
128		let end = self.end_offset().0 as usize;
129		debug_assert!(source.len() >= end, "attempted to index out of bounds ({} < {})", source.len(), end);
130		&source[start..end]
131	}
132
133	/// Returns token value with leading and trailing syntax removed.
134	pub fn value_slice<'a>(&self, source: &'a str) -> &'a str {
135		let start = (self.0.0 + self.1.leading_len()) as usize;
136		let end = (self.0.0 + self.len() - self.1.trailing_len()) as usize;
137		&source[start..end]
138	}
139}
140
141impl<T: SourceToken> ToSpan for Cursor<T> {
142	fn to_span(&self) -> Span {
143		self.span()
144	}
145}
146
147impl<T: SourceToken> From<Cursor<T>> for Span {
148	fn from(cursor: Cursor<T>) -> Self {
149		cursor.span()
150	}
151}
152
153impl<T: SourceToken> PartialEq<Span> for Cursor<T> {
154	fn eq(&self, other: &Span) -> bool {
155		self.span() == *other
156	}
157}
158
159impl<T: PartialEq<char>> PartialEq<char> for Cursor<T> {
160	fn eq(&self, other: &char) -> bool {
161		self.1 == *other
162	}
163}
164
165impl<T: PartialEq<char>> PartialEq<char> for &Cursor<T> {
166	fn eq(&self, other: &char) -> bool {
167		self.token_ref() == other
168	}
169}
170
171#[cfg(feature = "miette")]
172impl<T: SourceToken> From<Cursor<T>> for miette::SourceSpan {
173	fn from(cursor: Cursor<T>) -> Self {
174		cursor.span().into()
175	}
176}
177
178#[cfg(feature = "serde")]
179impl<T: SourceToken> serde::Serialize for Cursor<T> {
180	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
181	where
182		S: serde::Serializer,
183	{
184		use serde::ser::SerializeStruct;
185		if self.is_empty() {
186			return serializer.serialize_none();
187		}
188		let mut state = serializer.serialize_struct("Cursor", 3)?;
189		state.serialize_field("kind", self.1.kind_name())?;
190		state.serialize_field("offset", &self.0)?;
191		state.serialize_field("len", &self.len())?;
192		state.end()
193	}
194}
195
196#[cfg(test)]
197mod tests {
198	use super::*;
199
200	#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
201	#[repr(C)]
202	struct Token(u32, u32);
203
204	impl SourceToken for Token {
205		type Kind = u32;
206		const EMPTY: Self = Token(0, 0);
207
208		fn kind(self) -> Self::Kind {
209			self.0
210		}
211
212		fn len(self) -> u32 {
213			self.1
214		}
215
216		fn kind_name(self) -> &'static str {
217			"test"
218		}
219	}
220
221	#[test]
222	fn cursor_preserves_layout_and_span() {
223		let cursor = Cursor::new(SourceOffset(3), Token(1, 4));
224		assert_eq!(size_of::<Cursor<Token>>(), 12);
225		assert_eq!(cursor.span(), Span::new(SourceOffset(3), SourceOffset(7)));
226	}
227
228	#[test]
229	fn dummy_cursor_keeps_dummy_end_offset() {
230		let cursor = Cursor::dummy(Token(1, 4));
231		assert_eq!(cursor.end_offset(), SourceOffset::DUMMY);
232	}
233}
source_tools/cursor.rs

source_tools/
cursor.rs
