use std::{iter::Peekable, str::Chars, sync::Arc}; use codemap::File; use crate::Token; const FORM_FEED: char = '\x0C'; #[derive(Debug, Clone)] pub(crate) struct Lexer { buf: Vec, cursor: usize, amt_peeked: usize, } impl Lexer { fn peek_cursor(&self) -> usize { self.cursor + self.amt_peeked } pub fn peek(&self) -> Option { self.buf.get(self.peek_cursor()).copied() } pub fn reset_cursor(&mut self) { self.amt_peeked = 0; } pub fn peek_next(&mut self) -> Option { self.amt_peeked += 1; self.peek() } pub fn peek_previous(&mut self) -> Option { self.buf.get(self.peek_cursor().checked_sub(1)?).copied() } pub fn peek_forward(&mut self, n: usize) -> Option { self.amt_peeked += n; self.peek() } /// Peeks `n` from current peeked position without modifying cursor pub fn peek_n(&self, n: usize) -> Option { self.buf.get(self.peek_cursor() + n).copied() } pub fn peek_backward(&mut self, n: usize) -> Option { self.amt_peeked = self.amt_peeked.checked_sub(n)?; self.peek() } pub fn truncate_iterator_to_cursor(&mut self) { self.cursor += self.amt_peeked; self.amt_peeked = 0; } /// Set cursor to position and reset peek pub fn set_cursor(&mut self, cursor: usize) { self.cursor = cursor; self.amt_peeked = 0; } pub fn cursor(&self) -> usize { self.cursor } } impl Iterator for Lexer { type Item = Token; fn next(&mut self) -> Option { self.buf.get(self.cursor).copied().map(|tok| { self.cursor += 1; self.amt_peeked = self.amt_peeked.saturating_sub(1); tok }) } } struct TokenLexer<'a> { buf: Peekable>, cursor: usize, file: Arc, } impl<'a> Iterator for TokenLexer<'a> { type Item = Token; fn next(&mut self) -> Option { let kind = match self.buf.next()? { FORM_FEED => '\n', '\r' => { if self.buf.peek() == Some(&'\n') { self.cursor += 1; self.buf.next(); } '\n' } c => c, }; let len = kind.len_utf8(); let pos = self .file .span .subspan(self.cursor as u64, (self.cursor + len) as u64); self.cursor += len; Some(Token { pos, kind }) } } impl Lexer { pub fn new_from_file(file: &Arc) -> Self { let buf = TokenLexer { file: Arc::clone(file), buf: file.source().chars().peekable(), cursor: 0, } .collect(); Self::new(buf) } pub fn new(buf: Vec) -> Lexer { Lexer { buf, cursor: 0, amt_peeked: 0, } } }