Refactor tokens into separate file

src/function.rs

@@ -52,7 +52,7 @@ impl Function {
                 match &tok.kind {
                     TokenKind::AtRule(rule) => body.push(AtRule::from_tokens(
                         rule,
-                        tok.pos,
+                        tok.pos(),
                         toks,
                         &mut scope,
                         &Selector::new(),

src/lib.rs

@@ -85,7 +85,7 @@ use std::iter::{Iterator, Peekable};
 use std::path::Path;
 
 use crate::atrule::{AtRule, AtRuleKind};
-use crate::common::{Keyword, Op, Pos, Symbol, Whitespace};
+use crate::common::{Pos, Symbol, Whitespace};
 use crate::css::Css;
 use crate::error::SassError;
 pub use crate::error::SassResult;
@@ -97,7 +97,8 @@ use crate::mixin::{eat_include, Mixin};
 use crate::scope::{insert_global_var, Scope, GLOBAL_SCOPE};
 use crate::selector::Selector;
 use crate::style::Style;
-use crate::utils::{devour_whitespace, eat_variable_value, IsComment, IsWhitespace, VariableDecl};
+pub(crate) use crate::token::{Token, TokenKind};
+use crate::utils::{devour_whitespace, eat_variable_value, VariableDecl};
 use crate::value::Value;
 
 mod args;
@@ -115,6 +116,7 @@ mod mixin;
 mod scope;
 mod selector;
 mod style;
+mod token;
 mod unit;
 mod utils;
 mod value;
@@ -123,110 +125,6 @@ pub(crate) fn error<E: Into<String>>(msg: E) -> ! {
     eprintln!("Error: {}", msg.into());
     std::process::exit(1);
 }
-
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub(crate) struct Token {
-    pos: Pos,
-    pub kind: TokenKind,
-}
-
-impl Token {
-    pub fn is_symbol(&self, s: Symbol) -> bool {
-        self.kind.is_symbol(s)
-    }
-
-    pub fn from_string(s: String) -> Self {
-        Token {
-            kind: TokenKind::Ident(s),
-            pos: Pos::new(),
-        }
-    }
-
-    pub fn from_symbol(s: Symbol) -> Self {
-        Token {
-            kind: TokenKind::Symbol(s),
-            pos: Pos::new(),
-        }
-    }
-}
-
-impl IsWhitespace for Token {
-    fn is_whitespace(&self) -> bool {
-        if let TokenKind::Whitespace(_) = self.kind {
-            return true;
-        }
-        false
-    }
-}
-
-impl IsWhitespace for &Token {
-    fn is_whitespace(&self) -> bool {
-        if let TokenKind::Whitespace(_) = self.kind {
-            return true;
-        }
-        false
-    }
-}
-
-impl IsComment for Token {
-    fn is_comment(&self) -> bool {
-        if let TokenKind::MultilineComment(_) = self.kind {
-            return true;
-        }
-        false
-    }
-}
-
-impl IsComment for &Token {
-    fn is_comment(&self) -> bool {
-        if let TokenKind::MultilineComment(_) = self.kind {
-            return true;
-        }
-        false
-    }
-}
-
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub(crate) enum TokenKind {
-    Ident(String),
-    Symbol(Symbol),
-    AtRule(AtRuleKind),
-    Keyword(Keyword),
-    Number(String),
-    Whitespace(Whitespace),
-    Variable(String),
-    Op(Op),
-    MultilineComment(String),
-    Interpolation,
-    Unknown(char),
-}
-
-impl TokenKind {
-    pub fn is_symbol(&self, s: Symbol) -> bool {
-        self == &TokenKind::Symbol(s)
-    }
-}
-
-impl Display for TokenKind {
-    #[inline]
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            TokenKind::Ident(s) | TokenKind::Number(s) => write!(f, "{}", s),
-            TokenKind::Symbol(s) => write!(f, "{}", s),
-            TokenKind::AtRule(s) => write!(f, "{}", s),
-            TokenKind::Op(s) => write!(f, "{}", s),
-            TokenKind::Whitespace(s) => write!(f, "{}", s),
-            TokenKind::Keyword(kw) => write!(f, "{}", kw),
-            TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s),
-            TokenKind::Variable(s) => write!(f, "{}", s),
-            TokenKind::Unknown(s) => write!(f, "{}", s),
-            TokenKind::Interpolation => {
-                panic!("we don't want to format TokenKind::Interpolation using Display")
-            }
-        }
-    }
-}
-
 /// Represents a parsed SASS stylesheet with nesting
 #[derive(Debug, Clone)]
 pub struct StyleSheet(Vec<Stmt>);
@@ -657,6 +555,7 @@ pub(crate) fn eat_expr<I: Iterator<Item = Token>>(
                 let tok = toks
                     .next()
                     .expect("this must exist because we have already peeked");
+                let pos = tok.pos();
                 let name = match tok.kind {
                     TokenKind::Variable(n) => n,
                     _ => unsafe { std::hint::unreachable_unchecked() },
@@ -679,7 +578,7 @@ pub(crate) fn eat_expr<I: Iterator<Item = Token>>(
                 } else {
                     values.push(Token {
                         kind: TokenKind::Variable(name),
-                        pos: tok.pos,
+                        pos,
                     });
                 }
             }

src/token.rs

@@ -0,0 +1,112 @@
+use std::fmt;
+
+use crate::atrule::AtRuleKind;
+use crate::common::{Keyword, Op, Pos, Symbol, Whitespace};
+use crate::utils::{IsComment, IsWhitespace};
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct Token {
+    pub pos: Pos,
+    pub kind: TokenKind,
+}
+
+impl Token {
+    pub fn is_symbol(&self, s: Symbol) -> bool {
+        self.kind.is_symbol(s)
+    }
+
+    pub fn from_string(s: String) -> Self {
+        Token {
+            kind: TokenKind::Ident(s),
+            pos: Pos::new(),
+        }
+    }
+
+    pub fn from_symbol(s: Symbol) -> Self {
+        Token {
+            kind: TokenKind::Symbol(s),
+            pos: Pos::new(),
+        }
+    }
+
+    pub const fn pos(&self) -> Pos {
+        self.pos
+    }
+}
+
+impl IsWhitespace for Token {
+    fn is_whitespace(&self) -> bool {
+        if let TokenKind::Whitespace(_) = self.kind {
+            return true;
+        }
+        false
+    }
+}
+
+impl IsWhitespace for &Token {
+    fn is_whitespace(&self) -> bool {
+        if let TokenKind::Whitespace(_) = self.kind {
+            return true;
+        }
+        false
+    }
+}
+
+impl IsComment for Token {
+    fn is_comment(&self) -> bool {
+        if let TokenKind::MultilineComment(_) = self.kind {
+            return true;
+        }
+        false
+    }
+}
+
+impl IsComment for &Token {
+    fn is_comment(&self) -> bool {
+        if let TokenKind::MultilineComment(_) = self.kind {
+            return true;
+        }
+        false
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) enum TokenKind {
+    Ident(String),
+    Symbol(Symbol),
+    AtRule(AtRuleKind),
+    Keyword(Keyword),
+    Number(String),
+    Whitespace(Whitespace),
+    Variable(String),
+    Op(Op),
+    MultilineComment(String),
+    Interpolation,
+    Unknown(char),
+}
+
+impl TokenKind {
+    pub fn is_symbol(&self, s: Symbol) -> bool {
+        self == &TokenKind::Symbol(s)
+    }
+}
+
+impl fmt::Display for TokenKind {
+    #[inline]
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            TokenKind::Ident(s) | TokenKind::Number(s) => write!(f, "{}", s),
+            TokenKind::Symbol(s) => write!(f, "{}", s),
+            TokenKind::AtRule(s) => write!(f, "{}", s),
+            TokenKind::Op(s) => write!(f, "{}", s),
+            TokenKind::Whitespace(s) => write!(f, "{}", s),
+            TokenKind::Keyword(kw) => write!(f, "{}", kw),
+            TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s),
+            TokenKind::Variable(s) => write!(f, "{}", s),
+            TokenKind::Unknown(s) => write!(f, "{}", s),
+            TokenKind::Interpolation => {
+                panic!("we don't want to format TokenKind::Interpolation using Display")
+            }
+        }
+    }
+}