From 195a9b155218d16bfceb51678fdc242616e30640 Mon Sep 17 00:00:00 2001 From: ConnorSkees <39542938+ConnorSkees@users.noreply.github.com> Date: Thu, 19 Mar 2020 19:32:11 -0400 Subject: [PATCH] Refactor tokens into separate file --- src/function.rs | 2 +- src/lib.rs | 113 +++--------------------------------------------- src/token.rs | 112 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 108 deletions(-) create mode 100644 src/token.rs diff --git a/src/function.rs b/src/function.rs index f0f0266..e2d13bd 100644 --- a/src/function.rs +++ b/src/function.rs @@ -52,7 +52,7 @@ impl Function { match &tok.kind { TokenKind::AtRule(rule) => body.push(AtRule::from_tokens( rule, - tok.pos, + tok.pos(), toks, &mut scope, &Selector::new(), diff --git a/src/lib.rs b/src/lib.rs index 9a600c9..d236c4a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,7 +85,7 @@ use std::iter::{Iterator, Peekable}; use std::path::Path; use crate::atrule::{AtRule, AtRuleKind}; -use crate::common::{Keyword, Op, Pos, Symbol, Whitespace}; +use crate::common::{Pos, Symbol, Whitespace}; use crate::css::Css; use crate::error::SassError; pub use crate::error::SassResult; @@ -97,7 +97,8 @@ use crate::mixin::{eat_include, Mixin}; use crate::scope::{insert_global_var, Scope, GLOBAL_SCOPE}; use crate::selector::Selector; use crate::style::Style; -use crate::utils::{devour_whitespace, eat_variable_value, IsComment, IsWhitespace, VariableDecl}; +pub(crate) use crate::token::{Token, TokenKind}; +use crate::utils::{devour_whitespace, eat_variable_value, VariableDecl}; use crate::value::Value; mod args; @@ -115,6 +116,7 @@ mod mixin; mod scope; mod selector; mod style; +mod token; mod unit; mod utils; mod value; @@ -123,110 +125,6 @@ pub(crate) fn error>(msg: E) -> ! { eprintln!("Error: {}", msg.into()); std::process::exit(1); } - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct Token { - pos: Pos, - pub kind: TokenKind, -} - -impl Token { - pub fn is_symbol(&self, s: Symbol) -> bool { - self.kind.is_symbol(s) - } - - pub fn from_string(s: String) -> Self { - Token { - kind: TokenKind::Ident(s), - pos: Pos::new(), - } - } - - pub fn from_symbol(s: Symbol) -> Self { - Token { - kind: TokenKind::Symbol(s), - pos: Pos::new(), - } - } -} - -impl IsWhitespace for Token { - fn is_whitespace(&self) -> bool { - if let TokenKind::Whitespace(_) = self.kind { - return true; - } - false - } -} - -impl IsWhitespace for &Token { - fn is_whitespace(&self) -> bool { - if let TokenKind::Whitespace(_) = self.kind { - return true; - } - false - } -} - -impl IsComment for Token { - fn is_comment(&self) -> bool { - if let TokenKind::MultilineComment(_) = self.kind { - return true; - } - false - } -} - -impl IsComment for &Token { - fn is_comment(&self) -> bool { - if let TokenKind::MultilineComment(_) = self.kind { - return true; - } - false - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) enum TokenKind { - Ident(String), - Symbol(Symbol), - AtRule(AtRuleKind), - Keyword(Keyword), - Number(String), - Whitespace(Whitespace), - Variable(String), - Op(Op), - MultilineComment(String), - Interpolation, - Unknown(char), -} - -impl TokenKind { - pub fn is_symbol(&self, s: Symbol) -> bool { - self == &TokenKind::Symbol(s) - } -} - -impl Display for TokenKind { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - TokenKind::Ident(s) | TokenKind::Number(s) => write!(f, "{}", s), - TokenKind::Symbol(s) => write!(f, "{}", s), - TokenKind::AtRule(s) => write!(f, "{}", s), - TokenKind::Op(s) => write!(f, "{}", s), - TokenKind::Whitespace(s) => write!(f, "{}", s), - TokenKind::Keyword(kw) => write!(f, "{}", kw), - TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s), - TokenKind::Variable(s) => write!(f, "{}", s), - TokenKind::Unknown(s) => write!(f, "{}", s), - TokenKind::Interpolation => { - panic!("we don't want to format TokenKind::Interpolation using Display") - } - } - } -} - /// Represents a parsed SASS stylesheet with nesting #[derive(Debug, Clone)] pub struct StyleSheet(Vec); @@ -657,6 +555,7 @@ pub(crate) fn eat_expr>( let tok = toks .next() .expect("this must exist because we have already peeked"); + let pos = tok.pos(); let name = match tok.kind { TokenKind::Variable(n) => n, _ => unsafe { std::hint::unreachable_unchecked() }, @@ -679,7 +578,7 @@ pub(crate) fn eat_expr>( } else { values.push(Token { kind: TokenKind::Variable(name), - pos: tok.pos, + pos, }); } } diff --git a/src/token.rs b/src/token.rs new file mode 100644 index 0000000..d819e99 --- /dev/null +++ b/src/token.rs @@ -0,0 +1,112 @@ +use std::fmt; + +use crate::atrule::AtRuleKind; +use crate::common::{Keyword, Op, Pos, Symbol, Whitespace}; +use crate::utils::{IsComment, IsWhitespace}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Token { + pub pos: Pos, + pub kind: TokenKind, +} + +impl Token { + pub fn is_symbol(&self, s: Symbol) -> bool { + self.kind.is_symbol(s) + } + + pub fn from_string(s: String) -> Self { + Token { + kind: TokenKind::Ident(s), + pos: Pos::new(), + } + } + + pub fn from_symbol(s: Symbol) -> Self { + Token { + kind: TokenKind::Symbol(s), + pos: Pos::new(), + } + } + + pub const fn pos(&self) -> Pos { + self.pos + } +} + +impl IsWhitespace for Token { + fn is_whitespace(&self) -> bool { + if let TokenKind::Whitespace(_) = self.kind { + return true; + } + false + } +} + +impl IsWhitespace for &Token { + fn is_whitespace(&self) -> bool { + if let TokenKind::Whitespace(_) = self.kind { + return true; + } + false + } +} + +impl IsComment for Token { + fn is_comment(&self) -> bool { + if let TokenKind::MultilineComment(_) = self.kind { + return true; + } + false + } +} + +impl IsComment for &Token { + fn is_comment(&self) -> bool { + if let TokenKind::MultilineComment(_) = self.kind { + return true; + } + false + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum TokenKind { + Ident(String), + Symbol(Symbol), + AtRule(AtRuleKind), + Keyword(Keyword), + Number(String), + Whitespace(Whitespace), + Variable(String), + Op(Op), + MultilineComment(String), + Interpolation, + Unknown(char), +} + +impl TokenKind { + pub fn is_symbol(&self, s: Symbol) -> bool { + self == &TokenKind::Symbol(s) + } +} + +impl fmt::Display for TokenKind { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TokenKind::Ident(s) | TokenKind::Number(s) => write!(f, "{}", s), + TokenKind::Symbol(s) => write!(f, "{}", s), + TokenKind::AtRule(s) => write!(f, "{}", s), + TokenKind::Op(s) => write!(f, "{}", s), + TokenKind::Whitespace(s) => write!(f, "{}", s), + TokenKind::Keyword(kw) => write!(f, "{}", kw), + TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s), + TokenKind::Variable(s) => write!(f, "{}", s), + TokenKind::Unknown(s) => write!(f, "{}", s), + TokenKind::Interpolation => { + panic!("we don't want to format TokenKind::Interpolation using Display") + } + } + } +} \ No newline at end of file