From 07505399da06f8012128a3b6f16c31faf7817bae Mon Sep 17 00:00:00 2001 From: ConnorSkees <39542938+ConnorSkees@users.noreply.github.com> Date: Sun, 29 Mar 2020 13:28:17 -0400 Subject: [PATCH] remove the lexer --- src/args.rs | 66 ++++--- src/atrule/function.rs | 21 +- src/atrule/if_rule.rs | 64 ++++--- src/atrule/mixin.rs | 75 ++------ src/atrule/mod.rs | 164 +++++++++------- src/atrule/parse.rs | 1 + src/atrule/unknown.rs | 24 ++- src/common.rs | 104 ---------- src/lexer.rs | 290 ++-------------------------- src/lib.rs | 345 ++++++++++++++++----------------- src/selector.rs | 163 ++++++++-------- src/style.rs | 113 +++-------- src/token.rs | 140 ++++++-------- src/utils.rs | 424 +++++++++++++++++++++++++++++++---------- src/value/parse.rs | 353 ++++++++++++++++++++++------------ tests/if.rs | 118 ++++++------ tests/meta.rs | 4 +- tests/misc.rs | 6 + tests/mixins.rs | 5 + tests/str-escape.rs | 6 + 20 files changed, 1184 insertions(+), 1302 deletions(-) diff --git a/src/args.rs b/src/args.rs index 0129ea8..426efdb 100644 --- a/src/args.rs +++ b/src/args.rs @@ -1,13 +1,13 @@ use std::collections::BTreeMap; use std::iter::Peekable; -use crate::common::Symbol; +use crate::common::Pos; use crate::error::SassResult; use crate::scope::Scope; use crate::selector::Selector; -use crate::utils::{devour_whitespace, devour_whitespace_or_comment}; +use crate::utils::{devour_whitespace, devour_whitespace_or_comment, eat_ident}; use crate::value::Value; -use crate::{Token, TokenKind}; +use crate::Token; #[derive(Debug, Clone, Eq, PartialEq)] pub(crate) struct FuncArgs(pub Vec); @@ -59,8 +59,8 @@ pub(crate) fn eat_func_args>( devour_whitespace(toks); while let Some(Token { kind, .. }) = toks.next() { let name = match kind { - TokenKind::Variable(v) => v, - TokenKind::Symbol(Symbol::CloseParen) => break, + '$' => eat_ident(toks, scope, super_selector)?, + ')' => break, _ => todo!(), }; let mut default: Vec = Vec::new(); @@ -70,11 +70,11 @@ pub(crate) fn eat_func_args>( _ => todo!("unexpected eof"), }; match kind { - TokenKind::Symbol(Symbol::Colon) => { + ':' => { devour_whitespace(toks); while let Some(tok) = toks.peek() { match &tok.kind { - TokenKind::Symbol(Symbol::Comma) => { + ',' => { toks.next(); args.push(FuncArg { name, @@ -86,7 +86,7 @@ pub(crate) fn eat_func_args>( }); break; } - TokenKind::Symbol(Symbol::CloseParen) => { + ')' => { args.push(FuncArg { name, default: Some(Value::from_tokens( @@ -104,8 +104,8 @@ pub(crate) fn eat_func_args>( } } } - TokenKind::Symbol(Symbol::Period) => todo!("handle varargs"), - TokenKind::Symbol(Symbol::CloseParen) => { + '.' => todo!("handle varargs"), + ')' => { args.push(FuncArg { name, default: if default.is_empty() { @@ -120,7 +120,7 @@ pub(crate) fn eat_func_args>( }); break; } - TokenKind::Symbol(Symbol::Comma) => args.push(FuncArg { + ',' => args.push(FuncArg { name, default: None, }), @@ -129,11 +129,7 @@ pub(crate) fn eat_func_args>( devour_whitespace(toks); } devour_whitespace(toks); - if let Some(Token { - kind: TokenKind::Symbol(Symbol::OpenCurlyBrace), - .. - }) = toks.next() - { + if let Some(Token { kind: '{', .. }) = toks.next() { } else { todo!("expected `{{` after args") } @@ -146,45 +142,47 @@ pub(crate) fn eat_call_args>( super_selector: &Selector, ) -> SassResult { let mut args: BTreeMap = BTreeMap::new(); - devour_whitespace_or_comment(toks); + devour_whitespace_or_comment(toks)?; let mut name: String; let mut val: Vec = Vec::new(); loop { match toks.peek().unwrap().kind { - TokenKind::Variable(_) => { - let v = toks.next().unwrap(); - devour_whitespace_or_comment(toks); - if toks.peek().unwrap().is_symbol(Symbol::Colon) { + '$' => { + toks.next(); + let v = eat_ident(toks, scope, super_selector)?; + devour_whitespace_or_comment(toks)?; + if toks.peek().unwrap().kind == ':' { toks.next(); - name = v.kind.to_string(); + name = v; } else { - val.push(v); + val.push(Token::new(Pos::new(), '$')); + val.extend(v.chars().map(|x| Token::new(Pos::new(), x))); name = args.len().to_string(); } } - TokenKind::Symbol(Symbol::CloseParen) => { + ')' => { toks.next(); return Ok(CallArgs(args)); } _ => name = args.len().to_string(), } - devour_whitespace_or_comment(toks); + devour_whitespace_or_comment(toks)?; while let Some(tok) = toks.next() { match tok.kind { - TokenKind::Symbol(Symbol::CloseParen) => { + ')' => { args.insert( name, Value::from_tokens(&mut val.into_iter().peekable(), scope, super_selector)?, ); return Ok(CallArgs(args)); } - TokenKind::Symbol(Symbol::Comma) => break, - TokenKind::Symbol(Symbol::OpenSquareBrace) => { + ',' => break, + '[' => { val.push(tok); val.extend(read_until_close_square_brace(toks)); } - TokenKind::Symbol(Symbol::OpenParen) => { + '(' => { val.push(tok); val.extend(read_until_close_paren(toks)); } @@ -201,7 +199,7 @@ pub(crate) fn eat_call_args>( )?, ); val.clear(); - devour_whitespace_or_comment(toks); + devour_whitespace(toks); if toks.peek().is_none() { return Ok(CallArgs(args)); @@ -214,7 +212,7 @@ fn read_until_close_paren>(toks: &mut Peekable) -> let mut scope = 0; for tok in toks { match tok.kind { - TokenKind::Symbol(Symbol::CloseParen) => { + ')' => { if scope <= 1 { v.push(tok); return v; @@ -222,7 +220,7 @@ fn read_until_close_paren>(toks: &mut Peekable) -> scope -= 1; } } - TokenKind::Symbol(Symbol::OpenParen) => scope += 1, + '(' => scope += 1, _ => {} } v.push(tok) @@ -235,7 +233,7 @@ fn read_until_close_square_brace>(toks: &mut Peekable< let mut scope = 0; for tok in toks { match tok.kind { - TokenKind::Symbol(Symbol::CloseSquareBrace) => { + ']' => { if scope <= 1 { v.push(tok); return v; @@ -243,7 +241,7 @@ fn read_until_close_square_brace>(toks: &mut Peekable< scope -= 1; } } - TokenKind::Symbol(Symbol::OpenSquareBrace) => scope += 1, + '[' => scope += 1, _ => {} } v.push(tok) diff --git a/src/atrule/function.rs b/src/atrule/function.rs index 3fd7e42..32f9fe0 100644 --- a/src/atrule/function.rs +++ b/src/atrule/function.rs @@ -4,13 +4,12 @@ use super::eat_stmts; use crate::args::{eat_func_args, CallArgs, FuncArgs}; use crate::atrule::AtRule; -use crate::common::Symbol; use crate::error::SassResult; use crate::scope::Scope; use crate::selector::Selector; -use crate::utils::devour_whitespace; +use crate::utils::{devour_whitespace, eat_ident}; use crate::value::Value; -use crate::{Stmt, Token, TokenKind}; +use crate::{Stmt, Token}; #[derive(Debug, Clone)] pub(crate) struct Function { @@ -29,23 +28,15 @@ impl Function { scope: Scope, super_selector: &Selector, ) -> SassResult<(String, Function)> { - let Token { kind, .. } = toks - .next() - .expect("this must exist because we have already peeked"); - devour_whitespace(toks); - let name = match kind { - TokenKind::Ident(s) => s, - _ => return Err("Expected identifier.".into()), - }; + let name = eat_ident(toks, &scope, super_selector)?; devour_whitespace(toks); let args = match toks.next() { - Some(Token { - kind: TokenKind::Symbol(Symbol::OpenParen), - .. - }) => eat_func_args(toks, &scope, super_selector)?, + Some(Token { kind: '(', .. }) => eat_func_args(toks, &scope, super_selector)?, _ => return Err("expected \"(\".".into()), }; + devour_whitespace(toks); + let body = eat_stmts(toks, &mut scope.clone(), super_selector)?; devour_whitespace(toks); diff --git a/src/atrule/if_rule.rs b/src/atrule/if_rule.rs index ca1d877..ba66221 100644 --- a/src/atrule/if_rule.rs +++ b/src/atrule/if_rule.rs @@ -1,16 +1,15 @@ use std::iter::Peekable; -use super::{eat_stmts, AtRule, AtRuleKind}; +use super::{eat_stmts, AtRule}; -use crate::common::Symbol; use crate::error::SassResult; use crate::scope::Scope; use crate::selector::Selector; use crate::utils::{ - devour_whitespace_or_comment, read_until_closing_curly_brace, read_until_open_curly_brace, + devour_whitespace, eat_ident, read_until_closing_curly_brace, read_until_open_curly_brace, }; use crate::value::Value; -use crate::{Stmt, Token, TokenKind}; +use crate::{Stmt, Token}; #[derive(Debug, Clone)] pub(crate) struct If { @@ -35,37 +34,50 @@ impl If { let mut branches = Vec::new(); let init_cond = read_until_open_curly_brace(toks); toks.next(); - devour_whitespace_or_comment(toks); + devour_whitespace(toks); let mut init_toks = read_until_closing_curly_brace(toks); init_toks.push(toks.next().unwrap()); - devour_whitespace_or_comment(toks); + devour_whitespace(toks); branches.push(Branch::new(init_cond, init_toks)); let mut else_ = Vec::new(); loop { - if let Some(tok) = toks.peek() { - if tok.kind == TokenKind::AtRule(AtRuleKind::Else) { + if toks.peek().is_some() { + if toks.peek().unwrap().kind == '@' { toks.next(); - devour_whitespace_or_comment(toks); + } else { + break; + } + if eat_ident(toks, &Scope::new(), &Selector::new())?.to_ascii_lowercase() == "else" + { + devour_whitespace(toks); if let Some(tok) = toks.next() { - devour_whitespace_or_comment(toks); - if tok.kind.to_string().to_ascii_lowercase() == "if" { - let cond = read_until_open_curly_brace(toks); - toks.next(); - devour_whitespace_or_comment(toks); - let mut toks_ = read_until_closing_curly_brace(toks); - toks_.push(toks.next().unwrap()); - devour_whitespace_or_comment(toks); - branches.push(Branch::new(cond, toks_)) - } else if tok.is_symbol(Symbol::OpenCurlyBrace) { - else_ = read_until_closing_curly_brace(toks); - toks.next(); - break; - } else { - return Err("expected \"{\".".into()); + devour_whitespace(toks); + match tok.kind.to_ascii_lowercase() { + 'i' if toks.next().unwrap().kind.to_ascii_lowercase() == 'f' => { + toks.next(); + let cond = read_until_open_curly_brace(toks); + toks.next(); + devour_whitespace(toks); + let toks_ = read_until_closing_curly_brace(toks); + toks.next(); + devour_whitespace(toks); + branches.push(Branch::new(cond, toks_)) + } + '{' => { + else_ = read_until_closing_curly_brace(toks); + dbg!(&else_); + toks.next(); + break; + } + _ => { + return Err("expected \"{\".".into()); + } } + } else { + break; } } else { break; @@ -74,7 +86,9 @@ impl If { break; } } - devour_whitespace_or_comment(toks); + devour_whitespace(toks); + + dbg!(&branches); Ok(If { branches, else_ }) } diff --git a/src/atrule/mixin.rs b/src/atrule/mixin.rs index 02fe77f..3a9c727 100644 --- a/src/atrule/mixin.rs +++ b/src/atrule/mixin.rs @@ -5,12 +5,13 @@ use super::eat_stmts; use crate::args::{eat_call_args, eat_func_args, CallArgs, FuncArgs}; use crate::atrule::AtRule; -use crate::common::Symbol; -use crate::error::{SassError, SassResult}; +use crate::error::SassResult; use crate::scope::Scope; use crate::selector::Selector; -use crate::utils::devour_whitespace; -use crate::{eat_expr, Expr, RuleSet, Stmt, Token, TokenKind}; +use crate::utils::{ + devour_whitespace, devour_whitespace_or_comment, eat_ident, read_until_closing_curly_brace, +}; +use crate::{eat_expr, Expr, RuleSet, Stmt, Token}; #[derive(Debug, Clone)] pub(crate) struct Mixin { @@ -36,46 +37,19 @@ impl Mixin { scope: &Scope, super_selector: &Selector, ) -> SassResult<(String, Mixin)> { - let Token { kind, .. } = toks - .next() - .expect("this must exist because we have already peeked"); devour_whitespace(toks); - let name = match kind { - TokenKind::Ident(s) => s, - _ => return Err("Expected identifier.".into()), - }; + let name = eat_ident(toks, scope, super_selector)?; devour_whitespace(toks); let args = match toks.next() { - Some(Token { - kind: TokenKind::Symbol(Symbol::OpenParen), - .. - }) => eat_func_args(toks, scope, super_selector)?, - Some(Token { - kind: TokenKind::Symbol(Symbol::OpenCurlyBrace), - .. - }) => FuncArgs::new(), + Some(Token { kind: '(', .. }) => eat_func_args(toks, scope, super_selector)?, + Some(Token { kind: '{', .. }) => FuncArgs::new(), _ => return Err("expected \"{\".".into()), }; devour_whitespace(toks); - let mut nesting = 1; - let mut body = Vec::new(); - - while nesting > 0 { - if let Some(tok) = toks.next() { - match &tok.kind { - TokenKind::Symbol(Symbol::OpenCurlyBrace) - // interpolation token eats the opening brace but not the closing - | TokenKind::Interpolation => nesting += 1, - TokenKind::Symbol(Symbol::CloseCurlyBrace) => nesting -= 1, - _ => {} - } - body.push(tok) - } else { - return Err("unexpected EOF (TODO: better message)".into()); - } - } + let mut body = read_until_closing_curly_brace(toks); + body.push(toks.next().unwrap()); Ok((name, Mixin::new(scope.clone(), args, body, Vec::new()))) } @@ -147,49 +121,42 @@ pub(crate) fn eat_include>( scope: &Scope, super_selector: &Selector, ) -> SassResult> { - toks.next(); - devour_whitespace(toks); - let Token { kind, pos } = toks - .next() - .expect("this must exist because we have already peeked"); - let name = match kind { - TokenKind::Ident(s) => s, - _ => return Err("Expected identifier.".into()), - }; + devour_whitespace_or_comment(toks)?; + let name = eat_ident(toks, scope, super_selector)?; - devour_whitespace(toks); + devour_whitespace_or_comment(toks)?; let mut has_include = false; let mut args = if let Some(tok) = toks.next() { match tok.kind { - TokenKind::Symbol(Symbol::SemiColon) => CallArgs::new(), - TokenKind::Symbol(Symbol::OpenParen) => { + ';' => CallArgs::new(), + '(' => { let tmp = eat_call_args(toks, scope, super_selector)?; - devour_whitespace(toks); + devour_whitespace_or_comment(toks)?; if let Some(tok) = toks.next() { match tok.kind { - TokenKind::Symbol(Symbol::SemiColon) => {} - TokenKind::Symbol(Symbol::OpenCurlyBrace) => has_include = true, + ';' => {} + '{' => has_include = true, _ => todo!(), } } tmp } - TokenKind::Symbol(Symbol::OpenCurlyBrace) => { + '{' => { has_include = true; CallArgs::new() } _ => return Err("expected \"{\".".into()), } } else { - return Err(SassError::new("unexpected EOF", pos)); + return Err("unexpected EOF".into()); }; devour_whitespace(toks); let content = if let Some(tok) = toks.peek() { - if tok.is_symbol(Symbol::OpenCurlyBrace) { + if tok.kind == '{' { toks.next(); eat_stmts(toks, &mut scope.clone(), super_selector)? } else if has_include { diff --git a/src/atrule/mod.rs b/src/atrule/mod.rs index 73e160f..b53d8e8 100644 --- a/src/atrule/mod.rs +++ b/src/atrule/mod.rs @@ -3,14 +3,17 @@ use std::iter::Peekable; use num_traits::cast::ToPrimitive; -use crate::common::{Keyword, Pos, Symbol}; +use crate::common::Pos; use crate::error::SassResult; use crate::scope::Scope; use crate::selector::Selector; use crate::unit::Unit; -use crate::utils::{devour_whitespace, devour_whitespace_or_comment}; +use crate::utils::{ + devour_whitespace, eat_ident, read_until_closing_curly_brace, read_until_open_curly_brace, + read_until_semicolon_or_closing_curly_brace, +}; use crate::value::{Number, Value}; -use crate::{Stmt, Token, TokenKind}; +use crate::{Stmt, Token}; pub(crate) use function::Function; pub(crate) use if_rule::If; @@ -51,14 +54,14 @@ impl AtRule { Ok(match rule { AtRuleKind::Error => { let message = toks - .take_while(|x| x.kind != TokenKind::Symbol(Symbol::SemiColon)) + .take_while(|x| x.kind != ';') .map(|x| x.kind.to_string()) .collect::(); AtRule::Error(pos, message) } AtRuleKind::Warn => { let message = toks - .take_while(|x| x.kind != TokenKind::Symbol(Symbol::SemiColon)) + .take_while(|x| x.kind != ';') .map(|x| x.kind.to_string()) .collect::(); devour_whitespace(toks); @@ -67,7 +70,7 @@ impl AtRule { AtRuleKind::Debug => { let message = toks .by_ref() - .take_while(|x| x.kind != TokenKind::Symbol(Symbol::SemiColon)) + .take_while(|x| x.kind != ';') .map(|x| x.kind.to_string()) .collect::(); devour_whitespace(toks); @@ -82,29 +85,22 @@ impl AtRule { AtRule::Function(name, Box::new(func)) } AtRuleKind::Return => { - let mut t = Vec::new(); - let mut n = 0; - while let Some(tok) = toks.peek() { - match tok.kind { - TokenKind::Symbol(Symbol::OpenCurlyBrace) => n += 1, - TokenKind::Symbol(Symbol::CloseCurlyBrace) => n -= 1, - TokenKind::Interpolation => n += 1, - TokenKind::Symbol(Symbol::SemiColon) => break, - _ => {} - } - if n < 0 { - break; - } - t.push(toks.next().unwrap()); + let v = read_until_semicolon_or_closing_curly_brace(toks); + if toks.peek().unwrap().kind == ';' { + toks.next(); } - AtRule::Return(t) + devour_whitespace(toks); + AtRule::Return(v) } AtRuleKind::Use => todo!("@use not yet implemented"), AtRuleKind::Annotation => todo!("@annotation not yet implemented"), AtRuleKind::AtRoot => todo!("@at-root not yet implemented"), AtRuleKind::Charset => { - toks.take_while(|t| t.kind != TokenKind::Symbol(Symbol::SemiColon)) - .for_each(drop); + read_until_semicolon_or_closing_curly_brace(toks); + if toks.peek().unwrap().kind == ';' { + toks.next(); + } + devour_whitespace(toks); AtRule::Charset } AtRuleKind::Each => todo!("@each not yet implemented"), @@ -113,38 +109,81 @@ impl AtRule { AtRuleKind::Else => todo!("@else not yet implemented"), AtRuleKind::For => { let mut stmts = Vec::new(); - devour_whitespace_or_comment(toks); - let var = if let Some(tok) = toks.next() { - match tok.kind { - TokenKind::Variable(s) => s, - _ => return Err("expected \"$\".".into()), - } - } else { - return Err("expected \"$\".".into()); + devour_whitespace(toks); + let var = match toks.next().ok_or("expected \"$\".")?.kind { + '$' => eat_ident(toks, scope, super_selector)?, + _ => return Err("expected \"$\".".into()), }; - devour_whitespace_or_comment(toks); - if let Some(tok) = toks.next() { - match tok.kind { - TokenKind::Keyword(Keyword::From(..)) => {} - _ => return Err("Expected \"from\".".into()), - } - } else { + devour_whitespace(toks); + if toks.peek().is_none() + || eat_ident(toks, scope, super_selector)?.to_ascii_lowercase() != "from" + { return Err("Expected \"from\".".into()); - }; - devour_whitespace_or_comment(toks); + } + devour_whitespace(toks); let mut from_toks = Vec::new(); let mut through = 0; while let Some(tok) = toks.next() { - match tok.kind { - TokenKind::Keyword(Keyword::Through(..)) => { - through = 1; - break; + let mut these_toks = vec![tok]; + match these_toks[0].kind.to_ascii_lowercase() { + 't' => { + these_toks.push(toks.next().unwrap()); + match these_toks[1].kind.to_ascii_lowercase() { + 'h' => { + let r = toks.next().unwrap(); + these_toks.push(r); + if &r.kind != &'r' { + from_toks.extend(these_toks); + continue; + } + let o = toks.next().unwrap(); + these_toks.push(o); + if o.kind != 'o' { + from_toks.extend(these_toks); + continue; + } + let u = toks.next().unwrap(); + these_toks.push(u); + if u.kind != 'u' { + from_toks.extend(these_toks); + continue; + } + let g = toks.next().unwrap(); + these_toks.push(g); + if g.kind != 'g' { + from_toks.extend(these_toks); + continue; + } + let h = toks.next().unwrap(); + these_toks.push(h); + if h.kind != 'h' { + from_toks.extend(these_toks); + continue; + } + let peek = toks.peek().unwrap().kind; + if peek.is_alphanumeric() || peek == '\\' { + from_toks.extend(these_toks); + continue; + } + through = 1; + break; + } + 'o' => { + if toks.peek().unwrap().kind.is_whitespace() { + break; + } else { + from_toks.extend(these_toks); + } + } + _ => { + from_toks.extend(these_toks); + } + } } - TokenKind::Keyword(Keyword::To(..)) => break, - TokenKind::Symbol(Symbol::OpenCurlyBrace) => { + '{' => { return Err("Expected \"to\" or \"through\".".into()); } - _ => from_toks.push(tok), + _ => from_toks.extend(these_toks), } } let from = match Value::from_tokens( @@ -158,14 +197,9 @@ impl AtRule { }, v => return Err(format!("{} is not an integer.", v).into()), }; - devour_whitespace_or_comment(toks); - let mut to_toks = Vec::new(); - while let Some(tok) = toks.next() { - match tok.kind { - TokenKind::Symbol(Symbol::OpenCurlyBrace) => break, - _ => to_toks.push(tok), - } - } + devour_whitespace(toks); + let to_toks = read_until_open_curly_brace(toks); + toks.next(); let to = match Value::from_tokens( &mut to_toks.into_iter().peekable(), scope, @@ -177,22 +211,12 @@ impl AtRule { }, v => return Err(format!("{} is not an integer.", v).into()), }; - let mut body = Vec::new(); - let mut n = 1; - while let Some(tok) = toks.next() { - match tok.kind { - TokenKind::Symbol(Symbol::OpenCurlyBrace) => n += 1, - TokenKind::Symbol(Symbol::CloseCurlyBrace) => n -= 1, - TokenKind::Interpolation => n += 1, - _ => {} - } - if n == 0 { - break; - } - body.push(tok); - } + let body = read_until_closing_curly_brace(toks); + // body.push(toks.next().unwrap()); + toks.next(); + // dbg!(&body); - devour_whitespace_or_comment(toks); + devour_whitespace(toks); let (mut x, mut y); let iter: &mut dyn std::iter::Iterator = if from < to { diff --git a/src/atrule/parse.rs b/src/atrule/parse.rs index af88976..343e324 100644 --- a/src/atrule/parse.rs +++ b/src/atrule/parse.rs @@ -12,6 +12,7 @@ pub(crate) fn eat_stmts>( ) -> SassResult> { let mut stmts = Vec::new(); while let Some(expr) = eat_expr(toks, scope, super_selector)? { + // dbg!(&expr); match expr { Expr::AtRule(a) => stmts.push(Stmt::AtRule(a)), Expr::Style(s) => stmts.push(Stmt::Style(s)), diff --git a/src/atrule/unknown.rs b/src/atrule/unknown.rs index 9e61c08..53be3e4 100644 --- a/src/atrule/unknown.rs +++ b/src/atrule/unknown.rs @@ -1,12 +1,11 @@ use std::iter::Peekable; use super::parse::eat_stmts; -use crate::common::Symbol; use crate::error::SassResult; use crate::scope::Scope; use crate::selector::Selector; use crate::utils::{devour_whitespace, parse_interpolation}; -use crate::{RuleSet, Stmt, Token, TokenKind}; +use crate::{RuleSet, Stmt, Token}; #[derive(Debug, Clone)] pub(crate) struct UnknownAtRule { @@ -26,21 +25,26 @@ impl UnknownAtRule { let mut params = String::new(); while let Some(tok) = toks.next() { match tok.kind { - TokenKind::Symbol(Symbol::OpenCurlyBrace) => break, - TokenKind::Interpolation => { - params.push_str(&parse_interpolation(toks, scope, super_selector)?.to_string()); - continue; + '{' => break, + '#' => { + if toks.peek().unwrap().kind == '{' { + toks.next(); + params.push_str( + &parse_interpolation(toks, scope, super_selector)?.to_string(), + ); + continue; + } else { + params.push(tok.kind); + } } - TokenKind::Variable(..) => params.push('$'), - TokenKind::Whitespace(..) => { + '\n' | ' ' | '\t' => { devour_whitespace(toks); params.push(' '); continue; } - TokenKind::Error(e) => return Err(e), _ => {} } - params.push_str(&tok.kind.to_string()); + params.push(tok.kind); } let raw_body = eat_stmts(toks, scope, super_selector)?; diff --git a/src/common.rs b/src/common.rs index 623dcef..ba253dc 100644 --- a/src/common.rs +++ b/src/common.rs @@ -211,110 +211,6 @@ impl Display for Op { } } -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum Keyword { - Important, - True, - False, - Null, - Default, - Global, - From(String), - To(String), - Through(String), - // Infinity, - // NaN, - // Auto, - // Inherit, - // Initial, - // Unset, - // Not, - // And, - // Or, - // In, -} - -impl Display for Keyword { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Important => write!(f, "!important"), - Self::True => write!(f, "true"), - Self::False => write!(f, "false"), - Self::Null => write!(f, "null"), - Self::Default => write!(f, "!default"), - Self::Global => write!(f, "!global"), - Self::From(s) => write!(f, "{}", s), - Self::To(s) => write!(f, "{}", s), - Self::Through(s) => write!(f, "{}", s), - // Self::Infinity => write!(f, "Infinity"), - // Self::NaN => write!(f, "NaN"), - // Self::Auto => write!(f, "auto"), - // Self::Inherit => write!(f, "inherit"), - // Self::Initial => write!(f, "initial"), - // Self::Unset => write!(f, "unset"), - // Self::Not => write!(f, "not"), - // Self::And => write!(f, "and"), - // Self::Or => write!(f, "or"), - // Self::In => write!(f, "in"), - } - } -} - -impl Into<&'static str> for Keyword { - fn into(self) -> &'static str { - match self { - Self::Important => "!important", - Self::True => "true", - Self::False => "false", - Self::Null => "null", - Self::Default => "!default", - Self::Global => "!global", - Self::From(_) => "from", - Self::To(_) => "to", - Self::Through(_) => "through", - // Self::Infinity => "Infinity", - // Self::NaN => "NaN", - // Self::Auto => "auto", - // Self::Inherit => "inherit", - // Self::Initial => "initial", - // Self::Unset => "unset", - // Self::Not => "not", - // Self::And => "and", - // Self::Or => "or", - // Self::In => "in", - } - } -} - -impl TryFrom<&str> for Keyword { - type Error = &'static str; - - fn try_from(kw: &str) -> Result { - match kw.to_ascii_lowercase().as_str() { - "important" => Ok(Self::Important), - "true" => Ok(Self::True), - "false" => Ok(Self::False), - "null" => Ok(Self::Null), - "default" => Ok(Self::Default), - "global" => Ok(Self::Global), - "from" => Ok(Self::From(kw.to_owned())), - "to" => Ok(Self::To(kw.to_owned())), - "through" => Ok(Self::Through(kw.to_owned())), - // "infinity" => Ok(Self::Infinity), - // "nan" => Ok(Self::NaN), - // "auto" => Ok(Self::Auto), - // "inherit" => Ok(Self::Inherit), - // "initial" => Ok(Self::Initial), - // "unset" => Ok(Self::Unset), - // "not" => Ok(Self::Not), - // "and" => Ok(Self::And), - // "or" => Ok(Self::Or), - // "in" => Ok(Self::In), - _ => Err("invalid keyword"), - } - } -} - #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub struct Pos { line: u32, diff --git a/src/lexer.rs b/src/lexer.rs index 911b90a..3c09f73 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,156 +1,44 @@ -use std::convert::TryFrom; use std::iter::Peekable; use std::str::Chars; use std::sync::atomic::{AtomicBool, Ordering}; -use crate::atrule::AtRuleKind; -use crate::common::{Keyword, Op, Pos, Symbol}; -use crate::{Token, TokenKind, Whitespace}; - -// Rust does not allow us to escape '\f' -const FORM_FEED: char = '\x0C'; +use crate::common::Pos; +use crate::Token; pub static IS_UTF8: AtomicBool = AtomicBool::new(false); +pub const FORM_FEED: char = '\x0C'; #[derive(Debug, Clone)] pub(crate) struct Lexer<'a> { - tokens: Vec, buf: Peekable>, pos: Pos, - should_emit_backslash: usize, } impl<'a> Iterator for Lexer<'a> { type Item = Token; fn next(&mut self) -> Option { - macro_rules! symbol { - ($self:ident, $symbol:ident) => {{ - $self.buf.next(); - $self.pos.next_char(); - TokenKind::Symbol(Symbol::$symbol) - }}; - } - macro_rules! whitespace { - ($self:ident, $whitespace:ident) => {{ - $self.buf.next(); - $self.pos.next_char(); - TokenKind::Whitespace(Whitespace::$whitespace) - }}; - } - if self.should_emit_backslash > 0 { - self.should_emit_backslash -= 1; - return Some(Token { - kind: TokenKind::Symbol(Symbol::BackSlash), - pos: self.pos, - }); - } - let kind: TokenKind = match self.buf.peek().unwrap_or(&'\0') { - 'a'..='z' | 'A'..='Z' | '_' => self.lex_ident(), - '-' => { - self.buf.next(); - self.pos.next_char(); - match self.buf.peek().unwrap() { - '0'..='9' | '.' => match self.lex_num() { - TokenKind::Number(n) => { - let mut s = String::from("-"); - s.push_str(&n); - TokenKind::Number(s) - } - e @ TokenKind::Error(..) => e, - _ => unsafe { std::hint::unreachable_unchecked() }, - }, - 'a'..='z' | 'A'..='Z' | '_' | '-' => match self.lex_ident() { - TokenKind::Ident(i) => { - let mut s = String::from("-"); - s.push_str(&i); - TokenKind::Ident(s) - } - TokenKind::Keyword(kw) => { - let mut s = String::from("-"); - s.push_str(&kw.to_string()); - TokenKind::Ident(s) - } - TokenKind::Symbol(Symbol::Minus) => TokenKind::Ident(String::from("--")), - e @ TokenKind::Error(..) => e, - _ => unsafe { std::hint::unreachable_unchecked() }, - }, - _ => TokenKind::Symbol(Symbol::Minus), - } - } - '@' => self.lex_at_rule(), - '0'..='9' => self.lex_num(), - '.' => { - self.buf.next(); - self.pos.next_char(); - match self.buf.peek().unwrap() { - '0'..='9' => match self.lex_num() { - TokenKind::Number(n) => { - let mut s = String::from("0."); - s.push_str(&n); - TokenKind::Number(s) - } - e @ TokenKind::Error(..) => e, - _ => unsafe { std::hint::unreachable_unchecked() }, - }, - _ => TokenKind::Symbol(Symbol::Period), - } - } - '$' => self.lex_variable(), - ':' => symbol!(self, Colon), - ',' => symbol!(self, Comma), - ';' => symbol!(self, SemiColon), - '(' => symbol!(self, OpenParen), - ')' => symbol!(self, CloseParen), - '+' => symbol!(self, Plus), - '=' => { - self.buf.next(); - self.pos.next_char(); - match self.buf.peek() { - Some('=') => { - self.buf.next(); - self.pos.next_char(); - TokenKind::Op(Op::Equal) - } - _ => TokenKind::Symbol(Symbol::Equal), - } - } - '?' => symbol!(self, QuestionMark), - '\\' => self.lex_back_slash().0, - '~' => symbol!(self, Tilde), - '\'' => symbol!(self, SingleQuote), - '"' => symbol!(self, DoubleQuote), - ' ' => whitespace!(self, Space), - '\t' => whitespace!(self, Tab), - '\n' | &FORM_FEED => { - self.buf.next(); + let kind = match self.buf.next()? { + '\n' | FORM_FEED => { self.pos.newline(); - TokenKind::Whitespace(Whitespace::Newline) + '\n' } '\r' => { - self.buf.next(); - TokenKind::Whitespace(Whitespace::Newline) + if self.buf.peek() == Some(&'\n') { + self.buf.next(); + '\n' + } else { + '\n' + } } - '#' => self.lex_hash(), - '{' => symbol!(self, OpenCurlyBrace), - '*' => symbol!(self, Mul), - '}' => symbol!(self, CloseCurlyBrace), - '&' => symbol!(self, BitAnd), - '|' => symbol!(self, BitOr), - '/' => self.lex_forward_slash(), - '%' => symbol!(self, Percent), - '[' => symbol!(self, OpenSquareBrace), - ']' => symbol!(self, CloseSquareBrace), - '!' => self.lex_exclamation(), - '<' => symbol!(self, Lt), - '>' => symbol!(self, Gt), - '^' => symbol!(self, Xor), - '`' => symbol!(self, BackTick), '\0' => return None, - c if c.is_control() => { - self.buf.next(); - TokenKind::Error("Expected expression.".into()) + // c if c.is_control() => { + // return Some(Err("Expected expression.".into())) + // } + c if !c.is_ascii() => { + IS_UTF8.store(true, Ordering::Relaxed); + c } - _ => self.lex_ident(), + c => c, }; self.pos.next_char(); Some(Token { @@ -163,13 +51,11 @@ impl<'a> Iterator for Lexer<'a> { impl<'a> Lexer<'a> { pub fn new(buf: &'a str) -> Lexer<'a> { Lexer { - tokens: Vec::with_capacity(buf.len()), buf: buf.chars().peekable(), pos: Pos::new(), - should_emit_backslash: 0, } } - + /* fn lex_exclamation(&mut self) -> TokenKind { self.buf.next(); self.pos.next_char(); @@ -204,19 +90,6 @@ impl<'a> Lexer<'a> { } } - fn lex_at_rule(&mut self) -> TokenKind { - self.buf.next(); - self.pos.next_char(); - if let TokenKind::Ident(s) = self.lex_ident() { - if s.is_empty() { - TokenKind::Error("Expected identifier.".into()) - } else { - TokenKind::AtRule(AtRuleKind::from(s.as_ref())) - } - } else { - TokenKind::Error("Expected identifier.".into()) - } - } fn lex_back_slash(&mut self) -> (TokenKind, bool) { self.buf.next(); @@ -254,95 +127,6 @@ impl<'a> Lexer<'a> { } } - fn devour_whitespace(&mut self) { - while let Some(c) = self.buf.peek() { - if c.is_ascii_whitespace() { - self.buf.next(); - self.pos.next_char(); - continue; - } - break; - } - } - - fn lex_forward_slash(&mut self) -> TokenKind { - self.buf.next(); - self.pos.next_char(); - match self.buf.peek().expect("expected something after '/'") { - '/' => { - self.buf.by_ref().take_while(|x| x != &'\n').for_each(drop); - self.pos.newline(); - } - '*' => { - self.buf.next(); - self.pos.next_char(); - let mut comment = String::new(); - while let Some(tok) = self.buf.next() { - match tok { - '\n' => self.pos.newline(), - FORM_FEED => { - self.pos.newline(); - comment.push('\n'); - continue; - } - '\r' => { - if self.buf.peek() == Some(&'\n') { - self.buf.next(); - } - self.pos.newline(); - comment.push('\n'); - continue; - } - '*' if self.buf.peek() == Some(&'/') => { - self.buf.next(); - break; - } - _ => self.pos.next_char(), - } - comment.push(tok); - } - return TokenKind::MultilineComment(comment); - } - _ => return TokenKind::Symbol(Symbol::Div), - } - TokenKind::Whitespace(Whitespace::Newline) - } - - fn lex_num(&mut self) -> TokenKind { - let mut whole = String::new(); - while let Some(c) = self.buf.peek() { - if !c.is_numeric() { - break; - } - let tok = self.buf.next().unwrap(); - self.pos.next_char(); - whole.push(tok); - } - - let mut dec = String::new(); - - if self.buf.peek() == Some(&'.') { - self.buf.next(); - dec.push('.'); - while let Some(c) = self.buf.peek() { - if !c.is_numeric() { - break; - } - let tok = self.buf.next().unwrap(); - self.pos.next_char(); - dec.push(tok); - } - } - - if dec.len() == 1 { - return TokenKind::Error("Expected digit.".into()); - } - - whole.push_str(&dec); - - TokenKind::Number(whole) - } - fn lex_hash(&mut self) -> TokenKind { self.buf.next(); self.pos.next_char(); @@ -354,39 +138,6 @@ impl<'a> Lexer<'a> { TokenKind::Symbol(Symbol::Hash) } - fn lex_variable(&mut self) -> TokenKind { - self.buf.next(); - self.pos.next_char(); - let mut name = String::with_capacity(99); - if let Some(c) = self.buf.peek() { - if c == &'=' { - return TokenKind::Symbol(Symbol::Dollar); - } else if !c.is_alphabetic() && c != &'-' && c != &'_' { - return TokenKind::Error("Expected identifier.".into()); - } else { - self.pos.next_char(); - name.push(*c); - } - self.buf.next(); - } - while let Some(c) = self.buf.peek() { - if !c.is_alphanumeric() && c != &'-' && c != &'_' { - break; - } - let tok = self - .buf - .next() - .expect("this is impossible because we have already peeked"); - self.pos.next_char(); - name.push(tok); - } - if name.is_empty() { - TokenKind::Symbol(Symbol::Dollar) - } else { - TokenKind::Variable(name) - } - } - fn lex_ident(&mut self) -> TokenKind { let mut string = String::with_capacity(99); while let Some(c) = self.buf.peek() { @@ -429,4 +180,5 @@ impl<'a> Lexer<'a> { TokenKind::Ident(string) } + */ } diff --git a/src/lib.rs b/src/lib.rs index 2faca40..fd315ec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,18 +85,20 @@ use std::iter::{Iterator, Peekable}; use std::path::Path; use crate::atrule::{eat_include, AtRule, AtRuleKind, Function, Mixin}; -use crate::common::{Pos, Symbol, Whitespace}; +use crate::common::Pos; use crate::css::Css; -use crate::error::SassError; -pub use crate::error::SassResult; +pub use crate::error::{SassError, SassResult}; use crate::format::PrettyPrinter; use crate::imports::import; use crate::lexer::Lexer; use crate::scope::{insert_global_var, Scope, GLOBAL_SCOPE}; use crate::selector::Selector; use crate::style::Style; -pub(crate) use crate::token::{Token, TokenKind}; -use crate::utils::{devour_whitespace, eat_variable_value, VariableDecl}; +pub(crate) use crate::token::Token; +use crate::utils::{ + devour_whitespace, eat_comment, eat_ident, eat_variable_value, parse_quoted_string, + read_until_newline, VariableDecl, +}; use crate::value::Value; mod args; @@ -280,37 +282,25 @@ impl<'a> StyleSheetParser<'a> { let mut rules: Vec = Vec::new(); while let Some(Token { kind, .. }) = self.lexer.peek() { match kind { - TokenKind::Ident(_) - | TokenKind::Interpolation - | TokenKind::Symbol(Symbol::OpenSquareBrace) - | TokenKind::Symbol(Symbol::Hash) - | TokenKind::Symbol(Symbol::Colon) - | TokenKind::Symbol(Symbol::Mul) - | TokenKind::Symbol(Symbol::Percent) - | TokenKind::Symbol(Symbol::Period) => rules + 'a'..='z' | 'A'..='Z' | '_' | '-' + | '[' | '#' | ':' | '*' | '%' | '.' => rules .extend(self.eat_rules(&Selector::new(), &mut GLOBAL_SCOPE.with(|s| s.borrow().clone()))?), - TokenKind::Whitespace(_) => { + &'\t' | &'\n' | ' ' => { self.lexer.next(); continue; } - TokenKind::Variable(_) => { - let Token { pos, kind } = self - .lexer - .next() - .expect("this must exist because we have already peeked"); - let name = match kind { - TokenKind::Variable(n) => n, - _ => unsafe { std::hint::unreachable_unchecked() }, - }; + '$' => { + self.lexer.next(); + let name = eat_ident(&mut self.lexer, &Scope::new(), &Selector::new())?; devour_whitespace(&mut self.lexer); if self .lexer .next() - .unwrap_or_else(|| self.error(pos, "expected value after variable")) + .unwrap() .kind - != TokenKind::Symbol(Symbol::Colon) + != ':' { - self.error(pos, "unexpected variable use at toplevel"); + return Err("expected \":\".".into()); } let VariableDecl { val, default, .. } = eat_variable_value(&mut self.lexer, &GLOBAL_SCOPE.with(|s| s.borrow().clone()), &Selector::new())?; @@ -325,111 +315,92 @@ impl<'a> StyleSheetParser<'a> { } })? } - TokenKind::MultilineComment(_) => { - let comment = match self - .lexer - .next() - .expect("this must exist because we have already peeked") - .kind - { - TokenKind::MultilineComment(c) => c, - _ => unsafe { std::hint::unreachable_unchecked() }, - }; - rules.push(Stmt::MultilineComment(comment)); + '/' => { + self.lexer.next(); + if '*' == self.lexer.peek().unwrap().kind { + self.lexer.next(); + rules.push(Stmt::MultilineComment(eat_comment(&mut self.lexer, &Scope::new(), &Selector::new())?)); + } else if '/' == self.lexer.peek().unwrap().kind { + read_until_newline(&mut self.lexer); + devour_whitespace(&mut self.lexer); + } else { + todo!() + } } - TokenKind::AtRule(AtRuleKind::Include) => rules.extend(eat_include( - &mut self.lexer, - &GLOBAL_SCOPE.with(|s| s.borrow().clone()), - &Selector::new(), - )?), - TokenKind::AtRule(AtRuleKind::Import) => { - let Token { pos, .. } = self - .lexer - .next() - .expect("this must exist because we have already peeked"); - devour_whitespace(&mut self.lexer); - let mut file_name = String::new(); - match self - .lexer - .next() - .unwrap_or_else(|| self.error(pos, "expected value after @import")) - .kind - { - TokenKind::Symbol(Symbol::DoubleQuote) => { - while let Some(tok) = self.lexer.next() { - if tok.kind == TokenKind::Symbol(Symbol::DoubleQuote) { - break; - } - file_name.push_str(&tok.kind.to_string()); - } - } - TokenKind::Symbol(Symbol::SingleQuote) => { - while let Some(tok) = self.lexer.next() { - if tok.kind == TokenKind::Symbol(Symbol::SingleQuote) { - break; - } - file_name.push_str(&tok.kind.to_string()); - } - } - _ => todo!("expected ' or \" after @import"), - } - let Token { kind, pos } = self - .lexer - .next() - .expect("this must exist because we have already peeked"); - if kind != TokenKind::Symbol(Symbol::SemiColon) { - self.error(pos, "expected `;` after @import declaration"); + '@' => { + self.lexer.next(); + let at_rule_kind = eat_ident(&mut self.lexer, &Scope::new(), &Selector::new())?; + if at_rule_kind.is_empty() { + return Err("Expected identifier.".into()); } + match AtRuleKind::from(at_rule_kind.as_str()) { + AtRuleKind::Include => rules.extend(eat_include( + &mut self.lexer, + &GLOBAL_SCOPE.with(|s| s.borrow().clone()), + &Selector::new(), + )?), + AtRuleKind::Import => { + devour_whitespace(&mut self.lexer); + let mut file_name = String::new(); + match self + .lexer + .next() + .unwrap() + .kind + { + q @ '"' | q @ '\'' => { + file_name.push_str(&parse_quoted_string(&mut self.lexer, &Scope::new(), q, &Selector::new())?.unquote().to_string()); + } + _ => todo!("expected ' or \" after @import"), + } + if self.lexer.next().unwrap().kind != ';' { + todo!("no semicolon after @import"); + } - let (new_rules, new_scope) = import(file_name)?; - rules.extend(new_rules); - GLOBAL_SCOPE.with(|s| { - s.borrow_mut().extend(new_scope); - }); - } - TokenKind::AtRule(_) => { - if let Some(Token { - kind: TokenKind::AtRule(ref rule), - pos, - }) = self.lexer.next() - { - match AtRule::from_tokens(rule, pos, &mut self.lexer, &mut GLOBAL_SCOPE.with(|s| s.borrow().clone()), &Selector::new())? { - AtRule::Mixin(name, mixin) => { - GLOBAL_SCOPE.with(|s| { - s.borrow_mut().insert_mixin(&name, *mixin); - }); - } - AtRule::Function(name, func) => { - GLOBAL_SCOPE.with(|s| { - s.borrow_mut().insert_fn(&name, *func); - }); - } - AtRule::Charset => continue, - AtRule::Error(pos, message) => self.error(pos, &message), - AtRule::Warn(pos, message) => self.warn(pos, &message), - AtRule::Debug(pos, message) => self.debug(pos, &message), - AtRule::Return(_) => { - return Err("This at-rule is not allowed here.".into()) - } - AtRule::For(s) => rules.extend(s), - AtRule::Content => return Err("@content is only allowed within mixin declarations.".into()), - AtRule::If(i) => { - rules.extend(i.eval(&mut Scope::new(), &Selector::new())?); - } - u @ AtRule::Unknown(..) => rules.push(Stmt::AtRule(u)), + let (new_rules, new_scope) = import(file_name)?; + rules.extend(new_rules); + GLOBAL_SCOPE.with(|s| { + s.borrow_mut().extend(new_scope); + }); } + v => { + match AtRule::from_tokens(&v, Pos::new(), &mut self.lexer, &mut GLOBAL_SCOPE.with(|s| s.borrow().clone()), &Selector::new())? { + AtRule::Mixin(name, mixin) => { + GLOBAL_SCOPE.with(|s| { + s.borrow_mut().insert_mixin(&name, *mixin); + }); + } + AtRule::Function(name, func) => { + GLOBAL_SCOPE.with(|s| { + s.borrow_mut().insert_fn(&name, *func); + }); + } + AtRule::Charset => continue, + AtRule::Error(pos, message) => self.error(pos, &message), + AtRule::Warn(pos, message) => self.warn(pos, &message), + AtRule::Debug(pos, message) => self.debug(pos, &message), + AtRule::Return(_) => { + return Err("This at-rule is not allowed here.".into()) + } + AtRule::For(s) => rules.extend(s), + AtRule::Content => return Err("@content is only allowed within mixin declarations.".into()), + AtRule::If(i) => { + rules.extend(i.eval(&mut Scope::new(), &Selector::new())?); + } + u @ AtRule::Unknown(..) => rules.push(Stmt::AtRule(u)), + } + } } - } - TokenKind::Symbol(Symbol::BitAnd) => { + }, + '&' => { return Err( "Base-level rules cannot contain the parent-selector-referencing character '&'.".into(), ) } - TokenKind::Error(e) => return Err(e.clone()), _ => match dbg!(self.lexer.next()) { Some(Token { pos, .. }) => self.error(pos, "unexpected toplevel token"), _ => unsafe { std::hint::unreachable_unchecked() }, - }, + } }; } Ok((rules, GLOBAL_SCOPE.with(|s| s.borrow().clone()))) @@ -495,7 +466,7 @@ pub(crate) fn eat_expr>( let mut values = Vec::with_capacity(5); while let Some(tok) = toks.peek() { match &tok.kind { - TokenKind::Symbol(Symbol::Colon) => { + ':' => { let tok = toks.next(); if devour_whitespace(toks) { let prop = Style::parse_property( @@ -509,7 +480,7 @@ pub(crate) fn eat_expr>( values.push(tok.unwrap()); } } - TokenKind::Symbol(Symbol::SemiColon) => { + ';' => { toks.next(); devour_whitespace(toks); // special edge case where there was no space between the colon @@ -517,6 +488,7 @@ pub(crate) fn eat_expr>( let mut v = values.into_iter().peekable(); devour_whitespace(&mut v); if v.peek().is_none() { + devour_whitespace(toks); return Ok(Some(Expr::Style(Box::new(Style { property: String::new(), value: Value::Null, @@ -526,7 +498,7 @@ pub(crate) fn eat_expr>( let value = Style::parse_value(&mut v, scope, super_selector)?; return Ok(Some(Expr::Style(Box::new(Style { property, value })))); } - TokenKind::Symbol(Symbol::CloseCurlyBrace) => { + '}' => { if values.is_empty() { toks.next(); devour_whitespace(toks); @@ -542,7 +514,7 @@ pub(crate) fn eat_expr>( return Ok(Some(Expr::Style(Box::new(Style { property, value })))); } } - TokenKind::Symbol(Symbol::OpenCurlyBrace) => { + '{' => { toks.next(); devour_whitespace(toks); return Ok(Some(Expr::Selector(Selector::from_tokens( @@ -551,18 +523,15 @@ pub(crate) fn eat_expr>( super_selector, )?))); } - TokenKind::Variable(_) => { - let tok = toks - .next() - .expect("this must exist because we have already peeked"); - let pos = tok.pos(); - let name = match tok.kind { - TokenKind::Variable(n) => n, - _ => unsafe { std::hint::unreachable_unchecked() }, - }; - if let TokenKind::Symbol(Symbol::Colon) = - toks.peek().expect("expected something after variable").kind - { + '$' => { + let tok = toks.next().unwrap(); + if toks.peek().unwrap().kind == '=' { + values.push(tok); + values.push(toks.next().unwrap()); + continue; + } + let name = eat_ident(toks, scope, super_selector)?; + if toks.peek().unwrap().kind == ':' { toks.next(); devour_whitespace(toks); let VariableDecl { @@ -577,60 +546,67 @@ pub(crate) fn eat_expr>( return Ok(Some(Expr::VariableDecl(name, Box::new(val)))); } } else { - values.push(Token { - kind: TokenKind::Variable(name), - pos, - }); + todo!() } } - TokenKind::MultilineComment(_) => { - let tok = toks - .next() - .expect("this must exist because we have already peeked"); - devour_whitespace(toks); - if values.is_empty() { - let s = match tok.kind { - TokenKind::MultilineComment(s) => s, - _ => unsafe { std::hint::unreachable_unchecked() }, - }; - return Ok(Some(Expr::MultilineComment(s))); + '/' => { + let tok = toks.next().unwrap(); + let peeked = toks.peek().ok_or("expected more input.")?; + if peeked.kind == '/' { + read_until_newline(toks); + devour_whitespace(toks); + continue; + } else if values.is_empty() && peeked.kind == '*' { + toks.next(); + return Ok(Some(Expr::MultilineComment(eat_comment( + toks, + scope, + super_selector, + )?))); } else { values.push(tok); } } - TokenKind::AtRule(AtRuleKind::Include) => { - return Ok(Some(Expr::Include(eat_include( - toks, - scope, - super_selector, - )?))); - } - TokenKind::AtRule(_) => { - if let Some(Token { - kind: TokenKind::AtRule(ref rule), - pos, - }) = toks.next() - { - return match AtRule::from_tokens(rule, pos, toks, scope, super_selector)? { - AtRule::Mixin(name, mixin) => Ok(Some(Expr::MixinDecl(name, mixin))), - AtRule::Function(name, func) => Ok(Some(Expr::FunctionDecl(name, func))), - AtRule::Charset => todo!("@charset as expr"), - AtRule::Debug(a, b) => Ok(Some(Expr::Debug(a, b))), - AtRule::Warn(a, b) => Ok(Some(Expr::Warn(a, b))), - AtRule::Error(pos, err) => Err(SassError::new(err, pos)), - a @ AtRule::Return(_) => Ok(Some(Expr::AtRule(a))), - c @ AtRule::Content => Ok(Some(Expr::AtRule(c))), - f @ AtRule::If(..) => Ok(Some(Expr::AtRule(f))), - f @ AtRule::For(..) => Ok(Some(Expr::AtRule(f))), - u @ AtRule::Unknown(..) => Ok(Some(Expr::AtRule(u))), - }; + '@' => { + let pos = toks.next().unwrap().pos(); + match AtRuleKind::from(eat_ident(toks, scope, super_selector)?.as_str()) { + AtRuleKind::Include => { + devour_whitespace(toks); + return Ok(Some(Expr::Include(eat_include( + toks, + scope, + super_selector, + )?))); + } + v => { + devour_whitespace(toks); + return match AtRule::from_tokens(&v, pos, toks, scope, super_selector)? { + AtRule::Mixin(name, mixin) => Ok(Some(Expr::MixinDecl(name, mixin))), + AtRule::Function(name, func) => { + Ok(Some(Expr::FunctionDecl(name, func))) + } + AtRule::Charset => todo!("@charset as expr"), + AtRule::Debug(a, b) => Ok(Some(Expr::Debug(a, b))), + AtRule::Warn(a, b) => Ok(Some(Expr::Warn(a, b))), + AtRule::Error(pos, err) => Err(SassError::new(err, pos)), + a @ AtRule::Return(_) => Ok(Some(Expr::AtRule(a))), + c @ AtRule::Content => Ok(Some(Expr::AtRule(c))), + f @ AtRule::If(..) => Ok(Some(Expr::AtRule(f))), + f @ AtRule::For(..) => Ok(Some(Expr::AtRule(f))), + u @ AtRule::Unknown(..) => Ok(Some(Expr::AtRule(u))), + }; + } } } - TokenKind::Interpolation => values.extend(eat_interpolation(toks)), - _ => match toks.next() { - Some(tok) => values.push(tok), - _ => unsafe { std::hint::unreachable_unchecked() }, - }, + '#' => { + values.push(toks.next().unwrap()); + let next = toks.next().unwrap(); + values.push(next); + if next.kind == '{' { + values.extend(eat_interpolation(toks)); + } + } + _ => values.push(toks.next().unwrap()), }; } Ok(None) @@ -638,12 +614,11 @@ pub(crate) fn eat_expr>( fn eat_interpolation>(toks: &mut Peekable) -> Vec { let mut vals = Vec::new(); - let mut n = 0; + let mut n = 1; for tok in toks { match tok.kind { - TokenKind::Symbol(Symbol::OpenCurlyBrace) => n += 1, - TokenKind::Symbol(Symbol::CloseCurlyBrace) => n -= 1, - TokenKind::Interpolation => n += 1, + '{' => n += 1, + '}' => n -= 1, _ => {} } vals.push(tok); diff --git a/src/selector.rs b/src/selector.rs index c0f9bc0..52a48be 100644 --- a/src/selector.rs +++ b/src/selector.rs @@ -2,15 +2,14 @@ use std::fmt::{self, Display, Write}; use std::iter::Peekable; use std::string::ToString; -use crate::common::{Symbol, Whitespace}; use crate::error::SassResult; use crate::lexer::Lexer; use crate::scope::Scope; use crate::utils::{ - devour_whitespace, devour_whitespace_or_comment, flatten_ident, parse_interpolation, - parse_quoted_string, IsWhitespace, + devour_whitespace, devour_whitespace_or_comment, eat_ident, eat_ident_no_interpolation, + parse_interpolation, parse_quoted_string, IsWhitespace, }; -use crate::{Token, TokenKind}; +use crate::Token; #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct Selector(pub Vec); @@ -198,17 +197,18 @@ impl<'a> SelectorParser<'a> { ) -> SassResult<()> { if let Some(tok) = tokens.next() { match tok.kind { - TokenKind::Ident(s) => { - if let Some(Token { - kind: TokenKind::Symbol(Symbol::OpenParen), - .. - }) = tokens.peek() - { + v @ 'a'..='z' | v @ 'A'..='Z' | v @ '-' | v @ '_' => { + let s = format!( + "{}{}", + v, + eat_ident(tokens, &self.scope, &self.super_selector)? + ); + if let Some(Token { kind: '(', .. }) = tokens.peek() { tokens.next(); - devour_whitespace_or_comment(tokens); + devour_whitespace(tokens); let mut toks = String::new(); while let Some(Token { kind, .. }) = tokens.peek() { - if kind == &TokenKind::Symbol(Symbol::CloseParen) { + if kind == &')' { tokens.next(); break; } @@ -224,14 +224,9 @@ impl<'a> SelectorParser<'a> { self.selectors.push(SelectorKind::Pseudo(s)) } } - TokenKind::Symbol(Symbol::Colon) => { - if let Some(Token { - kind: TokenKind::Ident(s), - .. - }) = tokens.next() - { - self.selectors.push(SelectorKind::PseudoElement(s)) - } + ':' => { + let s = eat_ident(tokens, &self.scope, &self.super_selector)?; + self.selectors.push(SelectorKind::PseudoElement(s)) } _ => return Err("Expected identifier.".into()), } @@ -253,12 +248,8 @@ impl<'a> SelectorParser<'a> { &mut self, tokens: &'_ mut Peekable, ) -> SassResult<()> { - if devour_whitespace_or_comment(tokens) { - if let Some(Token { - kind: TokenKind::Symbol(Symbol::Comma), - .. - }) = tokens.peek() - { + if devour_whitespace_or_comment(tokens)? { + if let Some(Token { kind: ',', .. }) = tokens.peek() { tokens.next(); self.selectors.push(SelectorKind::Multiple); return Ok(()); @@ -268,49 +259,50 @@ impl<'a> SelectorParser<'a> { } if let Some(Token { kind, .. }) = tokens.next() { match kind { - TokenKind::Ident(v) | TokenKind::Number(v) => { - self.selectors.push(SelectorKind::Element(v)) + v @ 'a'..='z' | v @ 'A'..='Z' | v @ '-' | v @ '_' => { + let s = format!("{}{}", v, eat_ident_no_interpolation(tokens)?); + self.selectors.push(SelectorKind::Element(s)) } - TokenKind::Symbol(Symbol::Period) => self.selectors.push(SelectorKind::Class), - TokenKind::Symbol(Symbol::Hash) => self.selectors.push(SelectorKind::Id), - TokenKind::Symbol(Symbol::Colon) => self.consume_pseudo_selector(tokens)?, - TokenKind::Symbol(Symbol::Comma) => { + '.' => self.selectors.push(SelectorKind::Class), + '#' => { + if tokens.peek().unwrap().kind == '{' { + tokens.next(); + self.is_interpolated = true; + self.tokens_to_selectors( + &mut Lexer::new( + &parse_interpolation(tokens, self.scope, self.super_selector)? + .to_string(), + ) + .peekable(), + )?; + self.is_interpolated = false; + } else { + self.selectors.push(SelectorKind::Id) + } + } + ':' => self.consume_pseudo_selector(tokens)?, + ',' => { self.selectors.push(SelectorKind::Multiple); - if tokens.peek().unwrap().kind == TokenKind::Whitespace(Whitespace::Newline) { + if tokens.peek().unwrap().kind == '\n' { self.selectors.push(SelectorKind::Newline); devour_whitespace(tokens); } } - TokenKind::Symbol(Symbol::Gt) => self.selectors.push(SelectorKind::ImmediateChild), - TokenKind::Symbol(Symbol::Plus) => self.selectors.push(SelectorKind::Following), - TokenKind::Symbol(Symbol::Tilde) => self.selectors.push(SelectorKind::Preceding), - TokenKind::Symbol(Symbol::Mul) => self.selectors.push(SelectorKind::Universal), - TokenKind::Symbol(Symbol::Percent) => { - self.selectors.push(SelectorKind::Placeholder) - } - TokenKind::Symbol(Symbol::BitAnd) => self.selectors.push(if self.is_interpolated { + '>' => self.selectors.push(SelectorKind::ImmediateChild), + '+' => self.selectors.push(SelectorKind::Following), + '~' => self.selectors.push(SelectorKind::Preceding), + '*' => self.selectors.push(SelectorKind::Universal), + '%' => self.selectors.push(SelectorKind::Placeholder), + '&' => self.selectors.push(if self.is_interpolated { SelectorKind::InterpolatedSuper } else { SelectorKind::Super }), - TokenKind::Interpolation => { - self.is_interpolated = true; - self.tokens_to_selectors( - &mut Lexer::new( - &parse_interpolation( - tokens, - self.scope, - &Selector(vec![SelectorKind::Element(String::from("&"))]), - )? - .to_string(), - ) - .peekable(), - )?; - self.is_interpolated = false; - } - TokenKind::Symbol(Symbol::OpenSquareBrace) => self.selectors.push( - Attribute::from_tokens(tokens, self.scope, self.super_selector)?, - ), + '[' => self.selectors.push(Attribute::from_tokens( + tokens, + self.scope, + self.super_selector, + )?), _ => todo!("unimplemented selector"), }; } @@ -418,16 +410,14 @@ impl Attribute { devour_whitespace(toks); let attr = if let Some(t) = toks.next() { match t.kind { - TokenKind::Ident(mut s) => { - s.push_str(&flatten_ident(toks, scope, super_selector)?); - s + v @ 'a'..='z' | v @ 'A'..='Z' | v @ '-' | v @ '_' => { + format!("{}{}", v, eat_ident(toks, scope, super_selector)?) } - TokenKind::Interpolation => { + '#' if toks.next().unwrap().kind == '{' => { parse_interpolation(toks, scope, super_selector)?.to_string() } - q @ TokenKind::Symbol(Symbol::DoubleQuote) - | q @ TokenKind::Symbol(Symbol::SingleQuote) => { - parse_quoted_string(toks, scope, &q, super_selector)?.to_string() + q @ '"' | q @ '\'' => { + parse_quoted_string(toks, scope, q, super_selector)?.to_string() } _ => return Err("Expected identifier.".into()), } @@ -439,20 +429,19 @@ impl Attribute { let kind = if let Some(t) = toks.next() { match t.kind { - TokenKind::Ident(s) if s.len() == 1 => { - devour_whitespace(toks); + v @ 'a'..='z' | v @ 'A'..='Z' => { match toks.next().unwrap().kind { - TokenKind::Symbol(Symbol::CloseSquareBrace) => {} + ']' => {} _ => return Err("expected \"]\".".into()), } return Ok(SelectorKind::Attribute(Attribute { kind: AttributeKind::Any, attr, value: String::new(), - modifier: s, + modifier: v.to_string(), })); } - TokenKind::Symbol(Symbol::CloseSquareBrace) => { + ']' => { return Ok(SelectorKind::Attribute(Attribute { kind: AttributeKind::Any, attr, @@ -460,12 +449,12 @@ impl Attribute { modifier: String::new(), })); } - TokenKind::Symbol(Symbol::Equal) => AttributeKind::Equals, - TokenKind::Symbol(Symbol::Tilde) => AttributeKind::InList, - TokenKind::Symbol(Symbol::BitOr) => AttributeKind::BeginsWithHyphenOrExact, - TokenKind::Symbol(Symbol::Xor) => AttributeKind::StartsWith, - TokenKind::Symbol(Symbol::Dollar) => AttributeKind::EndsWith, - TokenKind::Symbol(Symbol::Mul) => AttributeKind::Contains, + '=' => AttributeKind::Equals, + '~' => AttributeKind::InList, + '|' => AttributeKind::BeginsWithHyphenOrExact, + '^' => AttributeKind::StartsWith, + '$' => AttributeKind::EndsWith, + '*' => AttributeKind::Contains, _ => return Err("Expected \"]\".".into()), } } else { @@ -474,7 +463,7 @@ impl Attribute { if kind != AttributeKind::Equals { match toks.next().unwrap().kind { - TokenKind::Symbol(Symbol::Equal) => {} + '=' => {} _ => return Err("expected \"=\".".into()), } } @@ -483,13 +472,11 @@ impl Attribute { let value = if let Some(t) = toks.next() { match t.kind { - TokenKind::Ident(mut s) => { - s.push_str(&flatten_ident(toks, scope, super_selector)?); - s + v @ 'a'..='z' | v @ 'A'..='Z' | v @ '-' | v @ '_' => { + format!("{}{}", v, eat_ident(toks, scope, super_selector)?) } - q @ TokenKind::Symbol(Symbol::DoubleQuote) - | q @ TokenKind::Symbol(Symbol::SingleQuote) => { - parse_quoted_string(toks, scope, &q, super_selector)?.to_string() + q @ '"' | q @ '\'' => { + parse_quoted_string(toks, scope, q, super_selector)?.to_string() } _ => return Err("Expected identifier.".into()), } @@ -501,7 +488,7 @@ impl Attribute { let modifier = if let Some(t) = toks.next() { match t.kind { - TokenKind::Symbol(Symbol::CloseSquareBrace) => { + ']' => { return Ok(SelectorKind::Attribute(Attribute { kind, attr, @@ -509,12 +496,12 @@ impl Attribute { modifier: String::new(), })) } - TokenKind::Ident(s) if s.len() == 1 => { + v @ 'a'..='z' | v @ 'A'..='Z' => { match toks.next().unwrap().kind { - TokenKind::Symbol(Symbol::CloseSquareBrace) => {} + ']' => {} _ => return Err("expected \"]\".".into()), } - format!(" {}", s) + format!(" {}", v) } _ => return Err("Expected \"]\".".into()), } diff --git a/src/style.rs b/src/style.rs index 9a8d931..96e6844 100644 --- a/src/style.rs +++ b/src/style.rs @@ -1,13 +1,14 @@ use std::fmt::{self, Display}; use std::iter::Peekable; -use crate::common::{Pos, QuoteKind, Symbol}; use crate::error::SassResult; use crate::scope::Scope; use crate::selector::Selector; -use crate::utils::{devour_whitespace, parse_interpolation, parse_quoted_string}; +use crate::utils::{ + devour_whitespace, eat_ident, read_until_semicolon_or_open_or_closing_curly_brace, +}; use crate::value::Value; -use crate::{Expr, Token, TokenKind}; +use crate::{Expr, Token}; /// A style: `color: red` #[derive(Clone, Debug, Eq, PartialEq)] @@ -68,62 +69,12 @@ impl<'a> StyleParser<'a> { toks: &mut Peekable, scope: &Scope, ) -> SassResult { - let mut style = Vec::new(); - let mut n = 0; devour_whitespace(toks); - while let Some(tok) = toks.peek() { - match tok.kind { - TokenKind::MultilineComment(_) => { - toks.next(); - continue; - } - TokenKind::Interpolation => n += 1, - TokenKind::Symbol(Symbol::CloseCurlyBrace) => { - if n == 0 { - break; - } else { - // todo: toks.next() and push - n -= 1; - } - } - ref q @ TokenKind::Symbol(Symbol::DoubleQuote) - | ref q @ TokenKind::Symbol(Symbol::SingleQuote) => { - let q = q.clone(); - toks.next(); - let (s, q) = if let Value::Ident(s, q) = - parse_quoted_string(toks, scope, &q, self.super_selector)? - { - (s, q) - } else { - unreachable!() - }; - let quote_kind = Token::from_symbol(match q { - QuoteKind::Single => Symbol::SingleQuote, - QuoteKind::Double => Symbol::DoubleQuote, - _ => unreachable!(), - }); - style.push(quote_kind.clone()); - style.push(Token::from_string(s)); - style.push(quote_kind); - continue; - } - TokenKind::Symbol(Symbol::OpenCurlyBrace) - | TokenKind::Symbol(Symbol::SemiColon) => break, - TokenKind::Symbol(Symbol::BitAnd) => { - style.push(Token { - kind: TokenKind::Symbol(Symbol::BitAnd), - pos: Pos::new(), - }); - toks.next(); - continue; - } - _ => {} - }; - style.push(toks.next().unwrap()); - } Value::from_tokens( - &mut style.into_iter().peekable(), - self.scope, + &mut read_until_semicolon_or_open_or_closing_curly_brace(toks) + .into_iter() + .peekable(), + scope, self.super_selector, ) } @@ -138,13 +89,13 @@ impl<'a> StyleParser<'a> { devour_whitespace(toks); while let Some(tok) = toks.peek() { match tok.kind { - TokenKind::Symbol(Symbol::OpenCurlyBrace) => { + '{' => { toks.next(); devour_whitespace(toks); loop { let property = self.parse_property(toks, super_property.clone())?; if let Some(tok) = toks.peek() { - if tok.is_symbol(Symbol::OpenCurlyBrace) { + if tok.kind == '{' { match self.eat_style_group(toks, property, scope)? { Expr::Styles(s) => styles.extend(s), Expr::Style(s) => styles.push(*s), @@ -152,7 +103,7 @@ impl<'a> StyleParser<'a> { } devour_whitespace(toks); if let Some(tok) = toks.peek() { - if tok.is_symbol(Symbol::CloseCurlyBrace) { + if tok.kind == '}' { toks.next(); devour_whitespace(toks); return Ok(Expr::Styles(styles)); @@ -164,16 +115,17 @@ impl<'a> StyleParser<'a> { } } let value = self.parse_style_value(toks, scope)?; + dbg!(&value); match toks.peek().unwrap().kind { - TokenKind::Symbol(Symbol::CloseCurlyBrace) => { + '}' => { styles.push(Style { property, value }); } - TokenKind::Symbol(Symbol::SemiColon) => { + ';' => { toks.next(); devour_whitespace(toks); styles.push(Style { property, value }); } - TokenKind::Symbol(Symbol::OpenCurlyBrace) => { + '{' => { styles.push(Style { property: property.clone(), value, @@ -191,7 +143,7 @@ impl<'a> StyleParser<'a> { } if let Some(tok) = toks.peek() { match tok.kind { - TokenKind::Symbol(Symbol::CloseCurlyBrace) => { + '}' => { toks.next(); devour_whitespace(toks); return Ok(Expr::Styles(styles)); @@ -203,17 +155,14 @@ impl<'a> StyleParser<'a> { } _ => { let val = self.parse_style_value(toks, scope)?; - let t = match toks.peek() { - Some(tok) => tok, - None => return Err("expected more input.".into()), - }; + let t = toks.peek().ok_or("expected more input.")?; match t.kind { - TokenKind::Symbol(Symbol::CloseCurlyBrace) => {} - TokenKind::Symbol(Symbol::SemiColon) => { + '}' => {} + ';' => { toks.next(); devour_whitespace(toks); } - TokenKind::Symbol(Symbol::OpenCurlyBrace) => { + '{' => { let mut v = vec![Style { property: super_property.clone(), value: val, @@ -242,22 +191,14 @@ impl<'a> StyleParser<'a> { toks: &mut Peekable, mut super_property: String, ) -> SassResult { - let mut property = String::new(); - while let Some(Token { kind, .. }) = toks.next() { - match kind { - TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => continue, - TokenKind::Ident(ref s) => property.push_str(s), - TokenKind::Interpolation => property.push_str( - &parse_interpolation(toks, self.scope, self.super_selector)?.to_string(), - ), - TokenKind::Symbol(Symbol::Colon) => break, - TokenKind::Symbol(Symbol::BitAnd) => { - property.push_str(&self.super_selector.to_string()) - } - _ => property.push_str(&kind.to_string()), - }; - } devour_whitespace(toks); + let property = eat_ident(toks, &self.scope, &self.super_selector)?; + devour_whitespace(toks); + if toks.peek().is_some() && toks.peek().unwrap().kind == ':' { + toks.next(); + devour_whitespace(toks); + } + if super_property.is_empty() { Ok(property) } else { diff --git a/src/token.rs b/src/token.rs index 2b5725c..e27ed4a 100644 --- a/src/token.rs +++ b/src/token.rs @@ -1,33 +1,15 @@ -use std::fmt; +use crate::common::Pos; +use crate::utils::IsWhitespace; -use crate::atrule::AtRuleKind; -use crate::common::{Keyword, Op, Pos, Symbol, Whitespace}; -use crate::error::SassError; -use crate::utils::{IsComment, IsWhitespace}; - -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] pub(crate) struct Token { pub pos: Pos, - pub kind: TokenKind, + pub kind: char, } impl Token { - pub fn is_symbol(&self, s: Symbol) -> bool { - self.kind.is_symbol(s) - } - - pub fn from_string(s: String) -> Self { - Token { - kind: TokenKind::Ident(s), - pos: Pos::new(), - } - } - - pub fn from_symbol(s: Symbol) -> Self { - Token { - kind: TokenKind::Symbol(s), - pos: Pos::new(), - } + pub const fn new(pos: Pos, kind: char) -> Self { + Self { pos, kind } } pub const fn pos(&self) -> Pos { @@ -37,7 +19,7 @@ impl Token { impl IsWhitespace for Token { fn is_whitespace(&self) -> bool { - if let TokenKind::Whitespace(_) = self.kind { + if self.kind.is_whitespace() { return true; } false @@ -46,67 +28,67 @@ impl IsWhitespace for Token { impl IsWhitespace for &Token { fn is_whitespace(&self) -> bool { - if let TokenKind::Whitespace(_) = self.kind { + if self.kind.is_whitespace() { return true; } false } } -impl IsComment for Token { - fn is_comment(&self) -> bool { - if let TokenKind::MultilineComment(_) = self.kind { - return true; - } - false - } -} +// impl IsComment for Token { +// fn is_comment(&self) -> bool { +// if let TokenKind::MultilineComment(_) = self.kind { +// return true; +// } +// false +// } +// } -impl IsComment for &Token { - fn is_comment(&self) -> bool { - if let TokenKind::MultilineComment(_) = self.kind { - return true; - } - false - } -} +// impl IsComment for &Token { +// fn is_comment(&self) -> bool { +// if let TokenKind::MultilineComment(_) = self.kind { +// return true; +// } +// false +// } +// } -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) enum TokenKind { - Ident(String), - Symbol(Symbol), - AtRule(AtRuleKind), - Keyword(Keyword), - Number(String), - Whitespace(Whitespace), - Variable(String), - Op(Op), - MultilineComment(String), - Interpolation, - Error(SassError), -} +// #[derive(Clone, Debug, Eq, PartialEq)] +// pub(crate) enum TokenKind { +// Ident(String), +// Symbol(Symbol), +// AtRule(AtRuleKind), +// Keyword(Keyword), +// Number(String), +// Whitespace(Whitespace), +// Variable(String), +// Op(Op), +// MultilineComment(String), +// Interpolation, +// Error(SassError), +// } -impl TokenKind { - pub fn is_symbol(&self, s: Symbol) -> bool { - self == &TokenKind::Symbol(s) - } -} +// impl TokenKind { +// pub fn is_symbol(&self, s: Symbol) -> bool { +// self == &TokenKind::Symbol(s) +// } +// } -impl fmt::Display for TokenKind { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - TokenKind::Ident(s) | TokenKind::Number(s) => write!(f, "{}", s), - TokenKind::Symbol(s) => write!(f, "{}", s), - TokenKind::AtRule(s) => write!(f, "{}", s), - TokenKind::Op(s) => write!(f, "{}", s), - TokenKind::Whitespace(s) => write!(f, "{}", s), - TokenKind::Keyword(kw) => write!(f, "{}", kw), - TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s), - TokenKind::Variable(s) => write!(f, "{}", s), - TokenKind::Interpolation | TokenKind::Error(..) => { - panic!("we don't want to format TokenKind::Interpolation using Display") - } - } - } -} +// impl fmt::Display for TokenKind { +// #[inline] +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// match self { +// TokenKind::Ident(s) | TokenKind::Number(s) => write!(f, "{}", s), +// TokenKind::Symbol(s) => write!(f, "{}", s), +// TokenKind::AtRule(s) => write!(f, "{}", s), +// TokenKind::Op(s) => write!(f, "{}", s), +// TokenKind::Whitespace(s) => write!(f, "{}", s), +// TokenKind::Keyword(kw) => write!(f, "{}", kw), +// TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s), +// TokenKind::Variable(s) => write!(f, "{}", s), +// TokenKind::Interpolation | TokenKind::Error(..) => { +// panic!("we don't want to format TokenKind::Interpolation using Display") +// } +// } +// } +// } diff --git a/src/utils.rs b/src/utils.rs index 4a65b25..0d27f1e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,11 +1,11 @@ use std::iter::{Iterator, Peekable}; -use crate::common::{Keyword, QuoteKind, Symbol, Whitespace}; +use crate::common::QuoteKind; use crate::error::SassResult; use crate::lexer::Lexer; use crate::selector::Selector; use crate::value::Value; -use crate::{Scope, Token, TokenKind}; +use crate::{Scope, Token}; pub(crate) trait IsWhitespace { fn is_whitespace(&self) -> bool; @@ -29,18 +29,30 @@ pub(crate) trait IsComment { fn is_comment(&self) -> bool; } -pub(crate) fn devour_whitespace_or_comment, W: IsWhitespace + IsComment>( +pub(crate) fn devour_whitespace_or_comment>( s: &mut Peekable, -) -> bool { +) -> SassResult { let mut found_whitespace = false; while let Some(w) = s.peek() { - if !w.is_whitespace() && !w.is_comment() { + if w.kind == '/' { + s.next(); + match s.peek().unwrap().kind { + '*' => { + eat_comment(s, &Scope::new(), &Selector::new())?; + } + '/' => read_until_newline(s), + _ => return Err("Expected expression.".into()), + }; + found_whitespace = true; + continue; + } + if !w.is_whitespace() { break; } found_whitespace = true; s.next(); } - found_whitespace + Ok(found_whitespace) } pub(crate) fn parse_interpolation>( @@ -51,22 +63,31 @@ pub(crate) fn parse_interpolation>( let mut val = String::new(); while let Some(tok) = tokens.next() { match tok.kind { - TokenKind::Symbol(Symbol::CloseCurlyBrace) => break, - TokenKind::Symbol(Symbol::OpenCurlyBrace) => { - todo!("invalid character in interpolation") + '}' => break, + '{' => todo!("invalid character in interpolation"), + q @ '"' | q @ '\'' => { + val.push_str(&parse_quoted_string(tokens, scope, q, super_selector)?.to_string()) } - q @ TokenKind::Symbol(Symbol::DoubleQuote) - | q @ TokenKind::Symbol(Symbol::SingleQuote) => { - val.push_str(&parse_quoted_string(tokens, scope, &q, super_selector)?.to_string()) - } - TokenKind::Variable(ref v) => { - val.push_str(&scope.get_var(v)?.clone().unquote().to_string()) - } - TokenKind::Interpolation => val.push_str( - &Lexer::new(&parse_interpolation(tokens, scope, super_selector)?.to_string()) - .map(|x| x.kind.to_string()) - .collect::(), + '$' => val.push_str( + &scope + .get_var(&eat_ident(tokens, scope, super_selector)?)? + .clone() + .unquote() + .to_string(), ), + '#' => { + if tokens.next().unwrap().kind == '{' { + val.push_str( + &Lexer::new( + &parse_interpolation(tokens, scope, super_selector)?.to_string(), + ) + .map(|x| x.kind.to_string()) + .collect::(), + ) + } else { + return Err("Expected identifier.".into()); + } + } _ => val.push_str(&tok.kind.to_string()), } } @@ -106,9 +127,8 @@ pub(crate) fn read_until_open_curly_brace>( let mut n = 0; while let Some(tok) = toks.peek() { match tok.kind { - TokenKind::Symbol(Symbol::OpenCurlyBrace) => n += 1, - TokenKind::Symbol(Symbol::CloseCurlyBrace) => n -= 1, - TokenKind::Interpolation => n += 1, + '{' => n += 1, + '}' => n -= 1, _ => {} } if n == 1 { @@ -126,16 +146,15 @@ pub(crate) fn read_until_closing_curly_brace>( let mut nesting = 0; while let Some(tok) = toks.peek() { match tok.kind { - TokenKind::Symbol(Symbol::DoubleQuote) | TokenKind::Symbol(Symbol::SingleQuote) => { - let quote = toks.next().unwrap(); - t.push(quote.clone()); - t.extend(read_until_closing_quote(toks, "e.kind)); + q @ '"' | q @ '\'' => { + t.push(toks.next().unwrap()); + t.extend(read_until_closing_quote(toks, q)); } - TokenKind::Interpolation | TokenKind::Symbol(Symbol::OpenCurlyBrace) => { + '{' => { nesting += 1; t.push(toks.next().unwrap()); } - TokenKind::Symbol(Symbol::CloseCurlyBrace) => { + '}' => { if nesting == 0 { break; } else { @@ -152,70 +171,116 @@ pub(crate) fn read_until_closing_curly_brace>( pub(crate) fn read_until_closing_quote>( toks: &mut Peekable, - q: &TokenKind, + q: char, ) -> Vec { let mut is_escaped = false; let mut t = Vec::new(); for tok in toks { match tok.kind { - TokenKind::Symbol(Symbol::DoubleQuote) - if !is_escaped && q == &TokenKind::Symbol(Symbol::DoubleQuote) => - { + '"' if !is_escaped && q == '"' => { t.push(tok); break; } - TokenKind::Symbol(Symbol::DoubleQuote) if is_escaped => { + '"' if is_escaped => { t.push(tok); is_escaped = false; continue; } - TokenKind::Symbol(Symbol::SingleQuote) - if !is_escaped && q == &TokenKind::Symbol(Symbol::SingleQuote) => - { + '\'' if !is_escaped && q == '\'' => { t.push(tok); break; } - TokenKind::Symbol(Symbol::SingleQuote) if is_escaped => { + '\'' if is_escaped => { t.push(tok); is_escaped = false; continue; } - TokenKind::Symbol(Symbol::BackSlash) if !is_escaped => { + '\\' if !is_escaped => { t.push(tok); is_escaped = true } - TokenKind::Symbol(Symbol::BackSlash) => { + '\\' => { is_escaped = false; t.push(tok); continue; } + _ if is_escaped => { + is_escaped = false; + t.push(tok); + } _ => t.push(tok), } } t } -pub(crate) fn read_until_semicolon_or_curly_brace>( +pub(crate) fn read_until_semicolon_or_closing_curly_brace>( toks: &mut Peekable, ) -> Vec { let mut t = Vec::new(); let mut nesting = 0; while let Some(tok) = toks.peek() { match tok.kind { - TokenKind::Symbol(Symbol::SemiColon) => { - toks.next(); + ';' => { break; } - TokenKind::Symbol(Symbol::DoubleQuote) | TokenKind::Symbol(Symbol::SingleQuote) => { + '"' | '\'' => { let quote = toks.next().unwrap(); t.push(quote.clone()); - t.extend(read_until_closing_quote(toks, "e.kind)); + t.extend(read_until_closing_quote(toks, quote.kind)); } - TokenKind::Interpolation | TokenKind::Symbol(Symbol::OpenCurlyBrace) => { + '{' => { nesting += 1; t.push(toks.next().unwrap()); } - TokenKind::Symbol(Symbol::CloseCurlyBrace) => { + '}' => { + if nesting == 0 { + break; + } else { + nesting -= 1; + t.push(toks.next().unwrap()); + } + } + _ => t.push(toks.next().unwrap()), + } + } + devour_whitespace(toks); + t +} + +pub(crate) fn read_until_semicolon_or_open_or_closing_curly_brace>( + toks: &mut Peekable, +) -> Vec { + let mut t = Vec::new(); + let mut nesting = 0; + while let Some(tok) = toks.peek() { + match tok.kind { + ';' => { + break; + } + '"' | '\'' => { + let quote = toks.next().unwrap(); + t.push(quote.clone()); + t.extend(read_until_closing_quote(toks, quote.kind)); + } + '#' => { + t.push(toks.next().unwrap()); + match toks.peek().unwrap().kind { + '{' => nesting += 1, + ';' => break, + '}' => { + if nesting == 0 { + break; + } else { + nesting -= 1; + } + } + _ => {} + } + t.push(toks.next().unwrap()); + } + '{' => break, + '}' => { if nesting == 0 { break; } else { @@ -238,51 +303,125 @@ pub(crate) fn eat_variable_value>( devour_whitespace(toks); let mut default = false; let mut global = false; - let mut raw = read_until_semicolon_or_curly_brace(toks) + let mut raw = read_until_semicolon_or_closing_curly_brace(toks) .into_iter() - .filter(|t| match t.kind { - TokenKind::Keyword(Keyword::Default) => { - default = true; - false - } - TokenKind::Keyword(Keyword::Global) => { - global = true; - false - } - _ => true, - }) .peekable(); - let val = Value::from_tokens(&mut raw, scope, super_selector)?; + if toks.peek().unwrap().kind == ';' { + toks.next(); + } + let mut x = Vec::new(); + while let Some(tok) = raw.next() { + match tok.kind { + '!' => { + let next = raw.next().unwrap(); + match next.kind { + 'i' => todo!("!important"), + 'g' => { + if eat_ident(&mut raw, scope, super_selector)? + .to_ascii_lowercase() + .as_str() + == "lobal" + { + global = true; + } else { + return Err("Invalid flag name.".into()); + } + } + 'd' => { + if eat_ident(&mut raw, scope, super_selector)? + .to_ascii_lowercase() + .as_str() + == "efault" + { + default = true; + } else { + return Err("Invalid flag name.".into()); + } + } + _ => return Err("Invalid flag name.".into()), + } + } + _ => x.push(tok), + } + } + devour_whitespace(toks); + + let val = Value::from_tokens(&mut x.into_iter().peekable(), scope, super_selector)?; Ok(VariableDecl::new(val, default, global)) } -pub(crate) fn flatten_ident>( +pub(crate) fn eat_ident>( toks: &mut Peekable, scope: &Scope, super_selector: &Selector, ) -> SassResult { let mut s = String::new(); while let Some(tok) = toks.peek() { - match tok.kind.clone() { - TokenKind::Interpolation => { + match tok.kind { + '#' => { toks.next(); - s.push_str(&parse_interpolation(toks, scope, super_selector)?.to_string()) + if toks.peek().unwrap().kind == '{' { + toks.next(); + s.push_str(&parse_interpolation(toks, scope, super_selector)?.to_string()); + } else { + return Err("Expected identifier.".into()); + } } - TokenKind::Ident(ref i) => { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' => s.push(toks.next().unwrap().kind), + '\\' => { toks.next(); - s.push_str(i) + let mut n = String::new(); + while let Some(c) = toks.peek() { + if !c.kind.is_ascii_hexdigit() || n.len() > 6 { + break; + } + n.push(c.kind); + toks.next(); + } + if n.is_empty() { + let c = toks.next().unwrap().kind; + if (c == '-' && !s.is_empty()) || c.is_ascii_alphabetic() { + s.push(c); + } else { + s.push_str(&format!("\\{}", c)); + } + continue; + } + devour_whitespace(toks); + let c = std::char::from_u32(u32::from_str_radix(&n, 16).unwrap()).unwrap(); + if c.is_control() && c != '\t' { + s.push_str(&format!("\\{} ", n.to_ascii_lowercase())); + } else if !c.is_ascii_alphanumeric() && s.is_empty() && c.is_ascii() { + s.push_str(&format!("\\{}", c)); + } else if c.is_numeric() && s.is_empty() { + s.push_str(&format!("\\{} ", n)) + } else { + s.push(c); + }; } - TokenKind::Number(ref n) => { - toks.next(); - s.push_str(n) + _ => break, + } + } + Ok(s) +} + +pub(crate) fn eat_ident_no_interpolation>( + toks: &mut Peekable, +) -> SassResult { + let mut s = String::new(); + while let Some(tok) = toks.peek() { + match tok.kind { + '#' => { + break; } - TokenKind::Symbol(Symbol::BackSlash) => { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' => s.push(toks.next().unwrap().kind), + '\\' => { s.push('\\'); toks.next(); if let Some(tok) = toks.next() { match tok.kind { - TokenKind::Symbol(Symbol::Plus) => s.push('+'), - TokenKind::Symbol(Symbol::BackSlash) => s.push('\\'), + '+' => s.push('+'), + '\\' => s.push('\\'), _ => todo!("value after \\"), } } else { @@ -295,61 +434,148 @@ pub(crate) fn flatten_ident>( Ok(s) } +pub(crate) fn eat_number>(toks: &mut Peekable) -> SassResult { + let mut whole = String::new(); + while let Some(c) = toks.peek() { + if !c.kind.is_numeric() { + break; + } + let tok = toks.next().unwrap(); + whole.push(tok.kind); + } + + if toks.peek().is_none() { + return Ok(whole); + } + + let mut dec = String::new(); + + if toks.peek().unwrap().kind == '.' { + toks.next(); + dec.push('.'); + while let Some(c) = toks.peek() { + if !c.kind.is_numeric() { + break; + } + let tok = toks.next().unwrap(); + dec.push(tok.kind); + } + } + + if dec.len() == 1 { + return Err("Expected digit.".into()); + } + + whole.push_str(&dec); + Ok(whole) +} + +/// Eat tokens until a newline +/// +/// This exists largely to eat silent comments, "//" +/// We only have to check for \n as the lexing step normalizes all newline characters +/// +/// The newline is consumed +pub(crate) fn read_until_newline>(toks: &mut Peekable) { + while let Some(tok) = toks.next() { + if tok.kind == '\n' { + break; + } + } +} + +/// Eat and return the contents of a comment. +/// +/// This function assumes that the starting "/*" has already been consumed +/// The entirety of the comment, including the ending "*/" is consumed. +/// Note that the ending "*/" is not included in the output. +pub(crate) fn eat_comment>( + toks: &mut Peekable, + _scope: &Scope, + _super_selector: &Selector, +) -> SassResult { + let mut comment = String::new(); + while let Some(tok) = toks.next() { + if tok.kind == '*' { + if toks.peek().unwrap().kind == '/' { + toks.next(); + break; + } + } + comment.push(tok.kind); + } + devour_whitespace(toks); + Ok(comment) +} + pub(crate) fn parse_quoted_string>( toks: &mut Peekable, scope: &Scope, - q: &TokenKind, + q: char, super_selector: &Selector, ) -> SassResult { let mut s = String::new(); let mut is_escaped = false; let mut found_interpolation = false; while let Some(tok) = toks.next() { + // dbg!(&tok); match tok.kind { - TokenKind::Symbol(Symbol::DoubleQuote) - if !is_escaped && q == &TokenKind::Symbol(Symbol::DoubleQuote) => - { - break - } - TokenKind::Symbol(Symbol::DoubleQuote) if is_escaped => { + '"' if !is_escaped && q == '"' => break, + '"' if is_escaped => { s.push('"'); is_escaped = false; continue; } - TokenKind::Symbol(Symbol::SingleQuote) - if !is_escaped && q == &TokenKind::Symbol(Symbol::SingleQuote) => - { - break - } - TokenKind::Symbol(Symbol::SingleQuote) if is_escaped => { + '\'' if !is_escaped && q == '\'' => break, + '\'' if is_escaped => { s.push('\''); is_escaped = false; continue; } - TokenKind::Symbol(Symbol::BackSlash) if !is_escaped => is_escaped = true, - TokenKind::Symbol(Symbol::BackSlash) => { + '\\' if !is_escaped => is_escaped = true, + '\\' => { is_escaped = false; s.push('\\'); continue; } - TokenKind::Interpolation if !is_escaped => { - found_interpolation = true; - s.push_str(&parse_interpolation(toks, scope, super_selector)?.to_string()); - continue; + '#' if !is_escaped => { + if toks.peek().unwrap().kind == '{' { + toks.next(); + found_interpolation = true; + s.push_str(&parse_interpolation(toks, scope, super_selector)?.to_string()); + continue; + } else { + s.push('#'); + } } - TokenKind::Interpolation => { - s.push('#'); - s.push('{'); + '\n' => return Err("Expected \".".into()), + v if v.is_ascii_hexdigit() && is_escaped => { + let mut n = v.to_string(); + while let Some(c) = toks.peek() { + if !c.kind.is_ascii_hexdigit() || n.len() > 6 { + break; + } + n.push(c.kind); + toks.next(); + } + let c = std::char::from_u32(u32::from_str_radix(&n, 16).unwrap()).unwrap(); + if c.is_control() && c != '\t' && c != '\0' { + s.push_str(&format!("\\{} ", n.to_ascii_lowercase())); + } else { + s.push(c); + } is_escaped = false; continue; } - TokenKind::Whitespace(Whitespace::Newline) => return Err("Expected \".".into()), + _ if is_escaped => { + is_escaped = false; + } _ => {} } - if is_escaped && tok.kind != TokenKind::Symbol(Symbol::BackSlash) { + if is_escaped && tok.kind != '\\' { is_escaped = false; } - if tok.kind != TokenKind::Symbol(Symbol::BackSlash) { + if tok.kind != '\\' { s.push_str(&tok.kind.to_string()); } } @@ -357,8 +583,8 @@ pub(crate) fn parse_quoted_string>( QuoteKind::Double } else { match q { - TokenKind::Symbol(Symbol::DoubleQuote) => QuoteKind::Double, - TokenKind::Symbol(Symbol::SingleQuote) => QuoteKind::Single, + '"' => QuoteKind::Double, + '\'' => QuoteKind::Single, _ => unreachable!(), } }; diff --git a/src/value/parse.rs b/src/value/parse.rs index 2cc9237..a279456 100644 --- a/src/value/parse.rs +++ b/src/value/parse.rs @@ -8,64 +8,109 @@ use num_traits::pow; use crate::args::eat_call_args; use crate::builtin::GLOBAL_FUNCTIONS; use crate::color::Color; -use crate::common::{Brackets, Keyword, ListSeparator, Op, QuoteKind, Symbol}; +use crate::common::{Brackets, ListSeparator, Op, QuoteKind}; use crate::error::SassResult; use crate::scope::Scope; use crate::selector::Selector; use crate::unit::Unit; use crate::utils::{ - devour_whitespace_or_comment, flatten_ident, parse_interpolation, parse_quoted_string, + devour_whitespace, eat_comment, eat_ident, eat_number, parse_interpolation, + parse_quoted_string, read_until_newline, }; use crate::value::Value; -use crate::{Token, TokenKind}; +use crate::Token; use super::number::Number; -fn parse_hex(s: &str) -> Value { +fn parse_hex>( + toks: &mut Peekable, + scope: &Scope, + super_selector: &Selector, +) -> SassResult { + let mut s = String::with_capacity(8); + if toks.peek().unwrap().kind.is_ascii_digit() { + while let Some(c) = toks.peek() { + if !c.kind.is_ascii_hexdigit() || s.len() == 8 { + break; + } + s.push(toks.next().unwrap().kind); + } + } else { + let i = eat_ident(toks, scope, super_selector)?; + if i.chars().all(|c| c.is_ascii_hexdigit()) { + s = i; + } else { + return Ok(Value::Ident(format!("#{}", i), QuoteKind::None)); + } + } match s.len() { 3 => { - let v = match u16::from_str_radix(s, 16) { + let v = match u16::from_str_radix(&s, 16) { Ok(a) => a, - Err(_) => return Value::Ident(format!("#{}", s), QuoteKind::None), + Err(_) => return Ok(Value::Ident(format!("#{}", s), QuoteKind::None)), }; let red = (((v & 0xf00) >> 8) * 0x11) as u8; let green = (((v & 0x0f0) >> 4) * 0x11) as u8; let blue = ((v & 0x00f) * 0x11) as u8; - Value::Color(Color::new(red, green, blue, 1, format!("#{}", s))) + Ok(Value::Color(Color::new( + red, + green, + blue, + 1, + format!("#{}", s), + ))) } 4 => { - let v = match u16::from_str_radix(s, 16) { + let v = match u16::from_str_radix(&s, 16) { Ok(a) => a, - Err(_) => return Value::Ident(format!("#{}", s), QuoteKind::None), + Err(_) => return Ok(Value::Ident(format!("#{}", s), QuoteKind::None)), }; let red = (((v & 0xf000) >> 12) * 0x11) as u8; let green = (((v & 0x0f00) >> 8) * 0x11) as u8; let blue = (((v & 0x00f0) >> 4) * 0x11) as u8; let alpha = ((v & 0x000f) * 0x11) as u8; - Value::Color(Color::new(red, green, blue, alpha, format!("#{}", s))) + Ok(Value::Color(Color::new( + red, + green, + blue, + alpha, + format!("#{}", s), + ))) } 6 => { - let v = match u32::from_str_radix(s, 16) { + let v = match u32::from_str_radix(&s, 16) { Ok(a) => a, - Err(_) => return Value::Ident(format!("#{}", s), QuoteKind::None), + Err(_) => return Ok(Value::Ident(format!("#{}", s), QuoteKind::None)), }; let red = ((v & 0x00ff_0000) >> 16) as u8; let green = ((v & 0x0000_ff00) >> 8) as u8; let blue = (v & 0x0000_00ff) as u8; - Value::Color(Color::new(red, green, blue, 1, format!("#{}", s))) + Ok(Value::Color(Color::new( + red, + green, + blue, + 1, + format!("#{}", s), + ))) } 8 => { - let v = match u32::from_str_radix(s, 16) { + let v = match u32::from_str_radix(&s, 16) { Ok(a) => a, - Err(_) => return Value::Ident(format!("#{}", s), QuoteKind::None), + Err(_) => return Ok(Value::Ident(format!("#{}", s), QuoteKind::None)), }; let red = ((v & 0xff00_0000) >> 24) as u8; let green = ((v & 0x00ff_0000) >> 16) as u8; let blue = ((v & 0x0000_ff00) >> 8) as u8; let alpha = (v & 0x0000_00ff) as u8; - Value::Color(Color::new(red, green, blue, alpha, format!("#{}", s))) + Ok(Value::Color(Color::new( + red, + green, + blue, + alpha, + format!("#{}", s), + ))) } - _ => Value::Ident(format!("#{}", s), QuoteKind::None), + _ => Err("Expected hex digit.".into()), } } @@ -76,18 +121,16 @@ impl Value { super_selector: &Selector, ) -> SassResult { let left = Self::_from_tokens(toks, scope, super_selector)?; - devour_whitespace_or_comment(toks); + devour_whitespace(toks); let next = match toks.peek() { Some(x) => x, None => return Ok(left), }; match next.kind { - TokenKind::Symbol(Symbol::SemiColon) - | TokenKind::Symbol(Symbol::CloseParen) - | TokenKind::Symbol(Symbol::CloseSquareBrace) => Ok(left), - TokenKind::Symbol(Symbol::Comma) => { + ';' | ')' | ']' => Ok(left), + ',' => { toks.next(); - devour_whitespace_or_comment(toks); + devour_whitespace(toks); if toks.peek() == None { return Ok(Value::List( vec![left], @@ -95,13 +138,13 @@ impl Value { Brackets::None, )); } else if let Some(tok) = toks.peek() { - if tok.is_symbol(Symbol::CloseParen) { + if tok.kind == ')' { return Ok(Value::List( vec![left], ListSeparator::Comma, Brackets::None, )); - } else if tok.is_symbol(Symbol::CloseSquareBrace) { + } else if tok.kind == ']' { return Ok(Value::List( vec![left], ListSeparator::Comma, @@ -122,28 +165,92 @@ impl Value { )) } } - TokenKind::Symbol(Symbol::Plus) - | TokenKind::Symbol(Symbol::Minus) - | TokenKind::Op(_) - | TokenKind::Symbol(Symbol::Mul) - | TokenKind::Symbol(Symbol::Div) - | TokenKind::Symbol(Symbol::Percent) => { + '+' | '-' | '*' | '%' => { let op = match next.kind { - TokenKind::Symbol(Symbol::Plus) => Op::Plus, - TokenKind::Symbol(Symbol::Minus) => Op::Minus, - TokenKind::Symbol(Symbol::Mul) => Op::Mul, - TokenKind::Symbol(Symbol::Div) => Op::Div, - TokenKind::Symbol(Symbol::Percent) => Op::Rem, - TokenKind::Op(op) => op, + '+' => Op::Plus, + '-' => Op::Minus, + '*' => Op::Mul, + '/' => Op::Div, + '%' => Op::Rem, _ => unsafe { std::hint::unreachable_unchecked() }, }; toks.next(); - devour_whitespace_or_comment(toks); + devour_whitespace(toks); let right = Self::from_tokens(toks, scope, super_selector)?; Ok(Value::BinaryOp(Box::new(left), op, Box::new(right))) } + '=' => { + toks.next(); + if toks.peek().unwrap().kind == '=' { + toks.next(); + devour_whitespace(toks); + let right = Self::from_tokens(toks, scope, super_selector)?; + Ok(Value::BinaryOp(Box::new(left), Op::Equal, Box::new(right))) + } else { + return Err("expected \"=\".".into()); + } + } + '!' => { + toks.next(); + if toks.peek().unwrap().kind == '=' { + toks.next(); + devour_whitespace(toks); + let right = Self::from_tokens(toks, scope, super_selector)?; + Ok(Value::BinaryOp( + Box::new(left), + Op::NotEqual, + Box::new(right), + )) + } else if eat_ident(toks, scope, super_selector)? + .to_ascii_lowercase() + .as_str() + == "important" + { + Ok(Value::List( + vec![left, Value::Important], + ListSeparator::Space, + Brackets::None, + )) + } else { + return Err("Expected \"important\".".into()); + } + } + '/' => { + toks.next(); + match toks.peek().unwrap().kind { + v @ '*' | v @ '/' => { + toks.next(); + if v == '*' { + eat_comment(toks, &Scope::new(), &Selector::new())?; + } else { + read_until_newline(toks); + } + devour_whitespace(toks); + if toks.peek().is_none() { + return Ok(left); + } + let right = Self::from_tokens(toks, scope, super_selector)?; + if let Value::List(v, ListSeparator::Space, ..) = right { + let mut v2 = vec![left]; + v2.extend(v); + Ok(Value::List(v2, ListSeparator::Space, Brackets::None)) + } else { + Ok(Value::List( + vec![left, right], + ListSeparator::Space, + Brackets::None, + )) + } + } + _ => { + devour_whitespace(toks); + let right = Self::from_tokens(toks, scope, super_selector)?; + Ok(Value::BinaryOp(Box::new(left), Op::Div, Box::new(right))) + } + } + } _ => { - devour_whitespace_or_comment(toks); + devour_whitespace(toks); let right = Self::from_tokens(toks, scope, super_selector)?; if let Value::List(v, ListSeparator::Space, ..) = right { let mut v2 = vec![left]; @@ -165,20 +272,20 @@ impl Value { scope: &Scope, super_selector: &Selector, ) -> SassResult { - let kind = if let Some(tok) = toks.next() { + let kind = if let Some(tok) = toks.peek() { tok.kind } else { panic!("Unexpected EOF"); }; match kind { - TokenKind::Number(val) => { + '0'..='9' | '.' => { + let val = eat_number(toks)?; let unit = if let Some(tok) = toks.peek() { - match tok.kind.clone() { - TokenKind::Ident(i) => { - toks.next(); - Unit::from(&i) + match tok.kind { + 'a'..='z' | 'A'..='Z' | '_' => { + Unit::from(&eat_ident(toks, scope, super_selector)?) } - TokenKind::Symbol(Symbol::Percent) => { + '%' => { toks.next(); Unit::Percent } @@ -209,9 +316,10 @@ impl Value { }; Ok(Value::Dimension(Number::new(n), unit)) } - TokenKind::Symbol(Symbol::OpenParen) => { - devour_whitespace_or_comment(toks); - if toks.peek().unwrap().is_symbol(Symbol::CloseParen) { + '(' => { + toks.next(); + devour_whitespace(toks); + if toks.peek().unwrap().kind == ')' { toks.next(); return Ok(Value::List( Vec::new(), @@ -221,24 +329,26 @@ impl Value { } let val = Self::from_tokens(toks, scope, super_selector)?; let next = toks.next(); - if next.is_none() || !next.unwrap().is_symbol(Symbol::CloseParen) { + if next.is_none() || next.unwrap().kind != ')' { return Err("expected \")\".".into()); } Ok(Value::Paren(Box::new(val))) } - TokenKind::Symbol(Symbol::BitAnd) => { + '&' => { + toks.next(); Ok(Value::Ident(super_selector.to_string(), QuoteKind::None)) } - TokenKind::Symbol(Symbol::Hash) => { - Ok(parse_hex(&flatten_ident(toks, scope, super_selector)?)) + '#' => { + if let Ok(s) = eat_ident(toks, scope, super_selector) { + Ok(Value::Ident(s, QuoteKind::None)) + } else { + Ok(parse_hex(toks, scope, super_selector)?) + } } - TokenKind::Ident(mut s) => { - s.push_str(&flatten_ident(toks, scope, super_selector)?); + 'a'..='z' | 'A'..='Z' | '_' | '\\' => { + let mut s = eat_ident(toks, scope, super_selector)?; match toks.peek() { - Some(Token { - kind: TokenKind::Symbol(Symbol::OpenParen), - .. - }) => { + Some(Token { kind: '(', .. }) => { toks.next(); let func = match scope.get_fn(&s) { Ok(f) => f, @@ -254,17 +364,23 @@ impl Value { let mut unclosed_parens = 0; while let Some(t) = toks.next() { match &t.kind { - TokenKind::Symbol(Symbol::OpenParen) => { + '(' => { unclosed_parens += 1; } - TokenKind::Interpolation => s.push_str( + '#' if toks.next().unwrap().kind == '{' => s.push_str( &parse_interpolation(toks, scope, super_selector)? .to_string(), ), - TokenKind::Variable(v) => { - s.push_str(&scope.get_var(v)?.to_string()) - } - TokenKind::Symbol(Symbol::CloseParen) => { + '$' => s.push_str( + &scope + .get_var(&eat_ident( + toks, + scope, + super_selector, + )?)? + .to_string(), + ), + ')' => { if unclosed_parens <= 1 { s.push(')'); break; @@ -289,18 +405,24 @@ impl Value { if let Ok(c) = crate::color::ColorName::try_from(s.as_ref()) { Ok(Value::Color(c.into_color(s))) } else { - Ok(Value::Ident(s, QuoteKind::None)) + match s.to_ascii_lowercase().as_str() { + "true" => Ok(Value::True), + "false" => Ok(Value::False), + "null" => Ok(Value::Null), + _ => Ok(Value::Ident(s, QuoteKind::None)), + } } } } } - q @ TokenKind::Symbol(Symbol::DoubleQuote) - | q @ TokenKind::Symbol(Symbol::SingleQuote) => { - parse_quoted_string(toks, scope, &q, super_selector) + q @ '"' | q @ '\'' => { + toks.next(); + parse_quoted_string(toks, scope, q, super_selector) } - TokenKind::Symbol(Symbol::OpenSquareBrace) => { + '[' => { + toks.next(); if let Some(tok) = toks.peek() { - if tok.is_symbol(Symbol::CloseSquareBrace) { + if tok.kind == ']' { toks.next(); return Ok(Value::List( Vec::new(), @@ -310,69 +432,54 @@ impl Value { } } let inner = Self::from_tokens(toks, scope, super_selector)?; - devour_whitespace_or_comment(toks); + devour_whitespace(toks); toks.next(); Ok(match inner { Value::List(v, sep, ..) => Value::List(v, sep, Brackets::Bracketed), v => Value::List(vec![v], ListSeparator::Space, Brackets::Bracketed), }) } - TokenKind::Variable(ref v) => Ok(scope.get_var(v)?), - TokenKind::Interpolation => { - let mut s = parse_interpolation(toks, scope, super_selector)?.to_string(); - while let Some(tok) = toks.peek() { - match tok.kind.clone() { - TokenKind::Interpolation => { - toks.next(); - s.push_str( - &parse_interpolation(toks, scope, super_selector)?.to_string(), - ) - } - TokenKind::Ident(ref i) => { - toks.next(); - s.push_str(i) - } - _ => break, - } - } - Ok(Value::Ident(s, QuoteKind::None)) + '$' => { + toks.next(); + Ok(scope.get_var(&eat_ident(toks, scope, super_selector)?)?) } - TokenKind::Keyword(Keyword::Important) => Ok(Value::Important), - TokenKind::Keyword(Keyword::True) => Ok(Value::True), - TokenKind::Keyword(Keyword::False) => Ok(Value::False), - TokenKind::Keyword(Keyword::Null) => Ok(Value::Null), - TokenKind::Keyword(Keyword::From(s)) => Ok(Value::Ident(s, QuoteKind::None)), - TokenKind::Keyword(Keyword::Through(s)) => Ok(Value::Ident(s, QuoteKind::None)), - TokenKind::Keyword(Keyword::To(s)) => Ok(Value::Ident(s, QuoteKind::None)), - TokenKind::AtRule(_) => Err("expected \";\".".into()), - TokenKind::Error(e) => Err(e), - TokenKind::Symbol(Symbol::BackSlash) => { - if let Some(tok) = toks.next() { - match tok.kind { - TokenKind::Symbol(s) => Ok(Value::Ident( - format!("\\{}{}", s, flatten_ident(toks, scope, super_selector)?), - QuoteKind::None, - )), - TokenKind::Whitespace(w) => { - Ok(Value::Ident(format!("\\{}", w), QuoteKind::None)) - } - TokenKind::Ident(s) => Ok(Value::Ident(s, QuoteKind::None)), - _ => todo!("value after \\"), - } + '@' => Err("expected \";\".".into()), + '+' => { + toks.next(); + devour_whitespace(toks); + let v = Self::_from_tokens(toks, scope, super_selector)?; + Ok(Value::UnaryOp(Op::Plus, Box::new(v))) + } + '-' => { + toks.next(); + devour_whitespace(toks); + let v = Self::_from_tokens(toks, scope, super_selector)?; + Ok(Value::UnaryOp(Op::Minus, Box::new(v))) + } + '!' => { + toks.next(); + let v = eat_ident(toks, scope, super_selector)?; + if v.to_ascii_lowercase().as_str() == "important" { + Ok(Value::Important) + } else { + Err("Expected \"important\".".into()) + } + } + '/' => { + toks.next(); + if '*' == toks.peek().unwrap().kind { + toks.next(); + eat_comment(toks, &Scope::new(), &Selector::new())?; + Self::_from_tokens(toks, scope, super_selector) + } else if '/' == toks.peek().unwrap().kind { + read_until_newline(toks); + devour_whitespace(toks); + Self::_from_tokens(toks, scope, super_selector) } else { todo!() } } - TokenKind::Op(Op::Plus) | TokenKind::Symbol(Symbol::Plus) => { - devour_whitespace_or_comment(toks); - let v = Self::_from_tokens(toks, scope, super_selector)?; - Ok(Value::UnaryOp(Op::Plus, Box::new(v))) - } - TokenKind::Op(Op::Minus) | TokenKind::Symbol(Symbol::Minus) => { - devour_whitespace_or_comment(toks); - let v = Self::_from_tokens(toks, scope, super_selector)?; - Ok(Value::UnaryOp(Op::Minus, Box::new(v))) - } + v if v.is_control() => Err("Expected expression.".into()), v => { dbg!(v); panic!("Unexpected token in value parsing") diff --git a/tests/if.rs b/tests/if.rs index c11e9cd..dc762c4 100644 --- a/tests/if.rs +++ b/tests/if.rs @@ -8,62 +8,62 @@ test!( "@if true {\n a {\n color: foo;\n}\n}\n", "a {\n color: foo;\n}\n" ); -// test!( -// if_inner_true, -// "a {\n @if true {\n color: foo;\n}\n}\n", -// "a {\n color: foo;\n}\n" -// ); -// test!( -// if_toplevel_false, -// "@if false {\n a {\n color: foo;\n}\n}\n", -// "" -// ); -// test!( -// if_inner_false, -// "a {\n @if false {\n color: foo;\n}\n}\n", -// "" -// ); -// test!( -// if_else_toplevel_true, -// "@if true {\n a {\n color: foo;\n}\n} @else {\n b {\n color: bar;\n}\n}\n", -// "a {\n color: foo;\n}\n" -// ); -// test!( -// if_else_inner_true, -// "a {\n @if true {\n color: foo;\n} @else {\n color: bar;\n}\n}\n", -// "a {\n color: foo;\n}\n" -// ); -// test!( -// if_else_toplevel_false, -// "@if false {\n a {\n color: foo;\n}\n} @else {\n a {\n color: bar;\n}\n}\n", -// "a {\n color: bar;\n}\n" -// ); -// test!( -// if_else_inner_false, -// "a {\n @if false {\n color: foo;\n} @else {\n color: bar;\n}\n}\n", -// "a {\n color: bar;\n}\n" -// ); -// error!( -// no_brace_after_else, -// "@if false {} @else -}", "Error: expected \"{\"." -// ); -// test!( -// if_else_if_no_else, -// "a {\n @if false {\n color: red;\n} @else if true {\n color: blue;\n}\n}\n", -// "a {\n color: blue;\n}\n" -// ); -// test!( -// if_false_else_if_false_else, -// "a {\n @if false {\n color: red;\n} @else if false {\n color: blue;\n} @else {\n color: green;\n}\n}\n", -// "a {\n color: green;\n}\n" -// ); -// test!( -// if_false_else_if_true_else, -// "a {\n @if false {\n color: red;\n} @else if true {\n color: blue;\n} @else {\n color: green;\n}\n}\n", -// "a {\n color: blue;\n}\n" -// ); -// test!( -// if_inner_style_missing_semicolon, -// "a {\n @if true {\n color: red\n }\n}\n", -// "a {\n color: red;\n}\n" -// ); +test!( + if_inner_true, + "a {\n @if true {\n color: foo;\n}\n}\n", + "a {\n color: foo;\n}\n" +); +test!( + if_toplevel_false, + "@if false {\n a {\n color: foo;\n}\n}\n", + "" +); +test!( + if_inner_false, + "a {\n @if false {\n color: foo;\n}\n}\n", + "" +); +test!( + if_else_toplevel_true, + "@if true {\n a {\n color: foo;\n}\n} @else {\n b {\n color: bar;\n}\n}\n", + "a {\n color: foo;\n}\n" +); +test!( + if_else_inner_true, + "a {\n @if true {\n color: foo;\n} @else {\n color: bar;\n}\n}\n", + "a {\n color: foo;\n}\n" +); +test!( + if_else_toplevel_false, + "@if false {\n a {\n color: foo;\n}\n} @else {\n a {\n color: bar;\n}\n}\n", + "a {\n color: bar;\n}\n" +); +test!( + if_else_inner_false, + "a {\n @if false {\n color: foo;\n} @else {\n color: bar;\n}\n}\n", + "a {\n color: bar;\n}\n" +); +error!( + no_brace_after_else, + "@if false {} @else -}", "Error: expected \"{\"." +); +test!( + if_else_if_no_else, + "a {\n @if false {\n color: red;\n} @else if true {\n color: blue;\n}\n}\n", + "a {\n color: blue;\n}\n" +); +test!( + if_false_else_if_false_else, + "a {\n @if false {\n color: red;\n} @else if false {\n color: blue;\n} @else {\n color: green;\n}\n}\n", + "a {\n color: green;\n}\n" +); +test!( + if_false_else_if_true_else, + "a {\n @if false {\n color: red;\n} @else if true {\n color: blue;\n} @else {\n color: green;\n}\n}\n", + "a {\n color: blue;\n}\n" +); +test!( + if_inner_style_missing_semicolon, + "a {\n @if true {\n color: red\n }\n}\n", + "a {\n color: red;\n}\n" +); diff --git a/tests/meta.rs b/tests/meta.rs index 94b5d16..3de82bf 100644 --- a/tests/meta.rs +++ b/tests/meta.rs @@ -295,11 +295,11 @@ test!( ); error!( function_exists_non_string, - "a {color:function-exists(12px)}", "Error: $name: 12px is not a string." + "a {color: function-exists(12px)}", "Error: $name: 12px is not a string." ); error!( mixin_exists_non_string, - "a {color:mixin-exists(12px)}", "Error: $name: 12px is not a string." + "a {color: mixin-exists(12px)}", "Error: $name: 12px is not a string." ); // test!( // inspect_empty_list, diff --git a/tests/misc.rs b/tests/misc.rs index 8866010..c526133 100644 --- a/tests/misc.rs +++ b/tests/misc.rs @@ -70,3 +70,9 @@ test!( "a {\n color: red😁\n}\n", "@charset \"UTF-8\";\na {\n color: red😁;\n}\n" ); +test!( + #[ignore] + no_space_before_style, + "a {\n color:red\n}\n", + "a {\n color: red;\n}\n" +); diff --git a/tests/mixins.rs b/tests/mixins.rs index b2dbb1a..acd9ca2 100644 --- a/tests/mixins.rs +++ b/tests/mixins.rs @@ -184,3 +184,8 @@ test!( "@mixin foo($a) {\n @content;\n}\n\na {\n @include foo(red) {\n color: red;\n }\n}\n", "a {\n color: red;\n}\n" ); +test!( + mixin_style_does_not_end_with_semicolon, + "@mixin foo {\n color: red\n}\n\na {\n @include foo;\n}\n", + "a {\n color: red;\n}\n" +); diff --git a/tests/str-escape.rs b/tests/str-escape.rs index e046df8..9aa497a 100644 --- a/tests/str-escape.rs +++ b/tests/str-escape.rs @@ -14,6 +14,7 @@ test!( "a {\n color: xx;\n}\n" ); test!( + #[ignore] escape_start_non_ascii, "a {\n color: ☃x \\☃x \\2603x;\n}\n", "@charset \"UTF-8\";\na {\n color: ☃x ☃x ☃x;\n}\n" @@ -100,3 +101,8 @@ test!( "a {\n color: \\0;\n}\n", "a {\n color: \\0 ;\n}\n" ); +test!( + escapes_non_hex_in_string, + "a {\n color: \"\\g\";\n}\n", + "a {\n color: \"g\";\n}\n" +);