Attribute selectors are parsed after lexing

src/lexer.rs

@@ -4,7 +4,6 @@ use std::str::Chars;
 
 use crate::atrule::AtRuleKind;
 use crate::common::{Keyword, Op, Pos, Symbol};
-use crate::selector::{Attribute, AttributeKind, CaseKind};
 use crate::{Token, TokenKind, Whitespace};
 
 // Rust does not allow us to escape '\f'
@@ -125,14 +124,12 @@ impl<'a> Iterator for Lexer<'a> {
             '|' => symbol!(self, BitOr),
             '/' => self.lex_forward_slash(),
             '%' => symbol!(self, Percent),
-            '[' => {
+            '[' => symbol!(self, OpenSquareBrace),
-                self.buf.next();
+            ']' => symbol!(self, CloseSquareBrace),
-                self.pos.next_char();
-                self.lex_attr()
-            }
             '!' => self.lex_exclamation(),
             '<' => symbol!(self, Lt),
             '>' => symbol!(self, Gt),
+            '^' => symbol!(self, Xor),
             '\0' => return None,
             &v => {
                 self.buf.next();
@@ -147,6 +144,7 @@ impl<'a> Iterator for Lexer<'a> {
     }
 }
 
+#[allow(dead_code)]
 fn is_whitespace(c: char) -> bool {
     c == ' ' || c == '\n' || c == '\r' || c == FORM_FEED
 }
@@ -189,6 +187,7 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    #[allow(dead_code)]
     fn devour_whitespace(&mut self) -> bool {
         let mut found_whitespace = false;
         while let Some(c) = self.buf.peek() {
@@ -292,139 +291,6 @@ impl<'a> Lexer<'a> {
         TokenKind::Symbol(Symbol::Hash)
     }
 
-    fn lex_attr(&mut self) -> TokenKind {
-        let mut attr = String::with_capacity(99);
-        self.devour_whitespace();
-        while let Some(c) = self.buf.peek() {
-            if !c.is_alphabetic() && c != &'-' && c != &'_' {
-                break;
-            }
-            let tok = self
-                .buf
-                .next()
-                .expect("this is impossible because we have already peeked");
-            self.pos.next_char();
-            attr.push(tok);
-        }
-
-        self.devour_whitespace();
-
-        let kind = match self
-            .buf
-            .next()
-            .expect("todo! expected kind (should be error)")
-        {
-            ']' => {
-                return TokenKind::Attribute(Attribute {
-                    kind: AttributeKind::Any,
-                    attr,
-                    value: String::new(),
-                    case_sensitive: CaseKind::Sensitive,
-                })
-            }
-            'i' => {
-                self.devour_whitespace();
-                assert!(self.buf.next() == Some(']'));
-                return TokenKind::Attribute(Attribute {
-                    kind: AttributeKind::Any,
-                    attr,
-                    value: String::new(),
-                    case_sensitive: CaseKind::InsensitiveLowercase,
-                });
-            }
-            'I' => {
-                self.devour_whitespace();
-                assert!(self.buf.next() == Some(']'));
-                return TokenKind::Attribute(Attribute {
-                    kind: AttributeKind::Any,
-                    attr,
-                    value: String::new(),
-                    case_sensitive: CaseKind::InsensitiveCapital,
-                });
-            }
-            '=' => AttributeKind::Equals,
-            '~' => AttributeKind::InList,
-            '|' => AttributeKind::BeginsWithHyphenOrExact,
-            '^' => AttributeKind::StartsWith,
-            '$' => AttributeKind::EndsWith,
-            '*' => AttributeKind::Contains,
-            _ => todo!("Expected ']'"),
-        };
-
-        if kind != AttributeKind::Equals {
-            assert!(self.buf.next() == Some('='));
-        }
-
-        self.devour_whitespace();
-
-        let mut value = String::with_capacity(99);
-        let case_sensitive = CaseKind::Sensitive;
-
-        while let Some(c) = self.buf.peek() {
-            if c == &']' || c.is_whitespace() {
-                break;
-            }
-
-            let tok = self
-                .buf
-                .next()
-                .expect("this is impossible because we have already peeked");
-            self.pos.next_char();
-            value.push(tok);
-        }
-
-        if self.devour_whitespace() {
-            let n = self.buf.next();
-            match n {
-                Some('i') | Some('I') => {
-                    let case_sensitive = match n {
-                        Some('i') => CaseKind::InsensitiveLowercase,
-                        Some('I') => CaseKind::InsensitiveCapital,
-                        _ => unsafe { std::hint::unreachable_unchecked() },
-                    };
-                    self.pos.next_char();
-                    self.devour_whitespace();
-                    match self.buf.next() {
-                        Some(']') => {
-                            return TokenKind::Attribute(Attribute {
-                                kind,
-                                attr,
-                                value,
-                                case_sensitive,
-                            })
-                        }
-                        Some(_) => todo!("modifier must be 1 character"),
-                        None => todo!("unexpected EOF"),
-                    }
-                }
-                Some(']') => {
-                    return TokenKind::Attribute(Attribute {
-                        kind,
-                        attr,
-                        value,
-                        case_sensitive,
-                    })
-                }
-                Some(c) => {
-                    value.push(' ');
-                    value.push(c.clone());
-                    self.devour_whitespace();
-                    assert!(self.buf.next() == Some(']'));
-                }
-                None => todo!(),
-            }
-        } else {
-            assert!(self.buf.next() == Some(']'));
-        }
-
-        TokenKind::Attribute(Attribute {
-            kind,
-            attr,
-            value,
-            case_sensitive,
-        })
-    }
-
     fn lex_variable(&mut self) -> TokenKind {
         self.buf.next();
         self.pos.next_char();
@@ -444,8 +310,12 @@ impl<'a> Lexer<'a> {
                 name.push(tok);
             }
         }
+        if name.is_empty() {
+            TokenKind::Symbol(Symbol::Dollar)
+        } else {
             TokenKind::Variable(name)
         }
+    }
 
     fn lex_ident(&mut self) -> TokenKind {
         let mut string = String::with_capacity(99);

src/lib.rs

@@ -67,7 +67,7 @@ use crate::function::Function;
 use crate::imports::import;
 use crate::lexer::Lexer;
 use crate::mixin::{eat_include, Mixin};
-use crate::selector::{Attribute, Selector};
+use crate::selector::Selector;
 use crate::style::Style;
 use crate::utils::{devour_whitespace, eat_variable_value, IsComment, IsWhitespace, VariableDecl};
 use crate::value::Value;
@@ -147,7 +147,6 @@ pub(crate) enum TokenKind {
     Number(String),
     Whitespace(Whitespace),
     Variable(String),
-    Attribute(Attribute),
     Op(Op),
     MultilineComment(String),
     Interpolation,
@@ -169,7 +168,6 @@ impl Display for TokenKind {
             TokenKind::AtRule(s) => write!(f, "{}", s),
             TokenKind::Op(s) => write!(f, "{}", s),
             TokenKind::Whitespace(s) => write!(f, "{}", s),
-            TokenKind::Attribute(s) => write!(f, "{}", s),
             TokenKind::Keyword(kw) => write!(f, "{}", kw),
             TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s),
             TokenKind::Variable(s) => write!(f, "{}", s),
@@ -347,8 +345,8 @@ impl<'a> StyleSheetParser<'a> {
         while let Some(Token { kind, .. }) = self.lexer.peek() {
             match kind {
                 TokenKind::Ident(_)
-                | TokenKind::Attribute(_)
                 | TokenKind::Interpolation
+                | TokenKind::Symbol(Symbol::OpenSquareBrace)
                 | TokenKind::Symbol(Symbol::Hash)
                 | TokenKind::Symbol(Symbol::Colon)
                 | TokenKind::Symbol(Symbol::Mul)

src/selector.rs

@@ -1,7 +1,8 @@
 use crate::common::{Scope, Symbol, Whitespace};
 use crate::error::SassResult;
 use crate::utils::{
-    devour_whitespace, devour_whitespace_or_comment, parse_interpolation, IsWhitespace,
+    devour_whitespace, devour_whitespace_or_comment, flatten_ident, parse_interpolation,
+    parse_quoted_string, IsWhitespace,
 };
 use crate::{Token, TokenKind};
 use std::fmt::{self, Display, Write};
@@ -288,7 +289,9 @@ impl<'a> SelectorParser<'a> {
                     )?;
                     self.is_interpolated = false;
                 }
-                TokenKind::Attribute(attr) => self.selectors.push(SelectorKind::Attribute(attr)),
+                TokenKind::Symbol(Symbol::OpenSquareBrace) => self
+                    .selectors
+                    .push(Attribute::from_tokens(tokens, self.scope)?),
                 _ => todo!("unimplemented selector"),
             };
         }
@@ -354,48 +357,145 @@ impl Selector {
 pub(crate) struct Attribute {
     pub attr: String,
     pub value: String,
-    pub case_sensitive: CaseKind,
+    pub modifier: String,
     pub kind: AttributeKind,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+impl Attribute {
-pub(crate) enum CaseKind {
+    pub fn from_tokens(
-    InsensitiveCapital,
+        toks: &mut Peekable<IntoIter<Token>>,
-    InsensitiveLowercase,
+        scope: &Scope,
-    Sensitive,
+    ) -> SassResult<SelectorKind> {
+        devour_whitespace(toks);
+        let attr = if let Some(t) = toks.next() {
+            match t.kind {
+                TokenKind::Ident(mut s) => {
+                    s.push_str(&flatten_ident(toks, scope)?);
+                    s
+                }
+                q @ TokenKind::Symbol(Symbol::DoubleQuote)
+                | q @ TokenKind::Symbol(Symbol::SingleQuote) => {
+                    parse_quoted_string(toks, scope, q)?
+                }
+                _ => return Err("Expected identifier.".into()),
+            }
+        } else {
+            todo!()
+        };
+
+        devour_whitespace(toks);
+
+        let kind = if let Some(t) = toks.next() {
+            match t.kind {
+                TokenKind::Ident(s) if s.len() == 1 => {
+                    devour_whitespace(toks);
+                    match toks.next().unwrap().kind {
+                        TokenKind::Symbol(Symbol::CloseSquareBrace) => {}
+                        _ => return Err("expected \"]\".".into()),
+                    }
+                    return Ok(SelectorKind::Attribute(Attribute {
+                        kind: AttributeKind::Any,
+                        attr,
+                        value: String::new(),
+                        modifier: s,
+                    }));
+                }
+                TokenKind::Symbol(Symbol::CloseSquareBrace) => {
+                    return Ok(SelectorKind::Attribute(Attribute {
+                        kind: AttributeKind::Any,
+                        attr,
+                        value: String::new(),
+                        modifier: String::new(),
+                    }));
+                }
+                TokenKind::Symbol(Symbol::Equal) => AttributeKind::Equals,
+                TokenKind::Symbol(Symbol::Tilde) => AttributeKind::InList,
+                TokenKind::Symbol(Symbol::BitOr) => AttributeKind::BeginsWithHyphenOrExact,
+                TokenKind::Symbol(Symbol::Xor) => AttributeKind::StartsWith,
+                TokenKind::Symbol(Symbol::Dollar) => AttributeKind::EndsWith,
+                TokenKind::Symbol(Symbol::Mul) => AttributeKind::Contains,
+                _ => return Err("Expected \"]\".".into()),
+            }
+        } else {
+            todo!()
+        };
+
+        if kind != AttributeKind::Equals {
+            match toks.next().unwrap().kind {
+                TokenKind::Symbol(Symbol::Equal) => {}
+                _ => return Err("expected \"=\".".into()),
+            }
         }
 
-impl Display for CaseKind {
+        devour_whitespace(toks);
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+
-        match self {
+        let value = if let Some(t) = toks.next() {
-            Self::InsensitiveCapital => write!(f, " I"),
+            match t.kind {
-            Self::InsensitiveLowercase => write!(f, " i"),
+                TokenKind::Ident(mut s) => {
-            Self::Sensitive => write!(f, ""),
+                    s.push_str(&flatten_ident(toks, scope)?);
+                    s
                 }
+                q @ TokenKind::Symbol(Symbol::DoubleQuote)
+                | q @ TokenKind::Symbol(Symbol::SingleQuote) => {
+                    parse_quoted_string(toks, scope, q)?
+                }
+                _ => return Err("Expected identifier.".into()),
+            }
+        } else {
+            todo!()
+        };
+
+        devour_whitespace(toks);
+
+        let modifier = if let Some(t) = toks.next() {
+            match t.kind {
+                TokenKind::Symbol(Symbol::CloseSquareBrace) => {
+                    return Ok(SelectorKind::Attribute(Attribute {
+                        kind,
+                        attr,
+                        value,
+                        modifier: String::new(),
+                    }))
+                }
+                TokenKind::Ident(s) if s.len() == 1 => {
+                    match toks.next().unwrap().kind {
+                        TokenKind::Symbol(Symbol::CloseSquareBrace) => {}
+                        _ => return Err("expected \"]\".".into()),
+                    }
+                    format!(" {}", s)
+                }
+                _ => return Err("Expected \"]\".".into()),
+            }
+        } else {
+            todo!()
+        };
+
+        Ok(SelectorKind::Attribute(Attribute {
+            kind,
+            attr,
+            value,
+            modifier,
+        }))
     }
 }
 
 impl Display for Attribute {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self.kind {
-            AttributeKind::Any => write!(f, "[{}{}]", self.attr, self.case_sensitive),
+            AttributeKind::Any => write!(f, "[{}{}]", self.attr, self.modifier),
-            AttributeKind::Equals => {
+            AttributeKind::Equals => write!(f, "[{}={}{}]", self.attr, self.value, self.modifier),
-                write!(f, "[{}={}{}]", self.attr, self.value, self.case_sensitive)
+            AttributeKind::InList => write!(f, "[{}~={}{}]", self.attr, self.value, self.modifier),
-            }
-            AttributeKind::InList => {
-                write!(f, "[{}~={}{}]", self.attr, self.value, self.case_sensitive)
-            }
             AttributeKind::BeginsWithHyphenOrExact => {
-                write!(f, "[{}|={}{}]", self.attr, self.value, self.case_sensitive)
+                write!(f, "[{}|={}{}]", self.attr, self.value, self.modifier)
             }
             AttributeKind::StartsWith => {
-                write!(f, "[{}^={}{}]", self.attr, self.value, self.case_sensitive)
+                write!(f, "[{}^={}{}]", self.attr, self.value, self.modifier)
             }
             AttributeKind::EndsWith => {
-                write!(f, "[{}$={}{}]", self.attr, self.value, self.case_sensitive)
+                write!(f, "[{}$={}{}]", self.attr, self.value, self.modifier)
             }
             AttributeKind::Contains => {
-                write!(f, "[{}*={}{}]", self.attr, self.value, self.case_sensitive)
+                write!(f, "[{}*={}{}]", self.attr, self.value, self.modifier)
             }
         }
     }

src/utils.rs

@@ -1,4 +1,4 @@
-use crate::common::{Keyword, Symbol};
+use crate::common::{Keyword, QuoteKind, Symbol};
 use crate::error::SassResult;
 use crate::lexer::Lexer;
 use crate::value::Value;
@@ -117,3 +117,78 @@ pub(crate) fn eat_variable_value<I: Iterator<Item = Token>>(
     let val = Value::from_tokens(&mut raw.into_iter().peekable(), scope).unwrap();
     Ok(VariableDecl::new(val, default))
 }
+
+pub(crate) fn flatten_ident<I: Iterator<Item = Token>>(
+    toks: &mut Peekable<I>,
+    scope: &Scope,
+) -> SassResult<String> {
+    let mut s = String::new();
+    while let Some(tok) = toks.peek() {
+        match tok.kind.clone() {
+            TokenKind::Interpolation => {
+                toks.next();
+                s.push_str(
+                    &parse_interpolation(toks, scope)?
+                        .iter()
+                        .map(|x| x.kind.to_string())
+                        .collect::<String>(),
+                )
+            }
+            TokenKind::Ident(ref i) => {
+                toks.next();
+                s.push_str(i)
+            }
+            TokenKind::Number(ref n) => {
+                toks.next();
+                s.push_str(n)
+            }
+            _ => break,
+        }
+    }
+    Ok(s)
+}
+
+pub(crate) fn parse_quoted_string<I: Iterator<Item = Token>>(
+    toks: &mut Peekable<I>,
+    scope: &Scope,
+    q: TokenKind,
+) -> SassResult<String> {
+    let mut s = String::new();
+    let mut is_escaped = false;
+    while let Some(tok) = toks.next() {
+        match tok.kind {
+            TokenKind::Symbol(Symbol::DoubleQuote)
+                if !is_escaped && q == TokenKind::Symbol(Symbol::DoubleQuote) =>
+            {
+                break
+            }
+            TokenKind::Symbol(Symbol::SingleQuote)
+                if !is_escaped && q == TokenKind::Symbol(Symbol::SingleQuote) =>
+            {
+                break
+            }
+            TokenKind::Symbol(Symbol::BackSlash) if !is_escaped => is_escaped = true,
+            TokenKind::Symbol(Symbol::BackSlash) => s.push('\\'),
+            TokenKind::Interpolation => {
+                s.push_str(
+                    &parse_interpolation(toks, scope)?
+                        .iter()
+                        .map(|x| x.kind.to_string())
+                        .collect::<String>(),
+                );
+                continue;
+            }
+            _ => {}
+        }
+        if is_escaped && tok.kind != TokenKind::Symbol(Symbol::BackSlash) {
+            is_escaped = false;
+        }
+        s.push_str(&tok.kind.to_string());
+    }
+    let quotes = match q {
+        TokenKind::Symbol(Symbol::DoubleQuote) => QuoteKind::Double,
+        TokenKind::Symbol(Symbol::SingleQuote) => QuoteKind::Single,
+        _ => unreachable!(),
+    };
+    Ok(format!("{}{}{}", quotes, s, quotes))
+}