Allow utf-8 characters in idents

src/lexer.rs

@@ -134,13 +134,7 @@ impl<'a> Iterator for Lexer<'a> {
             '>' => symbol!(self, Gt),
             '^' => symbol!(self, Xor),
             '\0' => return None,
-            &v => {
-                self.buf.next();
-                if !v.is_ascii() {
-                    IS_UTF8.store(true, Ordering::Relaxed);
-                }
-                TokenKind::Unknown(v)
-            }
+            _ => self.lex_ident(),
         };
         self.pos.next_char();
         Some(Token {
@@ -313,9 +307,12 @@ impl<'a> Lexer<'a> {
         let mut string = String::with_capacity(99);
         while let Some(c) = self.buf.peek() {
             // we know that the first char is alphabetic from peeking
-            if !c.is_alphanumeric() && c != &'-' && c != &'_' {
+            if !c.is_alphanumeric() && c != &'-' && c != &'_' && c.is_ascii() {
                 break;
             }
+            if !c.is_ascii() {
+                IS_UTF8.store(true, Ordering::Relaxed);
+            }
             let tok = self
                 .buf
                 .next()

src/token.rs

@@ -82,7 +82,6 @@ pub(crate) enum TokenKind {
     Op(Op),
     MultilineComment(String),
     Interpolation,
-    Unknown(char),
 }
 
 impl TokenKind {
@@ -103,7 +102,6 @@ impl fmt::Display for TokenKind {
             TokenKind::Keyword(kw) => write!(f, "{}", kw),
             TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s),
             TokenKind::Variable(s) => write!(f, "{}", s),
-            TokenKind::Unknown(s) => write!(f, "{}", s),
             TokenKind::Interpolation => {
                 panic!("we don't want to format TokenKind::Interpolation using Display")
             }

src/value/parse.rs

@@ -292,7 +292,6 @@ impl Value {
             TokenKind::Keyword(Keyword::From(s)) => Ok(Value::Ident(s, QuoteKind::None)),
             TokenKind::Keyword(Keyword::Through(s)) => Ok(Value::Ident(s, QuoteKind::None)),
             TokenKind::Keyword(Keyword::To(s)) => Ok(Value::Ident(s, QuoteKind::None)),
-            TokenKind::Unknown(c) => Ok(Value::Ident(c.to_string(), QuoteKind::None)),
             TokenKind::AtRule(_) => return Err("expected \";\".".into()),
             v => {
                 dbg!(v);

tests/misc.rs

@@ -46,3 +46,13 @@ test!(
     "a {\n  color: red; ;\n}\n",
     "a {\n  color: red;\n}\n"
 );
+test!(
+    utf8_ident_before,
+    "a {\n  color: length(😀red);\n}\n",
+    "@charset \"UTF-8\";\na {\n  color: 1;\n}\n"
+);
+test!(
+    utf8_ident_after,
+    "a {\n  color: length(red😁)\n}\n",
+    "@charset \"UTF-8\";\na {\n  color: 1;\n}\n"
+);