diff --git a/src/lexer.rs b/src/lexer.rs index 3fc0120..d9ec049 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -134,7 +134,10 @@ impl<'a> Iterator for Lexer<'a> { '<' => symbol!(self, Lt), '>' => symbol!(self, Gt), '\0' => return None, - _ => todo!("unknown char"), + &v => { + self.buf.next(); + TokenKind::Unknown(v.clone()) + } }; self.pos.next_char(); Some(Token { diff --git a/src/lib.rs b/src/lib.rs index 2d276f2..b2f14d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -151,6 +151,7 @@ pub(crate) enum TokenKind { Op(Op), MultilineComment(String), Interpolation, + Unknown(char), } impl TokenKind { @@ -172,6 +173,7 @@ impl Display for TokenKind { TokenKind::Keyword(kw) => write!(f, "{}", kw), TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s), TokenKind::Variable(s) => write!(f, "{}", s), + TokenKind::Unknown(s) => write!(f, "{}", s), TokenKind::Interpolation => { panic!("we don't want to format TokenKind::Interpolation using Display") } diff --git a/src/value/parse.rs b/src/value/parse.rs index 35eaa4a..21f8648 100644 --- a/src/value/parse.rs +++ b/src/value/parse.rs @@ -343,6 +343,7 @@ impl Value { TokenKind::Keyword(Keyword::True) => Ok(Value::True), TokenKind::Keyword(Keyword::False) => Ok(Value::False), TokenKind::Keyword(Keyword::Null) => Ok(Value::Null), + TokenKind::Unknown(c) => Ok(Value::Ident(c.to_string(), QuoteKind::None)), _ => Err("Unexpected token in value parsing".into()), } } diff --git a/tests/misc.rs b/tests/misc.rs index ccfca53..21ee1d7 100644 --- a/tests/misc.rs +++ b/tests/misc.rs @@ -27,3 +27,4 @@ test!( "$a-b: red; $a_b: green; a {\n color: $a-b;\n}\n", "a {\n color: green;\n}\n" ); +test!(utf8_input, "a {\n color: 🦆;\n}\n");