From 4c3f5e24eeddfb0baa7c2f0056065625d39baa43 Mon Sep 17 00:00:00 2001
From: ConnorSkees <39542938+ConnorSkees@users.noreply.github.com>
Date: Sun, 23 Feb 2020 07:52:14 -0500
Subject: [PATCH] Handle utf8 input (a bit)

---
 src/lexer.rs       | 5 ++++-
 src/lib.rs         | 2 ++
 src/value/parse.rs | 1 +
 tests/misc.rs      | 1 +
 4 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/lexer.rs b/src/lexer.rs
index 3fc0120..d9ec049 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -134,7 +134,10 @@ impl<'a> Iterator for Lexer<'a> {
             '<' => symbol!(self, Lt),
             '>' => symbol!(self, Gt),
             '\0' => return None,
-            _ => todo!("unknown char"),
+            &v => {
+                self.buf.next();
+                TokenKind::Unknown(v.clone())
+            }
         };
         self.pos.next_char();
         Some(Token {
diff --git a/src/lib.rs b/src/lib.rs
index 2d276f2..b2f14d2 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -151,6 +151,7 @@ pub(crate) enum TokenKind {
     Op(Op),
     MultilineComment(String),
     Interpolation,
+    Unknown(char),
 }
 
 impl TokenKind {
@@ -172,6 +173,7 @@ impl Display for TokenKind {
             TokenKind::Keyword(kw) => write!(f, "{}", kw),
             TokenKind::MultilineComment(s) => write!(f, "/*{}*/", s),
             TokenKind::Variable(s) => write!(f, "{}", s),
+            TokenKind::Unknown(s) => write!(f, "{}", s),
             TokenKind::Interpolation => {
                 panic!("we don't want to format TokenKind::Interpolation using Display")
             }
diff --git a/src/value/parse.rs b/src/value/parse.rs
index 35eaa4a..21f8648 100644
--- a/src/value/parse.rs
+++ b/src/value/parse.rs
@@ -343,6 +343,7 @@ impl Value {
             TokenKind::Keyword(Keyword::True) => Ok(Value::True),
             TokenKind::Keyword(Keyword::False) => Ok(Value::False),
             TokenKind::Keyword(Keyword::Null) => Ok(Value::Null),
+            TokenKind::Unknown(c) => Ok(Value::Ident(c.to_string(), QuoteKind::None)),
             _ => Err("Unexpected token in value parsing".into()),
         }
     }
diff --git a/tests/misc.rs b/tests/misc.rs
index ccfca53..21ee1d7 100644
--- a/tests/misc.rs
+++ b/tests/misc.rs
@@ -27,3 +27,4 @@ test!(
     "$a-b: red; $a_b: green; a {\n  color: $a-b;\n}\n",
     "a {\n  color: green;\n}\n"
 );
+test!(utf8_input, "a {\n  color: 🦆;\n}\n");