Properly emit @charset

src/atrule.rs

@@ -18,8 +18,7 @@ pub(crate) enum AtRule {
     Mixin(String, Box<Mixin>),
     Function(String, Box<Function>),
     Return(Vec<Token>),
-    // todo: emit only when non-ascii char is found
+    Charset,
-    Charset(Vec<Token>),
     Unknown(UnknownAtRule),
 }
 
@@ -94,10 +93,10 @@ impl AtRule {
             AtRuleKind::Use => todo!("@use not yet implemented"),
             AtRuleKind::Annotation => todo!("@annotation not yet implemented"),
             AtRuleKind::AtRoot => todo!("@at-root not yet implemented"),
-            AtRuleKind::Charset => AtRule::Charset(
+            AtRuleKind::Charset => {
-                toks.take_while(|t| t.kind != TokenKind::Symbol(Symbol::SemiColon))
+                toks.take_while(|t| t.kind != TokenKind::Symbol(Symbol::SemiColon)).for_each(drop);
-                    .collect(),
+                AtRule::Charset
-            ),
+            },
             AtRuleKind::Each => todo!("@each not yet implemented"),
             AtRuleKind::Extend => todo!("@extend not yet implemented"),
             AtRuleKind::If => todo!("@if not yet implemented"),

src/css.rs

@@ -2,6 +2,8 @@
 use crate::atrule::AtRule;
 use crate::error::SassResult;
 use crate::{RuleSet, Selector, Stmt, Style, StyleSheet};
+use crate::lexer::IS_UTF8;
+use std::sync::atomic::Ordering;
 use std::fmt;
 use std::io::Write;
 
@@ -115,6 +117,9 @@ impl Css {
     pub fn pretty_print<W: Write>(self, buf: &mut W, nesting: usize) -> SassResult<()> {
         let mut has_written = false;
         let padding = vec![' '; nesting * 2].iter().collect::<String>();
+        if IS_UTF8.swap(false, Ordering::Relaxed) {
+            writeln!(buf, "@charset \"UTF-8\";")?;
+        }
         for block in self.blocks {
             match block {
                 Toplevel::RuleSet(selector, styles) => {
@@ -144,11 +149,6 @@ impl Css {
                             .unwrap();
                         writeln!(buf, "{}}}", padding)?;
                     }
-                    AtRule::Charset(toks) => write!(
-                        buf,
-                        "@charset {};",
-                        toks.iter().map(|x| x.kind.to_string()).collect::<String>()
-                    )?,
                     _ => todo!(),
                 },
                 Toplevel::Newline => {

src/format.rs

@@ -36,11 +36,6 @@ impl<W: Write> PrettyPrinter<W> {
             }
             Stmt::AtRule(r) => match r {
                 AtRule::Unknown(..) => todo!("Display @rules properly"),
-                AtRule::Charset(toks) => write!(
-                    self.buf,
-                    "@charset {};",
-                    toks.iter().map(|x| x.kind.to_string()).collect::<String>()
-                )?,
                 _ => todo!(),
             },
         }

src/lexer.rs

@@ -1,6 +1,7 @@
 use std::convert::TryFrom;
 use std::iter::Peekable;
 use std::str::Chars;
+use std::sync::atomic::{AtomicBool, Ordering};
 
 use crate::atrule::AtRuleKind;
 use crate::common::{Keyword, Op, Pos, Symbol};
@@ -9,6 +10,8 @@ use crate::{Token, TokenKind, Whitespace};
 // Rust does not allow us to escape '\f'
 const FORM_FEED: char = '\x0C';
 
+pub static IS_UTF8: AtomicBool = AtomicBool::new(false);
+
 #[derive(Debug, Clone)]
 pub(crate) struct Lexer<'a> {
     tokens: Vec<Token>,
@@ -133,6 +136,9 @@ impl<'a> Iterator for Lexer<'a> {
             '\0' => return None,
             &v => {
                 self.buf.next();
+                if !v.is_ascii() {
+                    IS_UTF8.store(true, Ordering::Relaxed);
+                }
                 TokenKind::Unknown(v.clone())
             }
         };

src/lib.rs

@@ -245,8 +245,6 @@ enum Expr {
     Style(Box<Style>),
     /// Several styles
     Styles(Vec<Style>),
-    /// A collection of styles, from a mixin or function
-    // Styles(Vec<Style>),
     /// A full selector `a > h1`
     Selector(Selector),
     /// A variable declaration `$var: 1px`
@@ -469,9 +467,7 @@ impl<'a> StyleSheetParser<'a> {
                             AtRule::Function(name, func) => {
                                 self.global_scope.insert_fn(&name, *func);
                             }
-                            AtRule::Charset(toks) => {
+                            AtRule::Charset => continue,
-                                rules.push(Stmt::AtRule(AtRule::Charset(toks)))
-                            }
                             AtRule::Error(pos, message) => self.error(pos, &message),
                             AtRule::Warn(pos, message) => self.warn(pos, &message),
                             AtRule::Debug(pos, message) => self.debug(pos, &message),
@@ -651,7 +647,7 @@ pub(crate) fn eat_expr<I: Iterator<Item = Token>>(
                     return match AtRule::from_tokens(rule, pos, toks, scope, super_selector)? {
                         AtRule::Mixin(name, mixin) => Ok(Some(Expr::MixinDecl(name, mixin))),
                         AtRule::Function(name, func) => Ok(Some(Expr::FunctionDecl(name, func))),
-                        AtRule::Charset(_) => todo!("@charset as expr"),
+                        AtRule::Charset => todo!("@charset as expr"),
                         AtRule::Debug(a, b) => Ok(Some(Expr::Debug(a, b))),
                         AtRule::Warn(a, b) => Ok(Some(Expr::Warn(a, b))),
                         AtRule::Error(pos, err) => Err(SassError::new(err, pos)),

tests/charset.rs

@@ -0,0 +1,20 @@
+#![cfg(test)]
+
+#[macro_use]
+mod macros;
+
+test!(
+    utf8_input,
+    "a {\n  color: 🦆;\n}\n",
+    "@charset \"UTF-8\";\na {\n  color: 🦆;\n}\n"
+);
+test!(
+    ascii_charset_utf8,
+    "@charset \"UTF-8\";\na {\n  color: red;\n}\n",
+    "a {\n  color: red;\n}\n"
+);
+test!(
+    unknown_charset,
+    "@charset \"foo\";\na {\n  color: red;\n}\n",
+    "a {\n  color: red;\n}\n"
+);

tests/misc.rs

@@ -31,7 +31,6 @@ test!(
     "$a-b: red; $a_b: green; a {\n  color: $a-b;\n}\n",
     "a {\n  color: green;\n}\n"
 );
-test!(utf8_input, "a {\n  color: 🦆;\n}\n");
 // test!(
 //     ends_with_several_semicolons,
 //     "a {\n  color: red;;\n}\n",