emit charset only when output contains utf-8

src/lexer.rs

@@ -1,11 +1,9 @@
 use std::iter::Peekable;
 use std::str::Chars;
-use std::sync::atomic::{AtomicBool, Ordering};
 
 use crate::common::Pos;
 use crate::Token;
 
-pub static IS_UTF8: AtomicBool = AtomicBool::new(false);
 pub const FORM_FEED: char = '\x0C';
 
 #[derive(Debug, Clone)]
@@ -31,10 +29,6 @@ impl<'a> Iterator for Lexer<'a> {
                 }
             }
             '\0' => return None,
-            c if !c.is_ascii() => {
-                IS_UTF8.store(true, Ordering::Relaxed);
-                c
-            }
             c => c,
         };
         self.pos.next_char();

src/lib.rs

@@ -86,11 +86,11 @@ use std::path::Path;
 
 use crate::atrule::{eat_include, AtRule, AtRuleKind, Function, Mixin};
 use crate::common::Pos;
-use crate::output::Css;
 pub use crate::error::{SassError, SassResult};
 use crate::format::PrettyPrinter;
 use crate::imports::import;
 use crate::lexer::Lexer;
+use crate::output::Css;
 use crate::scope::{insert_global_fn, insert_global_mixin, insert_global_var, Scope, GLOBAL_SCOPE};
 use crate::selector::Selector;
 use crate::style::Style;
@@ -106,11 +106,11 @@ mod atrule;
 mod builtin;
 mod color;
 mod common;
-mod output;
 mod error;
 mod format;
 mod imports;
 mod lexer;
+mod output;
 mod scope;
 mod selector;
 mod style;
@@ -264,7 +264,7 @@ impl StyleSheet {
     /// ```
     #[inline]
     pub fn print_as_css<W: Write>(self, buf: &mut W) -> SassResult<()> {
-        Css::from_stylesheet(self)?.pretty_print(buf, 0)
+        Css::from_stylesheet(self)?.pretty_print(buf)
     }
 }
 

src/output.rs

@@ -1,11 +1,9 @@
 //! # Convert from SCSS AST to CSS
 use std::fmt;
 use std::io::Write;
-use std::sync::atomic::Ordering;
 
 use crate::atrule::AtRule;
 use crate::error::SassResult;
-use crate::lexer::IS_UTF8;
 use crate::{RuleSet, Selector, Stmt, Style, StyleSheet};
 
 #[derive(Debug, Clone)]
@@ -121,12 +119,19 @@ impl Css {
         Ok(self)
     }
 
-    pub fn pretty_print<W: Write>(self, buf: &mut W, nesting: usize) -> SassResult<()> {
+    pub fn pretty_print<W: Write>(self, buf: &mut W) -> SassResult<()> {
-        let mut has_written = false;
+        let mut string = Vec::new();
-        let padding = vec![' '; nesting * 2].iter().collect::<String>();
+        self._inner_pretty_print(&mut string, 0)?;
-        if IS_UTF8.swap(false, Ordering::Relaxed) {
+        if string.iter().any(|s| !s.is_ascii()) {
             writeln!(buf, "@charset \"UTF-8\";")?;
         }
+        write!(buf, "{}", String::from_utf8(string).unwrap())?;
+        Ok(())
+    }
+
+    fn _inner_pretty_print(self, buf: &mut Vec<u8>, nesting: usize) -> SassResult<()> {
+        let mut has_written = false;
+        let padding = vec![' '; nesting * 2].iter().collect::<String>();
         for block in self.blocks {
             match block {
                 Toplevel::RuleSet(selector, styles) => {
@@ -152,7 +157,7 @@ impl Css {
                             writeln!(buf, "{}@{} {} {{", padding, u.name, u.params)?;
                         }
                         Css::from_stylesheet(StyleSheet::from_stmts(u.body))?
-                            .pretty_print(buf, nesting + 1)?;
+                            ._inner_pretty_print(buf, nesting + 1)?;
                         writeln!(buf, "{}}}", padding)?;
                     }
                     _ => todo!("at-rule other than unknown at toplevel"),

src/utils.rs

@@ -603,6 +603,8 @@ pub(crate) fn parse_quoted_string<I: Iterator<Item = Token>>(
                 let c = std::char::from_u32(u32::from_str_radix(&n, 16).unwrap()).unwrap();
                 if c.is_control() && c != '\t' && c != '\0' {
                     s.push_str(&format!("\\{}", n.to_ascii_lowercase()));
+                } else if c == '\0' {
+                    s.push('\u{FFFD}');
                 } else {
                     s.push(c);
                 }

tests/charset.rs

@@ -4,13 +4,11 @@
 mod macros;
 
 test!(
-    #[ignore]
     utf8_input,
     "a {\n  color: 🦆;\n}\n",
     "@charset \"UTF-8\";\na {\n  color: 🦆;\n}\n"
 );
 test!(
-    #[ignore]
     ascii_charset_utf8,
     "@charset \"UTF-8\";\na {\n  color: red;\n}\n",
     "a {\n  color: red;\n}\n"

tests/misc.rs

@@ -47,25 +47,21 @@ test!(
     "a {\n  color: red;\n}\n"
 );
 test!(
-    #[ignore]
     utf8_ident_before_len,
     "a {\n  color: length(😀red);\n}\n",
-    "@charset \"UTF-8\";\na {\n  color: 1;\n}\n"
+    "a {\n  color: 1;\n}\n"
 );
 test!(
-    #[ignore]
     utf8_ident_before,
     "a {\n  color: 😀red;\n}\n",
     "@charset \"UTF-8\";\na {\n  color: 😀red;\n}\n"
 );
 test!(
-    #[ignore]
     utf8_ident_after_len,
     "a {\n  color: length(red😁)\n}\n",
-    "@charset \"UTF-8\";\na {\n  color: 1;\n}\n"
+    "a {\n  color: 1;\n}\n"
 );
 test!(
-    #[ignore]
     utf8_ident_after,
     "a {\n  color: red😁\n}\n",
     "@charset \"UTF-8\";\na {\n  color: red😁;\n}\n"

tests/str-escape.rs

@@ -98,8 +98,8 @@ test!(
 );
 test!(
     single_character_escape_sequence_has_space_after,
-    "a {\n  color: \\0;\n}\n",
+    "a {\n  color: \\a;\n}\n",
-    "a {\n  color: \\0 ;\n}\n"
+    "a {\n  color: \\a ;\n}\n"
 );
 test!(
     escapes_non_hex_in_string,
@@ -127,6 +127,16 @@ test!(
     "a {\n  color: foo == f\\6F\\6F;\n}\n",
     "a {\n  color: true;\n}\n"
 );
+test!(
+    quoted_escape_zero,
+    "a {\n  color: \"\\0\";\n}\n",
+    "@charset \"UTF-8\";\na {\n  color: \"<EFBFBD>\";\n}\n"
+);
+test!(
+    unquoted_escape_zero,
+    "a {\n  color: \\0;\n}\n",
+    "a {\n  color: \\0 ;\n}\n"
+);
 // test!(
 //     quote_escape,
 //     "a {\n  color: quote(\\b);\n}\n",

tests/strings.rs

@@ -119,10 +119,9 @@ test!(
     "a {\n  color: 7;\n}\n"
 );
 test!(
-    #[ignore]
     str_len_double_wide,
     "a {\n  color: str-length(\"👭\");\n}\n",
-    "@charset \"UTF-8\";\na {\n  color: 1;\n}\n"
+    "a {\n  color: 1;\n}\n"
 );
 test!(
     str_len_combining,
@@ -215,7 +214,6 @@ test!(
     "a {\n  color: Xabcd;\n}\n"
 );
 test!(
-    #[ignore]
     str_insert_double_width_char,
     "a {\n  color: str-insert(\"👭\", \"c\", 2);\n}\n",
     "@charset \"UTF-8\";\na {\n  color: \"👭c\";\n}\n"

tests/variables.rs

@@ -98,15 +98,11 @@ test!(
     "a {\n  $a: red\n}\n\nb {\n  color: blue;\n}\n",
     "b {\n  color: blue;\n}\n"
 );
-// TODO: blocked on properly emitting @charset
+test!(
-// right now, we emit @charset if a utf-8 character
+    unicode_in_variables,
-// is found *anywhere*, but ideally we would only emit
+    "$vär: foo;\na {\n  color: $vär;\n}\n",
-// it if a utf-8 character is actually in the output
+    "a {\n  color: foo;\n}\n"
-// test!(
+);
-//     unicode_in_variables,
-//     "$vär: foo;\na {\n  color: $vär;\n}\n",
-//     "a {\n  color: foo;\n}\n"
-// );
 test!(
     variable_does_not_include_interpolation,
     "$input: foo;\na {\n  color: $input#{\"literal\"};\n}\n",