Properly emit @charset

This commit is contained in:
ConnorSkees 2020-02-28 18:27:32 -05:00
parent 1dfe77bcff
commit bc2c927aa7
7 changed files with 38 additions and 23 deletions

View File

@ -18,8 +18,7 @@ pub(crate) enum AtRule {
Mixin(String, Box<Mixin>), Mixin(String, Box<Mixin>),
Function(String, Box<Function>), Function(String, Box<Function>),
Return(Vec<Token>), Return(Vec<Token>),
// todo: emit only when non-ascii char is found Charset,
Charset(Vec<Token>),
Unknown(UnknownAtRule), Unknown(UnknownAtRule),
} }
@ -94,10 +93,10 @@ impl AtRule {
AtRuleKind::Use => todo!("@use not yet implemented"), AtRuleKind::Use => todo!("@use not yet implemented"),
AtRuleKind::Annotation => todo!("@annotation not yet implemented"), AtRuleKind::Annotation => todo!("@annotation not yet implemented"),
AtRuleKind::AtRoot => todo!("@at-root not yet implemented"), AtRuleKind::AtRoot => todo!("@at-root not yet implemented"),
AtRuleKind::Charset => AtRule::Charset( AtRuleKind::Charset => {
toks.take_while(|t| t.kind != TokenKind::Symbol(Symbol::SemiColon)) toks.take_while(|t| t.kind != TokenKind::Symbol(Symbol::SemiColon)).for_each(drop);
.collect(), AtRule::Charset
), },
AtRuleKind::Each => todo!("@each not yet implemented"), AtRuleKind::Each => todo!("@each not yet implemented"),
AtRuleKind::Extend => todo!("@extend not yet implemented"), AtRuleKind::Extend => todo!("@extend not yet implemented"),
AtRuleKind::If => todo!("@if not yet implemented"), AtRuleKind::If => todo!("@if not yet implemented"),

View File

@ -2,6 +2,8 @@
use crate::atrule::AtRule; use crate::atrule::AtRule;
use crate::error::SassResult; use crate::error::SassResult;
use crate::{RuleSet, Selector, Stmt, Style, StyleSheet}; use crate::{RuleSet, Selector, Stmt, Style, StyleSheet};
use crate::lexer::IS_UTF8;
use std::sync::atomic::Ordering;
use std::fmt; use std::fmt;
use std::io::Write; use std::io::Write;
@ -115,6 +117,9 @@ impl Css {
pub fn pretty_print<W: Write>(self, buf: &mut W, nesting: usize) -> SassResult<()> { pub fn pretty_print<W: Write>(self, buf: &mut W, nesting: usize) -> SassResult<()> {
let mut has_written = false; let mut has_written = false;
let padding = vec![' '; nesting * 2].iter().collect::<String>(); let padding = vec![' '; nesting * 2].iter().collect::<String>();
if IS_UTF8.swap(false, Ordering::Relaxed) {
writeln!(buf, "@charset \"UTF-8\";")?;
}
for block in self.blocks { for block in self.blocks {
match block { match block {
Toplevel::RuleSet(selector, styles) => { Toplevel::RuleSet(selector, styles) => {
@ -144,11 +149,6 @@ impl Css {
.unwrap(); .unwrap();
writeln!(buf, "{}}}", padding)?; writeln!(buf, "{}}}", padding)?;
} }
AtRule::Charset(toks) => write!(
buf,
"@charset {};",
toks.iter().map(|x| x.kind.to_string()).collect::<String>()
)?,
_ => todo!(), _ => todo!(),
}, },
Toplevel::Newline => { Toplevel::Newline => {

View File

@ -36,11 +36,6 @@ impl<W: Write> PrettyPrinter<W> {
} }
Stmt::AtRule(r) => match r { Stmt::AtRule(r) => match r {
AtRule::Unknown(..) => todo!("Display @rules properly"), AtRule::Unknown(..) => todo!("Display @rules properly"),
AtRule::Charset(toks) => write!(
self.buf,
"@charset {};",
toks.iter().map(|x| x.kind.to_string()).collect::<String>()
)?,
_ => todo!(), _ => todo!(),
}, },
} }

View File

@ -1,6 +1,7 @@
use std::convert::TryFrom; use std::convert::TryFrom;
use std::iter::Peekable; use std::iter::Peekable;
use std::str::Chars; use std::str::Chars;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::atrule::AtRuleKind; use crate::atrule::AtRuleKind;
use crate::common::{Keyword, Op, Pos, Symbol}; use crate::common::{Keyword, Op, Pos, Symbol};
@ -9,6 +10,8 @@ use crate::{Token, TokenKind, Whitespace};
// Rust does not allow us to escape '\f' // Rust does not allow us to escape '\f'
const FORM_FEED: char = '\x0C'; const FORM_FEED: char = '\x0C';
pub static IS_UTF8: AtomicBool = AtomicBool::new(false);
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub(crate) struct Lexer<'a> { pub(crate) struct Lexer<'a> {
tokens: Vec<Token>, tokens: Vec<Token>,
@ -133,6 +136,9 @@ impl<'a> Iterator for Lexer<'a> {
'\0' => return None, '\0' => return None,
&v => { &v => {
self.buf.next(); self.buf.next();
if !v.is_ascii() {
IS_UTF8.store(true, Ordering::Relaxed);
}
TokenKind::Unknown(v.clone()) TokenKind::Unknown(v.clone())
} }
}; };

View File

@ -245,8 +245,6 @@ enum Expr {
Style(Box<Style>), Style(Box<Style>),
/// Several styles /// Several styles
Styles(Vec<Style>), Styles(Vec<Style>),
/// A collection of styles, from a mixin or function
// Styles(Vec<Style>),
/// A full selector `a > h1` /// A full selector `a > h1`
Selector(Selector), Selector(Selector),
/// A variable declaration `$var: 1px` /// A variable declaration `$var: 1px`
@ -469,9 +467,7 @@ impl<'a> StyleSheetParser<'a> {
AtRule::Function(name, func) => { AtRule::Function(name, func) => {
self.global_scope.insert_fn(&name, *func); self.global_scope.insert_fn(&name, *func);
} }
AtRule::Charset(toks) => { AtRule::Charset => continue,
rules.push(Stmt::AtRule(AtRule::Charset(toks)))
}
AtRule::Error(pos, message) => self.error(pos, &message), AtRule::Error(pos, message) => self.error(pos, &message),
AtRule::Warn(pos, message) => self.warn(pos, &message), AtRule::Warn(pos, message) => self.warn(pos, &message),
AtRule::Debug(pos, message) => self.debug(pos, &message), AtRule::Debug(pos, message) => self.debug(pos, &message),
@ -651,7 +647,7 @@ pub(crate) fn eat_expr<I: Iterator<Item = Token>>(
return match AtRule::from_tokens(rule, pos, toks, scope, super_selector)? { return match AtRule::from_tokens(rule, pos, toks, scope, super_selector)? {
AtRule::Mixin(name, mixin) => Ok(Some(Expr::MixinDecl(name, mixin))), AtRule::Mixin(name, mixin) => Ok(Some(Expr::MixinDecl(name, mixin))),
AtRule::Function(name, func) => Ok(Some(Expr::FunctionDecl(name, func))), AtRule::Function(name, func) => Ok(Some(Expr::FunctionDecl(name, func))),
AtRule::Charset(_) => todo!("@charset as expr"), AtRule::Charset => todo!("@charset as expr"),
AtRule::Debug(a, b) => Ok(Some(Expr::Debug(a, b))), AtRule::Debug(a, b) => Ok(Some(Expr::Debug(a, b))),
AtRule::Warn(a, b) => Ok(Some(Expr::Warn(a, b))), AtRule::Warn(a, b) => Ok(Some(Expr::Warn(a, b))),
AtRule::Error(pos, err) => Err(SassError::new(err, pos)), AtRule::Error(pos, err) => Err(SassError::new(err, pos)),

20
tests/charset.rs Normal file
View File

@ -0,0 +1,20 @@
#![cfg(test)]
#[macro_use]
mod macros;
test!(
utf8_input,
"a {\n color: 🦆;\n}\n",
"@charset \"UTF-8\";\na {\n color: 🦆;\n}\n"
);
test!(
ascii_charset_utf8,
"@charset \"UTF-8\";\na {\n color: red;\n}\n",
"a {\n color: red;\n}\n"
);
test!(
unknown_charset,
"@charset \"foo\";\na {\n color: red;\n}\n",
"a {\n color: red;\n}\n"
);

View File

@ -31,7 +31,6 @@ test!(
"$a-b: red; $a_b: green; a {\n color: $a-b;\n}\n", "$a-b: red; $a_b: green; a {\n color: $a-b;\n}\n",
"a {\n color: green;\n}\n" "a {\n color: green;\n}\n"
); );
test!(utf8_input, "a {\n color: 🦆;\n}\n");
// test!( // test!(
// ends_with_several_semicolons, // ends_with_several_semicolons,
// "a {\n color: red;;\n}\n", // "a {\n color: red;;\n}\n",