Preserve toplevel multiline comments and add tests for them

src/css.rs

@@ -1,29 +1,53 @@
 //! # Convert from SCSS AST to CSS
 use crate::{RuleSet, Selector, Stmt, Style, StyleSheet};
+// use crate::common::AtRule;
+use std::fmt;
 use std::io;
 
 #[derive(Debug, Clone)]
-pub struct Block {
+pub enum Toplevel {
-    selector: Selector,
+    RuleSet(Selector, Vec<BlockEntry>),
-    styles: Vec<Style>,
+    MultilineComment(String),
+    // AtRule(AtRule),
 }
 
-impl Block {
+#[derive(Debug, Clone)]
-    const fn new(selector: Selector) -> Self {
+pub enum BlockEntry {
-        Block {
+    Style(Style),
-            selector,
+    MultilineComment(String),
-            styles: Vec::new(),
+    // AtRule(AtRule),
+}
+
+impl fmt::Display for BlockEntry {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BlockEntry::Style(s) => writeln!(f, "  {}", s),
+            BlockEntry::MultilineComment(s) => writeln!(f, "  /*{}*/", s),
+        }
+    }
+}
+
+impl Toplevel {
+    const fn new_rule(selector: Selector) -> Self {
+        Toplevel::RuleSet(selector, Vec::new())
+    }
+
+    fn push_style(&mut self, s: Style) {
+        if let Toplevel::RuleSet(_, entries) = self {
+            entries.push(BlockEntry::Style(s));
         }
     }
 
-    fn push(&mut self, s: Style) {
+    fn push_comment(&mut self, s: String) {
-        self.styles.push(s);
+        if let Toplevel::RuleSet(_, entries) = self {
+            entries.push(BlockEntry::MultilineComment(s));
+        }
     }
 }
 
 #[derive(Debug, Clone)]
 pub struct Css {
-    blocks: Vec<Block>,
+    blocks: Vec<Toplevel>,
     idx: usize,
 }
 
@@ -36,16 +60,19 @@ impl Css {
     }
 
     pub fn from_stylesheet(s: StyleSheet) -> Self {
-        Css {
+        Css::new().parse_stylesheet(s)
-            blocks: Vec::new(),
-            idx: 0,
-        }
-        .parse_stylesheet(s)
     }
 
     fn parse_stmt(&mut self, stmt: Stmt) {
         match stmt {
-            Stmt::Style(s) => self.blocks[self.idx - 1].push(s),
+            Stmt::Style(s) => self.blocks[self.idx - 1].push_style(s),
+            Stmt::MultilineComment(s) => {
+                if self.idx == 0 {
+                    self.blocks.push(Toplevel::MultilineComment(s));
+                } else {
+                    self.blocks[self.idx - 1].push_comment(s)
+                }
+            }
             Stmt::RuleSet(RuleSet {
                 selector,
                 super_selector,
@@ -53,14 +80,16 @@ impl Css {
             }) => {
                 if self.idx == 0 {
                     self.idx = self.blocks.len() + 1;
-                    self.blocks.push(Block::new(super_selector.zip(selector)));
+                    self.blocks
+                        .push(Toplevel::new_rule(super_selector.zip(selector)));
                     for rule in rules {
                         self.parse_stmt(rule);
                     }
                     self.idx = 0;
                 } else {
                     self.idx += 1;
-                    self.blocks.push(Block::new(super_selector.zip(selector)));
+                    self.blocks
+                        .push(Toplevel::new_rule(super_selector.zip(selector)));
                     for rule in rules {
                         self.parse_stmt(rule);
                     }
@@ -79,15 +108,22 @@ impl Css {
 
     pub fn pretty_print<W: std::io::Write>(self, buf: &mut W) -> io::Result<()> {
         for block in self.blocks {
-            if block.styles.is_empty() {
+            match block {
+                Toplevel::RuleSet(selector, styles) => {
+                    if styles.is_empty() {
                         continue;
                     }
-            writeln!(buf, "{} {{", block.selector)?;
+                    writeln!(buf, "{} {{", selector)?;
-            for style in block.styles {
+                    for style in styles {
-                writeln!(buf, "  {}", style)?;
+                        write!(buf, "{}", style)?;
                     }
                     writeln!(buf, "}}")?;
                 }
+                Toplevel::MultilineComment(s) => {
+                    writeln!(buf, "/*{}*/", s)?;
+                }
+            }
+        }
         Ok(())
     }
 }

src/format.rs

@@ -17,6 +17,7 @@ impl<W: Write> PrettyPrinter<W> {
     fn pretty_print_stmt(&mut self, stmt: &Stmt) -> io::Result<()> {
         let padding = vec![' '; self.scope * 2].iter().collect::<String>();
         match stmt {
+            Stmt::MultilineComment(s) => writeln!(self.buf, "{}/*{}*/", padding, s)?,
             Stmt::RuleSet(RuleSet {
                 selector, rules, ..
             }) => {
@@ -51,6 +52,7 @@ impl<W: Write> PrettyPrinter<W> {
     fn pretty_print_stmt_preserve_super_selectors(&mut self, stmt: &Stmt) -> io::Result<()> {
         let padding = vec![' '; self.scope * 2].iter().collect::<String>();
         match stmt {
+            Stmt::MultilineComment(s) => writeln!(self.buf, "/*{}*/", s)?,
             Stmt::RuleSet(RuleSet {
                 selector,
                 rules,
@@ -244,12 +246,6 @@ mod test_scss {
         "a {\n  height: 1 1px;\n}\n"
     );
 
-    test!(
-        removes_single_line_comment,
-        "// a { color: red }\na {\n  height: 1 1px;\n}\n",
-        "a {\n  height: 1 1px;\n}\n"
-    );
-
     test!(keyword_important, "a {\n  height: 1 !important;\n}\n");
     test!(
         keyword_important_uppercase,
@@ -272,6 +268,38 @@ mod test_scss {
         "a {\n  foo: -webkit-bar-baz;\n}\n"
     );
 
+    test!(
+        removes_inner_comments,
+        "a {\n  color: red/* hi */;\n}\n",
+        "a {\n  color: red;\n}\n"
+    );
+    test!(
+        removes_inner_comments_whitespace,
+        "a {\n  color: red    /* hi */;\n}\n",
+        "a {\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_outer_comments_before,
+        "a {\n  /* hi */\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_outer_comments_after,
+        "a {\n  color: red;\n  /* hi */\n}\n"
+    );
+    test!(
+        preserves_outer_comments_two,
+        "a {\n  /* foo */\n  /* bar */\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_toplevel_comment,
+        "/* foo */\na {\n  color: red;\n}\n"
+    );
+    test!(
+        removes_single_line_comment,
+        "// a { color: red }\na {\n  height: 1 1px;\n}\n",
+        "a {\n  height: 1 1px;\n}\n"
+    );
+
     test!(unit_none, "a {\n  height: 1;\n}\n");
     test!(unit_not_attached, "a {\n  height: 1 px;\n}\n");
     test!(unit_px, "a {\n  height: 1px;\n}\n");

src/lexer.rs

@@ -161,6 +161,8 @@ impl<'a> Lexer<'a> {
                 self.buf.by_ref().take_while(|x| x != &'\n').for_each(drop);
             }
             '*' => {
+                self.buf.next();
+                self.pos.next_char();
                 let mut comment = String::new();
                 while let Some(tok) = self.buf.next() {
                     if tok == '\n' {
@@ -174,7 +176,7 @@ impl<'a> Lexer<'a> {
                     }
                     comment.push(tok);
                 }
-                return TokenKind::MultilineComment(comment)
+                return TokenKind::MultilineComment(comment);
             }
             _ => return TokenKind::Symbol(Symbol::Div),
         }

src/main.rs

@@ -51,6 +51,12 @@ mod units;
 
 type SassResult<T> = Result<T, SassError>;
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct Token {
+    pos: Pos,
+    pub kind: TokenKind,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum TokenKind {
     Ident(String),
@@ -88,12 +94,6 @@ impl Display for TokenKind {
     }
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct Token {
-    pos: Pos,
-    pub kind: TokenKind,
-}
-
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub struct StyleSheet {
     rules: Vec<Stmt>,
@@ -103,7 +103,7 @@ pub struct StyleSheet {
 pub enum Stmt {
     Style(Style),
     RuleSet(RuleSet),
-    // MultilineComment(String),
+    MultilineComment(String),
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -119,6 +119,7 @@ enum Expr {
     Style(Style),
     Selector(Selector),
     VariableDecl(String, Vec<Token>),
+    MultilineComment(String),
 }
 
 impl StyleSheet {
@@ -201,8 +202,8 @@ impl<'a> StyleSheetParser<'a> {
                     self.global_variables.insert(name, val);
                 }
                 TokenKind::MultilineComment(comment) => {
-                    todo!("MultilineComment");
+                    self.lexer.next();
-                    // rules.push(Stmt::MultilineComment(comment));
+                    rules.push(Stmt::MultilineComment(comment));
                 }
                 TokenKind::AtRule(_) => self.eat_at_rule(),
                 _ => todo!("unexpected toplevel token"),
@@ -315,6 +316,7 @@ impl<'a> StyleSheetParser<'a> {
                         vars.insert(name, val);
                     }
                 }
+                Expr::MultilineComment(s) => stmts.push(Stmt::MultilineComment(s)),
             }
         }
         stmts
@@ -351,6 +353,14 @@ impl<'a> StyleSheetParser<'a> {
                         });
                     }
                 }
+                TokenKind::MultilineComment(ref s) => {
+                    self.devour_whitespace();
+                    if values.is_empty() {
+                        return Ok(Expr::MultilineComment(s.clone()));
+                    } else {
+                        values.push(tok.clone())
+                    }
+                }
                 _ => values.push(tok.clone()),
             };
         }
@@ -424,6 +434,42 @@ mod test_css {
     test!(basic_style, "a {\n  color: red;\n}\n");
     test!(two_styles, "a {\n  color: red;\n  color: blue;\n}\n");
     test!(selector_mul, "a, b {\n  color: red;\n}\n");
-    test!(removes_empty_outer_styles, "a {\n  b {\n    color: red;\n  }\n", "a b {\n  color: red;\n}\n");
+    test!(
+        removes_empty_outer_styles,
+        "a {\n  b {\n    color: red;\n  }\n",
+        "a b {\n  color: red;\n}\n"
+    );
     test!(removes_empty_styles, "a {}\n", "");
+
+    test!(
+        removes_inner_comments,
+        "a {\n  color: red/* hi */;\n}\n",
+        "a {\n  color: red;\n}\n"
+    );
+    test!(
+        removes_inner_comments_whitespace,
+        "a {\n  color: red    /* hi */;\n}\n",
+        "a {\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_outer_comments_before,
+        "a {\n  /* hi */\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_outer_comments_after,
+        "a {\n  color: red;\n  /* hi */\n}\n"
+    );
+    test!(
+        preserves_outer_comments_two,
+        "a {\n  /* foo */\n  /* bar */\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_toplevel_comment,
+        "/* foo */\na {\n  color: red;\n}\n"
+    );
+    test!(
+        removes_single_line_comment,
+        "// a { color: red }\na {\n  height: 1 1px;\n}\n",
+        "a {\n  height: 1 1px;\n}\n"
+    );
 }

src/style.rs

@@ -66,22 +66,21 @@ impl<'a> StyleParser<'a> {
         val
     }
 
-    fn devour_whitespace(&mut self) {
+    fn devour_whitespace_or_comment(&mut self) {
-        while let Some(tok) = self.tokens.peek() {
+        while let Some(Token { kind, .. }) = self.tokens.peek() {
-            if let TokenKind::Whitespace(_) = tok.kind {
+            match kind {
-                self.tokens.next();
+                TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => self.tokens.next(),
-            } else {
+                _ => break,
-                break;
+            };
-            }
         }
     }
 
     fn parse(&mut self) -> Style {
         let mut property = String::new();
         // read property
-        while let Some(tok) = self.tokens.next() {
+        while let Some(Token { kind, .. }) = self.tokens.next() {
-            match tok.kind {
+            match kind {
-                TokenKind::Whitespace(_) => continue,
+                TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => continue,
                 TokenKind::Ident(ref s) => {
                     property = s.clone();
                     break;
@@ -91,9 +90,9 @@ impl<'a> StyleParser<'a> {
         }
 
         // read until `:`
-        while let Some(tok) = self.tokens.next() {
+        while let Some(Token { kind, .. }) = self.tokens.next() {
-            match tok.kind {
+            match kind {
-                TokenKind::Whitespace(_) => continue,
+                TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => continue,
                 TokenKind::Symbol(Symbol::Colon) => break,
                 _ => todo!("found tokens before style value"),
             }
@@ -105,23 +104,28 @@ impl<'a> StyleParser<'a> {
         while let Some(tok) = self.tokens.next() {
             match &tok.kind {
                 TokenKind::Whitespace(_) => {
-                    while let Some(w) = self.tokens.peek() {
+                    while let Some(Token { kind, .. }) = self.tokens.peek() {
-                        if let TokenKind::Whitespace(_) = w.kind {
+                        match kind {
+                            TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => {
                                 self.tokens.next();
                                 continue;
-                        } else if let TokenKind::Ident(ref s) = w.kind {
+                            }
+                            TokenKind::Ident(ref s) => {
                                 if s == &String::from("-") {
                                     self.tokens.next();
                                     value.push('-');
-                                self.devour_whitespace();
+                                    self.devour_whitespace_or_comment();
                                     break;
                                 }
                             }
+                            _ => {}
+                        }
                         value.push(' ');
                         break;
                     }
                 }
                 TokenKind::Variable(_) => value.push_str(&self.deref_variable(&tok.kind)),
+                TokenKind::MultilineComment(_) => continue,
                 _ => value.push_str(&tok.kind.to_string()),
             }
         }