From a3ecfbbf3e6453d1552628b4ffbd35038de86d98 Mon Sep 17 00:00:00 2001
From: ConnorSkees <39542938+ConnorSkees@users.noreply.github.com>
Date: Wed, 8 Jan 2020 20:39:05 -0500
Subject: [PATCH] Preserve toplevel multiline comments and add tests for them

---
 src/css.rs    | 88 ++++++++++++++++++++++++++++++++++++---------------
 src/format.rs | 40 +++++++++++++++++++----
 src/lexer.rs  |  4 ++-
 src/main.rs   | 66 ++++++++++++++++++++++++++++++++------
 src/style.rs  | 48 +++++++++++++++-------------
 5 files changed, 181 insertions(+), 65 deletions(-)

diff --git a/src/css.rs b/src/css.rs
index fce4cef..daf236e 100644
--- a/src/css.rs
+++ b/src/css.rs
@@ -1,29 +1,53 @@
 //! # Convert from SCSS AST to CSS
 use crate::{RuleSet, Selector, Stmt, Style, StyleSheet};
+// use crate::common::AtRule;
+use std::fmt;
 use std::io;
 
 #[derive(Debug, Clone)]
-pub struct Block {
-    selector: Selector,
-    styles: Vec<Style>,
+pub enum Toplevel {
+    RuleSet(Selector, Vec<BlockEntry>),
+    MultilineComment(String),
+    // AtRule(AtRule),
 }
 
-impl Block {
-    const fn new(selector: Selector) -> Self {
-        Block {
-            selector,
-            styles: Vec::new(),
+#[derive(Debug, Clone)]
+pub enum BlockEntry {
+    Style(Style),
+    MultilineComment(String),
+    // AtRule(AtRule),
+}
+
+impl fmt::Display for BlockEntry {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BlockEntry::Style(s) => writeln!(f, "  {}", s),
+            BlockEntry::MultilineComment(s) => writeln!(f, "  /*{}*/", s),
+        }
+    }
+}
+
+impl Toplevel {
+    const fn new_rule(selector: Selector) -> Self {
+        Toplevel::RuleSet(selector, Vec::new())
+    }
+
+    fn push_style(&mut self, s: Style) {
+        if let Toplevel::RuleSet(_, entries) = self {
+            entries.push(BlockEntry::Style(s));
         }
     }
 
-    fn push(&mut self, s: Style) {
-        self.styles.push(s);
+    fn push_comment(&mut self, s: String) {
+        if let Toplevel::RuleSet(_, entries) = self {
+            entries.push(BlockEntry::MultilineComment(s));
+        }
     }
 }
 
 #[derive(Debug, Clone)]
 pub struct Css {
-    blocks: Vec<Block>,
+    blocks: Vec<Toplevel>,
     idx: usize,
 }
 
@@ -36,16 +60,19 @@ impl Css {
     }
 
     pub fn from_stylesheet(s: StyleSheet) -> Self {
-        Css {
-            blocks: Vec::new(),
-            idx: 0,
-        }
-        .parse_stylesheet(s)
+        Css::new().parse_stylesheet(s)
     }
 
     fn parse_stmt(&mut self, stmt: Stmt) {
         match stmt {
-            Stmt::Style(s) => self.blocks[self.idx - 1].push(s),
+            Stmt::Style(s) => self.blocks[self.idx - 1].push_style(s),
+            Stmt::MultilineComment(s) => {
+                if self.idx == 0 {
+                    self.blocks.push(Toplevel::MultilineComment(s));
+                } else {
+                    self.blocks[self.idx - 1].push_comment(s)
+                }
+            }
             Stmt::RuleSet(RuleSet {
                 selector,
                 super_selector,
@@ -53,14 +80,16 @@ impl Css {
             }) => {
                 if self.idx == 0 {
                     self.idx = self.blocks.len() + 1;
-                    self.blocks.push(Block::new(super_selector.zip(selector)));
+                    self.blocks
+                        .push(Toplevel::new_rule(super_selector.zip(selector)));
                     for rule in rules {
                         self.parse_stmt(rule);
                     }
                     self.idx = 0;
                 } else {
                     self.idx += 1;
-                    self.blocks.push(Block::new(super_selector.zip(selector)));
+                    self.blocks
+                        .push(Toplevel::new_rule(super_selector.zip(selector)));
                     for rule in rules {
                         self.parse_stmt(rule);
                     }
@@ -79,14 +108,21 @@ impl Css {
 
     pub fn pretty_print<W: std::io::Write>(self, buf: &mut W) -> io::Result<()> {
         for block in self.blocks {
-            if block.styles.is_empty() {
-                continue;
+            match block {
+                Toplevel::RuleSet(selector, styles) => {
+                    if styles.is_empty() {
+                        continue;
+                    }
+                    writeln!(buf, "{} {{", selector)?;
+                    for style in styles {
+                        write!(buf, "{}", style)?;
+                    }
+                    writeln!(buf, "}}")?;
+                }
+                Toplevel::MultilineComment(s) => {
+                    writeln!(buf, "/*{}*/", s)?;
+                }
             }
-            writeln!(buf, "{} {{", block.selector)?;
-            for style in block.styles {
-                writeln!(buf, "  {}", style)?;
-            }
-            writeln!(buf, "}}")?;
         }
         Ok(())
     }
diff --git a/src/format.rs b/src/format.rs
index 4a35f9c..e5ed279 100644
--- a/src/format.rs
+++ b/src/format.rs
@@ -17,6 +17,7 @@ impl<W: Write> PrettyPrinter<W> {
     fn pretty_print_stmt(&mut self, stmt: &Stmt) -> io::Result<()> {
         let padding = vec![' '; self.scope * 2].iter().collect::<String>();
         match stmt {
+            Stmt::MultilineComment(s) => writeln!(self.buf, "{}/*{}*/", padding, s)?,
             Stmt::RuleSet(RuleSet {
                 selector, rules, ..
             }) => {
@@ -51,6 +52,7 @@ impl<W: Write> PrettyPrinter<W> {
     fn pretty_print_stmt_preserve_super_selectors(&mut self, stmt: &Stmt) -> io::Result<()> {
         let padding = vec![' '; self.scope * 2].iter().collect::<String>();
         match stmt {
+            Stmt::MultilineComment(s) => writeln!(self.buf, "/*{}*/", s)?,
             Stmt::RuleSet(RuleSet {
                 selector,
                 rules,
@@ -244,12 +246,6 @@ mod test_scss {
         "a {\n  height: 1 1px;\n}\n"
     );
 
-    test!(
-        removes_single_line_comment,
-        "// a { color: red }\na {\n  height: 1 1px;\n}\n",
-        "a {\n  height: 1 1px;\n}\n"
-    );
-
     test!(keyword_important, "a {\n  height: 1 !important;\n}\n");
     test!(
         keyword_important_uppercase,
@@ -272,6 +268,38 @@ mod test_scss {
         "a {\n  foo: -webkit-bar-baz;\n}\n"
     );
 
+    test!(
+        removes_inner_comments,
+        "a {\n  color: red/* hi */;\n}\n",
+        "a {\n  color: red;\n}\n"
+    );
+    test!(
+        removes_inner_comments_whitespace,
+        "a {\n  color: red    /* hi */;\n}\n",
+        "a {\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_outer_comments_before,
+        "a {\n  /* hi */\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_outer_comments_after,
+        "a {\n  color: red;\n  /* hi */\n}\n"
+    );
+    test!(
+        preserves_outer_comments_two,
+        "a {\n  /* foo */\n  /* bar */\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_toplevel_comment,
+        "/* foo */\na {\n  color: red;\n}\n"
+    );
+    test!(
+        removes_single_line_comment,
+        "// a { color: red }\na {\n  height: 1 1px;\n}\n",
+        "a {\n  height: 1 1px;\n}\n"
+    );
+
     test!(unit_none, "a {\n  height: 1;\n}\n");
     test!(unit_not_attached, "a {\n  height: 1 px;\n}\n");
     test!(unit_px, "a {\n  height: 1px;\n}\n");
diff --git a/src/lexer.rs b/src/lexer.rs
index 88ccd77..ea592c1 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -161,6 +161,8 @@ impl<'a> Lexer<'a> {
                 self.buf.by_ref().take_while(|x| x != &'\n').for_each(drop);
             }
             '*' => {
+                self.buf.next();
+                self.pos.next_char();
                 let mut comment = String::new();
                 while let Some(tok) = self.buf.next() {
                     if tok == '\n' {
@@ -174,7 +176,7 @@ impl<'a> Lexer<'a> {
                     }
                     comment.push(tok);
                 }
-                return TokenKind::MultilineComment(comment)
+                return TokenKind::MultilineComment(comment);
             }
             _ => return TokenKind::Symbol(Symbol::Div),
         }
diff --git a/src/main.rs b/src/main.rs
index 1932cfb..dbf5851 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -51,6 +51,12 @@ mod units;
 
 type SassResult<T> = Result<T, SassError>;
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct Token {
+    pos: Pos,
+    pub kind: TokenKind,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum TokenKind {
     Ident(String),
@@ -88,12 +94,6 @@ impl Display for TokenKind {
     }
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct Token {
-    pos: Pos,
-    pub kind: TokenKind,
-}
-
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub struct StyleSheet {
     rules: Vec<Stmt>,
@@ -103,7 +103,7 @@ pub struct StyleSheet {
 pub enum Stmt {
     Style(Style),
     RuleSet(RuleSet),
-    // MultilineComment(String),
+    MultilineComment(String),
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -119,6 +119,7 @@ enum Expr {
     Style(Style),
     Selector(Selector),
     VariableDecl(String, Vec<Token>),
+    MultilineComment(String),
 }
 
 impl StyleSheet {
@@ -201,8 +202,8 @@ impl<'a> StyleSheetParser<'a> {
                     self.global_variables.insert(name, val);
                 }
                 TokenKind::MultilineComment(comment) => {
-                    todo!("MultilineComment");
-                    // rules.push(Stmt::MultilineComment(comment));
+                    self.lexer.next();
+                    rules.push(Stmt::MultilineComment(comment));
                 }
                 TokenKind::AtRule(_) => self.eat_at_rule(),
                 _ => todo!("unexpected toplevel token"),
@@ -315,6 +316,7 @@ impl<'a> StyleSheetParser<'a> {
                         vars.insert(name, val);
                     }
                 }
+                Expr::MultilineComment(s) => stmts.push(Stmt::MultilineComment(s)),
             }
         }
         stmts
@@ -351,6 +353,14 @@ impl<'a> StyleSheetParser<'a> {
                         });
                     }
                 }
+                TokenKind::MultilineComment(ref s) => {
+                    self.devour_whitespace();
+                    if values.is_empty() {
+                        return Ok(Expr::MultilineComment(s.clone()));
+                    } else {
+                        values.push(tok.clone())
+                    }
+                }
                 _ => values.push(tok.clone()),
             };
         }
@@ -424,6 +434,42 @@ mod test_css {
     test!(basic_style, "a {\n  color: red;\n}\n");
     test!(two_styles, "a {\n  color: red;\n  color: blue;\n}\n");
     test!(selector_mul, "a, b {\n  color: red;\n}\n");
-    test!(removes_empty_outer_styles, "a {\n  b {\n    color: red;\n  }\n", "a b {\n  color: red;\n}\n");
+    test!(
+        removes_empty_outer_styles,
+        "a {\n  b {\n    color: red;\n  }\n",
+        "a b {\n  color: red;\n}\n"
+    );
     test!(removes_empty_styles, "a {}\n", "");
+
+    test!(
+        removes_inner_comments,
+        "a {\n  color: red/* hi */;\n}\n",
+        "a {\n  color: red;\n}\n"
+    );
+    test!(
+        removes_inner_comments_whitespace,
+        "a {\n  color: red    /* hi */;\n}\n",
+        "a {\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_outer_comments_before,
+        "a {\n  /* hi */\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_outer_comments_after,
+        "a {\n  color: red;\n  /* hi */\n}\n"
+    );
+    test!(
+        preserves_outer_comments_two,
+        "a {\n  /* foo */\n  /* bar */\n  color: red;\n}\n"
+    );
+    test!(
+        preserves_toplevel_comment,
+        "/* foo */\na {\n  color: red;\n}\n"
+    );
+    test!(
+        removes_single_line_comment,
+        "// a { color: red }\na {\n  height: 1 1px;\n}\n",
+        "a {\n  height: 1 1px;\n}\n"
+    );
 }
diff --git a/src/style.rs b/src/style.rs
index 69643a1..d847fbc 100644
--- a/src/style.rs
+++ b/src/style.rs
@@ -66,22 +66,21 @@ impl<'a> StyleParser<'a> {
         val
     }
 
-    fn devour_whitespace(&mut self) {
-        while let Some(tok) = self.tokens.peek() {
-            if let TokenKind::Whitespace(_) = tok.kind {
-                self.tokens.next();
-            } else {
-                break;
-            }
+    fn devour_whitespace_or_comment(&mut self) {
+        while let Some(Token { kind, .. }) = self.tokens.peek() {
+            match kind {
+                TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => self.tokens.next(),
+                _ => break,
+            };
         }
     }
 
     fn parse(&mut self) -> Style {
         let mut property = String::new();
         // read property
-        while let Some(tok) = self.tokens.next() {
-            match tok.kind {
-                TokenKind::Whitespace(_) => continue,
+        while let Some(Token { kind, .. }) = self.tokens.next() {
+            match kind {
+                TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => continue,
                 TokenKind::Ident(ref s) => {
                     property = s.clone();
                     break;
@@ -91,9 +90,9 @@ impl<'a> StyleParser<'a> {
         }
 
         // read until `:`
-        while let Some(tok) = self.tokens.next() {
-            match tok.kind {
-                TokenKind::Whitespace(_) => continue,
+        while let Some(Token { kind, .. }) = self.tokens.next() {
+            match kind {
+                TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => continue,
                 TokenKind::Symbol(Symbol::Colon) => break,
                 _ => todo!("found tokens before style value"),
             }
@@ -105,23 +104,28 @@ impl<'a> StyleParser<'a> {
         while let Some(tok) = self.tokens.next() {
             match &tok.kind {
                 TokenKind::Whitespace(_) => {
-                    while let Some(w) = self.tokens.peek() {
-                        if let TokenKind::Whitespace(_) = w.kind {
-                            self.tokens.next();
-                            continue;
-                        } else if let TokenKind::Ident(ref s) = w.kind {
-                            if s == &String::from("-") {
+                    while let Some(Token { kind, .. }) = self.tokens.peek() {
+                        match kind {
+                            TokenKind::Whitespace(_) | TokenKind::MultilineComment(_) => {
                                 self.tokens.next();
-                                value.push('-');
-                                self.devour_whitespace();
-                                break;
+                                continue;
                             }
+                            TokenKind::Ident(ref s) => {
+                                if s == &String::from("-") {
+                                    self.tokens.next();
+                                    value.push('-');
+                                    self.devour_whitespace_or_comment();
+                                    break;
+                                }
+                            }
+                            _ => {}
                         }
                         value.push(' ');
                         break;
                     }
                 }
                 TokenKind::Variable(_) => value.push_str(&self.deref_variable(&tok.kind)),
+                TokenKind::MultilineComment(_) => continue,
                 _ => value.push_str(&tok.kind.to_string()),
             }
         }