shadowfacts/grass
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Normalize newlines in comments

Browse Source
This commit is contained in:
ConnorSkees 2020-01-26 16:50:08 -05:00
parent f3787ef305
commit a8ebc91702
2 changed files with 43 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/lexer.rs
View File

@ -7,6 +7,9 @@ use crate::common::{Keyword, Op, Pos, Symbol};
use crate::selector::{Attribute, AttributeKind, CaseKind};
use crate::{Token, TokenKind, Whitespace};
// Rust does not allow us to escape '\f'
const FORM_FEED: char = '\x0C';
#[derive(Debug, Clone)]
pub(crate) struct Lexer<'a> {
tokens: Vec<Token>,
@ -51,7 +54,7 @@ impl<'a> Iterator for Lexer<'a> {
'"' => symbol!(self, DoubleQuote),
' ' => whitespace!(self, Space),
'\t' => whitespace!(self, Tab),
'\n' | '\x0C' => {
'\n' | &FORM_FEED => {
self.buf.next();
self.pos.newline();
TokenKind::Whitespace(Whitespace::Newline)
@ -170,15 +173,27 @@ impl<'a> Lexer<'a> {
self.pos.next_char();
let mut comment = String::new();
while let Some(tok) = self.buf.next() {
if tok == '\n' {
self.pos.newline()
} else {
self.pos.next_char();
match tok {
'\n' => self.pos.newline(),
FORM_FEED => {
self.pos.newline();
comment.push('\n');
continue;
}
if tok == '*' && self.buf.peek() == Some(&'/') {
'\r' => {
if self.buf.peek() == Some(&'\n') {
self.buf.next();
}
self.pos.newline();
comment.push('\n');
continue;
}
'*' if self.buf.peek() == Some(&'/') => {
self.buf.next();
break;
}
_ => self.pos.next_char(),
}
comment.push(tok);
}
return TokenKind::MultilineComment(comment);

19
tests/main.rs
View File

@ -345,6 +345,21 @@ mod test_comments {
"// a { color: red }\na {\n height: 1 1px;\n}\n",
"a {\n height: 1 1px;\n}\n"
);
test!(
converts_form_feed_in_comment,
"a {\n /* \x0C*/ color: red;\n}\n",
"a {\n /* \n*/\n color: red;\n}\n"
);
test!(
converts_crlf_in_comment,
"a {\n /* \r\n*/ color: red;\n}\n",
"a {\n /* \n*/\n color: red;\n}\n"
);
test!(
converts_cr_in_comment,
"a {\n /* \r*/ color: red;\n}\n",
"a {\n /* \n*/\n color: red;\n}\n"
);
}
#[cfg(test)]
@ -699,6 +714,10 @@ mod test_values {
test!(preserves_keyword_or, "a {\n color: or;\n}\n");
test!(preserves_keyword_unset, "a {\n color: unset;\n}\n");
test!(preserves_keyword_nan, "a {\n color: NaN;\n}\n");
test!(
preserves_quotes,
"a {\n color: \"'red' \\\"blue\\\"\";\n}\n"
);
test!(
whitespace_space_list_number,
"a {\n color: 1 2 3 ;\n}\n",