diff --git a/Lib/fontTools/feaLib/lexer.py b/Lib/fontTools/feaLib/lexer.py index db850fe4d..6076a3bd7 100644 --- a/Lib/fontTools/feaLib/lexer.py +++ b/Lib/fontTools/feaLib/lexer.py @@ -32,8 +32,8 @@ class Lexer(object): MODE_NORMAL_ = "NORMAL" MODE_FILENAME_ = "FILENAME" - _OMITS = {NEWLINE} - _OMITC = {NEWLINE, COMMENT} + _OMITMINIMUM = {NEWLINE} + _OMITCOMMENTS = {NEWLINE, COMMENT} def __init__(self, text, filename): self.filename_ = filename @@ -47,13 +47,13 @@ class Lexer(object): def __iter__(self): return self - def next(self, comments = False): # Python 2 + def next(self, comments=False): # Python 2 return self.__next__(comments) - def __next__(self, comments = False): # Python 3 + def __next__(self, comments=False): # Python 3 while True: token_type, token, location = self.next_() - if token_type not in (Lexer._OMITS if comments else Lexer._OMITC): + if token_type not in (Lexer._OMITMINIMUM if comments else Lexer._OMITCOMMENTS): return (token_type, token, location) def location_(self): @@ -195,10 +195,10 @@ class IncludingLexer(object): def __iter__(self): return self - def next(self, comments = False): # Python 2 + def next(self, comments=False): # Python 2 return self.__next__(comments) - def __next__(self, comments = False): # Python 3 + def __next__(self, comments=False): # Python 3 while self.lexers_: lexer = self.lexers_[-1] try: diff --git a/Lib/fontTools/feaLib/parser.py b/Lib/fontTools/feaLib/parser.py index 23d7dba05..b889c142b 100644 --- a/Lib/fontTools/feaLib/parser.py +++ b/Lib/fontTools/feaLib/parser.py @@ -10,6 +10,7 @@ import os import re + log = logging.getLogger(__name__) @@ -17,7 +18,7 @@ class Parser(object): extensions = {} ast = ast - ignore_comments = True + ignore_comments=True def __init__(self, featurefile, glyphMap): self.glyphMap_ = glyphMap @@ -33,12 +34,12 @@ class Parser(object): self.cur_comments_ = [] self.next_token_location_ = None self.lexer_ = IncludingLexer(featurefile) - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) def parse(self): statements = self.doc_.statements while self.next_token_type_ is not None: - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.cur_token_type_ is Lexer.GLYPHCLASS: @@ -792,7 +793,7 @@ class Parser(object): def parse_table_GDEF_(self, table): statements = table.statements while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)): - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.is_cur_keyword_("Attach"): @@ -812,7 +813,7 @@ class Parser(object): def parse_table_head_(self, table): statements = table.statements while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)): - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.is_cur_keyword_("FontRevision"): @@ -825,7 +826,7 @@ class Parser(object): statements = table.statements fields = ("CaretOffset", "Ascender", "Descender", "LineGap") while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)): - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.cur_token_type_ is Lexer.NAME and self.cur_token_ in fields: @@ -846,7 +847,7 @@ class Parser(object): statements = table.statements fields = ("VertTypoAscender", "VertTypoDescender", "VertTypoLineGap") while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)): - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.cur_token_type_ is Lexer.NAME and self.cur_token_ in fields: @@ -866,7 +867,7 @@ class Parser(object): def parse_table_name_(self, table): statements = table.statements while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)): - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.is_cur_keyword_("nameid"): @@ -949,7 +950,7 @@ class Parser(object): def parse_table_BASE_(self, table): statements = table.statements while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)): - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.is_cur_keyword_("HorizAxis.BaseTagList"): @@ -976,7 +977,7 @@ class Parser(object): "WeightClass", "WidthClass", "LowerOpSize", "UpperOpSize") ranges = ("UnicodeRange", "CodePageRange") while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)): - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.cur_token_type_ is Lexer.NAME: @@ -1187,7 +1188,7 @@ class Parser(object): statements = block.statements while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)): - self.advance_lexer_(comments = True) + self.advance_lexer_(comments=True) if self.cur_token_type_ is Lexer.COMMENT: statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_)) elif self.cur_token_type_ is Lexer.GLYPHCLASS: @@ -1375,7 +1376,7 @@ class Parser(object): while True: try: (self.next_token_type_, self.next_token_, - self.next_token_location_) = self.lexer_.next(comments = True) + self.next_token_location_) = self.lexer_.next(comments=True) except StopIteration: self.next_token_type_, self.next_token_ = (None, None) if self.next_token_type_ != Lexer.COMMENT: diff --git a/Tests/feaLib/lexer_test.py b/Tests/feaLib/lexer_test.py index ddcd4e558..e7fa038bf 100644 --- a/Tests/feaLib/lexer_test.py +++ b/Tests/feaLib/lexer_test.py @@ -9,6 +9,16 @@ import unittest def lex(s): return [(typ, tok) for (typ, tok, _) in Lexer(s, "test.fea")] +def lex_with_comments(s): + l = Lexer(s, "test.fea") + res = [] + while True: + try: + (typ, tok, _) = l.next(comments=True) + res.append((typ, tok)) + except StopIteration : + break + return res class LexerTest(unittest.TestCase): def __init__(self, methodName): @@ -82,6 +92,10 @@ class LexerTest(unittest.TestCase): def test_comment(self): self.assertEqual(lex("# Comment\n#"), []) + def test_comment_kept(self): + self.assertEqual(lex_with_comments("# Comment\n#"), + [(Lexer.COMMENT, "# Comment"), (Lexer.COMMENT, "#")]) + def test_string(self): self.assertEqual(lex('"foo" "bar"'), [(Lexer.STRING, "foo"), (Lexer.STRING, "bar")]) diff --git a/Tests/feaLib/parser_test.py b/Tests/feaLib/parser_test.py index 4257fac53..69137a6cd 100644 --- a/Tests/feaLib/parser_test.py +++ b/Tests/feaLib/parser_test.py @@ -55,6 +55,20 @@ class ParserTest(unittest.TestCase): if not hasattr(self, "assertRaisesRegex"): self.assertRaisesRegex = self.assertRaisesRegexp + def test_comments(self): + doc = self.parse( + """ # Initial + feature test { + sub A by B; # simple + } test;""", comments=True) + c1 = doc.statements[0] + c2 = doc.statements[1].statements[1] + self.assertEqual(type(c1), ast.Comment) + self.assertEqual(c1.text, "# Initial") + self.assertEqual(type(c2), ast.Comment) + self.assertEqual(c2.text, "# simple") + self.assertEqual(doc.statements[1].name, "test") + def test_anchor_format_a(self): doc = self.parse( "feature test {" @@ -1424,9 +1438,12 @@ class ParserTest(unittest.TestCase): doc = self.parse(";;;") self.assertFalse(doc.statements) - def parse(self, text, glyphMap=GLYPHMAP): + def parse(self, text, glyphMap=GLYPHMAP, comments=False): featurefile = UnicodeIO(text) - return Parser(featurefile, glyphMap).parse() + p = Parser(featurefile, glyphMap) + if comments : + p.ignore_comments = False + return p.parse() @staticmethod def getpath(testfile):