Add tests, rename _OMITs

Lib/fontTools/feaLib/lexer.py

@@ -32,8 +32,8 @@ class Lexer(object):
     MODE_NORMAL_ = "NORMAL"
     MODE_FILENAME_ = "FILENAME"
 
-    _OMITS = {NEWLINE}
-    _OMITC = {NEWLINE, COMMENT}
+    _OMITMINIMUM = {NEWLINE}
+    _OMITCOMMENTS = {NEWLINE, COMMENT}
 
     def __init__(self, text, filename):
         self.filename_ = filename
@@ -53,7 +53,7 @@ class Lexer(object):
     def __next__(self, comments=False):  # Python 3
         while True:
             token_type, token, location = self.next_()
-            if token_type not in (Lexer._OMITS if comments else Lexer._OMITC):
+            if token_type not in (Lexer._OMITMINIMUM if comments else Lexer._OMITCOMMENTS):
                 return (token_type, token, location)
 
     def location_(self):

Lib/fontTools/feaLib/parser.py

@@ -10,6 +10,7 @@ import os
 import re
 
 
+
 log = logging.getLogger(__name__)
 
 

Tests/feaLib/lexer_test.py

@@ -9,6 +9,16 @@ import unittest
 def lex(s):
     return [(typ, tok) for (typ, tok, _) in Lexer(s, "test.fea")]
 
+def lex_with_comments(s):
+    l = Lexer(s, "test.fea")
+    res = []
+    while True:
+        try:
+            (typ, tok, _) = l.next(comments=True)
+            res.append((typ, tok))
+        except StopIteration :
+            break
+    return res
 
 class LexerTest(unittest.TestCase):
     def __init__(self, methodName):
@@ -82,6 +92,10 @@ class LexerTest(unittest.TestCase):
     def test_comment(self):
         self.assertEqual(lex("# Comment\n#"), [])
 
+    def test_comment_kept(self):
+        self.assertEqual(lex_with_comments("# Comment\n#"),
+                         [(Lexer.COMMENT, "# Comment"), (Lexer.COMMENT, "#")])
+            
     def test_string(self):
         self.assertEqual(lex('"foo" "bar"'),
                          [(Lexer.STRING, "foo"), (Lexer.STRING, "bar")])

Tests/feaLib/parser_test.py

@@ -55,6 +55,20 @@ class ParserTest(unittest.TestCase):
         if not hasattr(self, "assertRaisesRegex"):
             self.assertRaisesRegex = self.assertRaisesRegexp
 
+    def test_comments(self):
+        doc = self.parse(
+            """ # Initial
+                feature test {
+                    sub A by B; # simple
+                } test;""", comments=True)
+        c1 = doc.statements[0]
+        c2 = doc.statements[1].statements[1]
+        self.assertEqual(type(c1), ast.Comment)
+        self.assertEqual(c1.text, "# Initial")
+        self.assertEqual(type(c2), ast.Comment)
+        self.assertEqual(c2.text, "# simple")
+        self.assertEqual(doc.statements[1].name, "test")
+
     def test_anchor_format_a(self):
         doc = self.parse(
             "feature test {"
@@ -1424,9 +1438,12 @@ class ParserTest(unittest.TestCase):
         doc = self.parse(";;;")
         self.assertFalse(doc.statements)
 
-    def parse(self, text, glyphMap=GLYPHMAP):
+    def parse(self, text, glyphMap=GLYPHMAP, comments=False):
         featurefile = UnicodeIO(text)
-        return Parser(featurefile, glyphMap).parse()
+        p = Parser(featurefile, glyphMap)
+        if comments :
+            p.ignore_comments = False
+        return p.parse()
 
     @staticmethod
     def getpath(testfile):