shadowfacts/fonttools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add tests, rename _OMITs

Browse Source
This commit is contained in:
Martin Hosken 2017-03-08 15:19:09 +00:00
parent 90b57d2dec
commit 45a4a1f249
4 changed files with 53 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Lib/fontTools/feaLib/lexer.py
View File

@ -32,8 +32,8 @@ class Lexer(object):
MODE_NORMAL_ = "NORMAL"
MODE_FILENAME_ = "FILENAME"
_OMITS = {NEWLINE}
_OMITC = {NEWLINE, COMMENT}
_OMITMINIMUM = {NEWLINE}
_OMITCOMMENTS = {NEWLINE, COMMENT}
def __init__(self, text, filename):
self.filename_ = filename
@ -47,13 +47,13 @@ class Lexer(object):
def __iter__(self):
return self
def next(self, comments = False): # Python 2
def next(self, comments=False): # Python 2
return self.__next__(comments)
def __next__(self, comments = False): # Python 3
def __next__(self, comments=False): # Python 3
while True:
token_type, token, location = self.next_()
if token_type not in (Lexer._OMITS if comments else Lexer._OMITC):
if token_type not in (Lexer._OMITMINIMUM if comments else Lexer._OMITCOMMENTS):
return (token_type, token, location)
def location_(self):
@ -195,10 +195,10 @@ class IncludingLexer(object):
def __iter__(self):
return self
def next(self, comments = False): # Python 2
def next(self, comments=False): # Python 2
return self.__next__(comments)
def __next__(self, comments = False): # Python 3
def __next__(self, comments=False): # Python 3
while self.lexers_:
lexer = self.lexers_[-1]
try:

25
Lib/fontTools/feaLib/parser.py
View File

@ -10,6 +10,7 @@ import os
import re
log = logging.getLogger(__name__)
@ -17,7 +18,7 @@ class Parser(object):
extensions = {}
ast = ast
ignore_comments = True
ignore_comments=True
def __init__(self, featurefile, glyphMap):
self.glyphMap_ = glyphMap
@ -33,12 +34,12 @@ class Parser(object):
self.cur_comments_ = []
self.next_token_location_ = None
self.lexer_ = IncludingLexer(featurefile)
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
def parse(self):
statements = self.doc_.statements
while self.next_token_type_ is not None:
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.cur_token_type_ is Lexer.GLYPHCLASS:
@ -792,7 +793,7 @@ class Parser(object):
def parse_table_GDEF_(self, table):
statements = table.statements
while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)):
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.is_cur_keyword_("Attach"):
@ -812,7 +813,7 @@ class Parser(object):
def parse_table_head_(self, table):
statements = table.statements
while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)):
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.is_cur_keyword_("FontRevision"):
@ -825,7 +826,7 @@ class Parser(object):
statements = table.statements
fields = ("CaretOffset", "Ascender", "Descender", "LineGap")
while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)):
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.cur_token_type_ is Lexer.NAME and self.cur_token_ in fields:
@ -846,7 +847,7 @@ class Parser(object):
statements = table.statements
fields = ("VertTypoAscender", "VertTypoDescender", "VertTypoLineGap")
while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)):
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.cur_token_type_ is Lexer.NAME and self.cur_token_ in fields:
@ -866,7 +867,7 @@ class Parser(object):
def parse_table_name_(self, table):
statements = table.statements
while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)):
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.is_cur_keyword_("nameid"):
@ -949,7 +950,7 @@ class Parser(object):
def parse_table_BASE_(self, table):
statements = table.statements
while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)):
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.is_cur_keyword_("HorizAxis.BaseTagList"):
@ -976,7 +977,7 @@ class Parser(object):
"WeightClass", "WidthClass", "LowerOpSize", "UpperOpSize")
ranges = ("UnicodeRange", "CodePageRange")
while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)):
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.cur_token_type_ is Lexer.NAME:
@ -1187,7 +1188,7 @@ class Parser(object):
statements = block.statements
while self.next_token_ != "}" or (not self.ignore_comments and len(self.cur_comments_)):
self.advance_lexer_(comments = True)
self.advance_lexer_(comments=True)
if self.cur_token_type_ is Lexer.COMMENT:
statements.append(self.ast.Comment(self.cur_token_location_, self.cur_token_))
elif self.cur_token_type_ is Lexer.GLYPHCLASS:
@ -1375,7 +1376,7 @@ class Parser(object):
while True:
try:
(self.next_token_type_, self.next_token_,
self.next_token_location_) = self.lexer_.next(comments = True)
self.next_token_location_) = self.lexer_.next(comments=True)
except StopIteration:
self.next_token_type_, self.next_token_ = (None, None)
if self.next_token_type_ != Lexer.COMMENT:

14
Tests/feaLib/lexer_test.py
View File

@ -9,6 +9,16 @@ import unittest
def lex(s):
return [(typ, tok) for (typ, tok, _) in Lexer(s, "test.fea")]
def lex_with_comments(s):
l = Lexer(s, "test.fea")
res = []
while True:
try:
(typ, tok, _) = l.next(comments=True)
res.append((typ, tok))
except StopIteration :
break
return res
class LexerTest(unittest.TestCase):
def __init__(self, methodName):
@ -82,6 +92,10 @@ class LexerTest(unittest.TestCase):
def test_comment(self):
self.assertEqual(lex("# Comment\n#"), [])
def test_comment_kept(self):
self.assertEqual(lex_with_comments("# Comment\n#"),
[(Lexer.COMMENT, "# Comment"), (Lexer.COMMENT, "#")])
def test_string(self):
self.assertEqual(lex('"foo" "bar"'),
[(Lexer.STRING, "foo"), (Lexer.STRING, "bar")])

21
Tests/feaLib/parser_test.py
View File

@ -55,6 +55,20 @@ class ParserTest(unittest.TestCase):
if not hasattr(self, "assertRaisesRegex"):
self.assertRaisesRegex = self.assertRaisesRegexp
def test_comments(self):
doc = self.parse(
""" # Initial
feature test {
sub A by B; # simple
} test;""", comments=True)
c1 = doc.statements[0]
c2 = doc.statements[1].statements[1]
self.assertEqual(type(c1), ast.Comment)
self.assertEqual(c1.text, "# Initial")
self.assertEqual(type(c2), ast.Comment)
self.assertEqual(c2.text, "# simple")
self.assertEqual(doc.statements[1].name, "test")
def test_anchor_format_a(self):
doc = self.parse(
"feature test {"
@ -1424,9 +1438,12 @@ class ParserTest(unittest.TestCase):
doc = self.parse(";;;")
self.assertFalse(doc.statements)
def parse(self, text, glyphMap=GLYPHMAP):
def parse(self, text, glyphMap=GLYPHMAP, comments=False):
featurefile = UnicodeIO(text)
return Parser(featurefile, glyphMap).parse()
p = Parser(featurefile, glyphMap)
if comments :
p.ignore_comments = False
return p.parse()
@staticmethod
def getpath(testfile):