Parse anonymous data blocks

http://www.adobe.com/devnet/opentype/afdko/topic_feature_file_syntax.html#10

For example, @mhosken is interested in experimenting with inlining
custom syntax (such as Python snippets) into feature files. After this
change, such experiments can be done on top of feaLib because the
Abstract Syntax Tree now contains the tag and content of `anonymous`
blocks.

Lib/fontTools/feaLib/ast.py

@@ -70,6 +70,12 @@ class MarkClassName(Expression):
         return self.markClass.glyphSet()
 
 
+class AnonymousBlock(Statement):
+    def __init__(self, tag, content, location):
+        Statement.__init__(self, location)
+        self.tag, self.content = tag, content
+
+
 class Block(Statement):
     def __init__(self, location):
         Statement.__init__(self, location)

Lib/fontTools/feaLib/builder_test.py

@@ -58,6 +58,7 @@ class BuilderTest(unittest.TestCase):
         spec5h1 spec6b_ii spec6d2 spec6e spec6f
         spec6h_ii spec6h_iii_1 spec6h_iii_3d spec8a spec8b spec8c
         spec9a spec9b spec9c1 spec9c2 spec9c3 spec9d spec9e spec9f
+        spec10
         bug453 bug463 bug501 bug502 bug504 bug505 bug506 bug509 bug512 bug568
         name size size2 multiple_feature_blocks
     """.split()

Lib/fontTools/feaLib/lexer.py

@@ -17,6 +17,7 @@ class Lexer(object):
     SYMBOL = "SYMBOL"
     COMMENT = "COMMENT"
     NEWLINE = "NEWLINE"
+    ANONYMOUS_BLOCK = "ANONYMOUS_BLOCK"
 
     CHAR_WHITESPACE_ = " \t"
     CHAR_NEWLINE_ = "\r\n"
@@ -53,10 +54,13 @@ class Lexer(object):
             if token_type not in {Lexer.COMMENT, Lexer.NEWLINE}:
                 return (token_type, token, location)
 
+    def location_(self):
+        column = self.pos_ - self.line_start_ + 1
+        return (self.filename_, self.line_, column)
+
     def next_(self):
         self.scan_over_(Lexer.CHAR_WHITESPACE_)
-        column = self.pos_ - self.line_start_ + 1
-        location = (self.filename_, self.line_, column)
+        location = self.location_()
         start = self.pos_
         text = self.text_
         limit = len(text)
@@ -166,6 +170,20 @@ class Lexer(object):
             p += 1
         self.pos_ = p
 
+    def scan_anonymous_block(self, tag):
+        location = self.location_()
+        tag = tag.strip()
+        self.scan_until_(Lexer.CHAR_NEWLINE_)
+        self.scan_over_(Lexer.CHAR_NEWLINE_)
+        regexp = r'}\s*' + tag + r'\s*;'
+        split = re.split(regexp, self.text_[self.pos_:], maxsplit=1)
+        if len(split) != 2:
+            raise FeatureLibError(
+                "Expected '} %s;' to terminate anonymous block" % tag,
+                location)
+        self.pos_ += len(split[0])
+        return (Lexer.ANONYMOUS_BLOCK, split[0], location)
+
 
 class IncludingLexer(object):
     def __init__(self, featurefile):
@@ -218,3 +236,6 @@ class IncludingLexer(object):
         if closing:
             fileobj.close()
         return Lexer(data, filename)
+
+    def scan_anonymous_block(self, tag):
+        return self.lexers_[-1].scan_anonymous_block(tag)

Lib/fontTools/feaLib/parser.py

@@ -34,6 +34,8 @@ class Parser(object):
             self.advance_lexer_()
             if self.cur_token_type_ is Lexer.GLYPHCLASS:
                 statements.append(self.parse_glyphclass_definition_())
+            elif self.is_cur_keyword_(("anon", "anonymous")):
+                statements.append(self.parse_anonymous_())
             elif self.is_cur_keyword_("anchorDef"):
                 statements.append(self.parse_anchordef_())
             elif self.is_cur_keyword_("languagesystem"):
@@ -123,6 +125,17 @@ class Parser(object):
         self.anchors_.define(name, anchordef)
         return anchordef
 
+    def parse_anonymous_(self):
+        assert self.is_cur_keyword_(("anon", "anonymous"))
+        tag = self.expect_tag_()
+        _, content, location = self.lexer_.scan_anonymous_block(tag)
+        self.advance_lexer_()
+        self.expect_symbol_('}')
+        end_tag = self.expect_tag_()
+        assert tag == end_tag, "bad splitting in Lexer.scan_anonymous_block()"
+        self.expect_symbol_(';')
+        return ast.AnonymousBlock(tag, content, location)
+
     def parse_attach_(self):
         assert self.is_cur_keyword_("Attach")
         location = self.cur_token_location_

Lib/fontTools/feaLib/parser_test.py

@@ -114,6 +114,24 @@ class ParserTest(unittest.TestCase):
         self.assertEqual(foo.y, 456)
         self.assertEqual(foo.contourpoint, 5)
 
+    def test_anon(self):
+        anon = self.parse("anon TEST { # a\nfoo\n } TEST; # qux").statements[0]
+        self.assertIsInstance(anon, ast.AnonymousBlock)
+        self.assertEqual(anon.tag, "TEST")
+        self.assertEqual(anon.content, "foo\n ")
+
+    def test_anonymous(self):
+        anon = self.parse("anonymous TEST {\nbar\n} TEST;").statements[0]
+        self.assertIsInstance(anon, ast.AnonymousBlock)
+        self.assertEqual(anon.tag, "TEST")
+        # feature file spec requires passing the final end-of-line
+        self.assertEqual(anon.content, "bar\n")
+
+    def test_anon_missingBrace(self):
+        self.assertRaisesRegex(
+            FeatureLibError, "Expected '} TEST;' to terminate anonymous block",
+            self.parse, "anon TEST { \n no end in sight")
+
     def test_attach(self):
         doc = self.parse("table GDEF {Attach [a e] 2;} GDEF;")
         s = doc.statements[0].statements[0]

Lib/fontTools/feaLib/testdata/spec10.fea

@@ -0,0 +1,12 @@
+# OpenType Feature File specification, section 10.
+# http://www.adobe.com/devnet/opentype/afdko/topic_feature_file_syntax.html
+
+anon sbit {
+    /* sbit table specifications */
+    72  % dpi
+    sizes {
+        10, 12, 14 source {
+            all "Generic/JGeneric"
+        }
+    }
+} sbit;

Lib/fontTools/feaLib/testdata/spec10.ttx

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ttFont>
+
+</ttFont>