Merge pull request #8 from tree-sitter/highlight-properties

Add property sheet for syntax highlighting

.travis.yml

@@ -5,7 +5,7 @@ sudo: false
 node_js: node
 
 env:
-  - CXX=clang
+  - CXX=clang++
 
 branches:
   only:

appveyor.yml

@@ -13,7 +13,7 @@ install:
   - npm install
 
 test_script:
-  - npm test
+  - npm run test-windows
 
 build: off
 

corpus/main.txt

@@ -148,12 +148,12 @@ Raw text elements
 ---
 
 (fragment
-  (raw_element
+  (script_element
     (start_tag (tag_name))
     (raw_text)
     (end_tag (tag_name)))
   (text)
-  (raw_element
+  (style_element
     (start_tag (tag_name))
     (raw_text)
     (end_tag (tag_name)))

grammar.js

@@ -8,7 +8,8 @@ module.exports = grammar({
 
   externals: $ => [
     $._start_tag_name,
-    $._start_raw_tag_name,
+    $._script_start_tag_name,
+    $._style_start_tag_name,
     $._end_tag_name,
     $.erroneous_end_tag_name,
     '/>',
@@ -33,8 +34,9 @@ module.exports = grammar({
       $.doctype,
       $.text,
       $.element,
-      $.erroneous_end_tag,
+      $.script_element,
-      $.raw_element
+      $.style_element,
+      $.erroneous_end_tag
     ),
 
     element: $ => choice(
@@ -46,8 +48,14 @@ module.exports = grammar({
       $.self_closing_tag
     ),
 
-    raw_element: $ => seq(
+    script_element: $ => seq(
-      alias($._raw_start_tag, $.start_tag),
+      alias($.script_start_tag, $.start_tag),
+      optional($.raw_text),
+      $.end_tag
+    ),
+
+    style_element: $ => seq(
+      alias($.style_start_tag, $.start_tag),
       optional($.raw_text),
       $.end_tag
     ),
@@ -59,9 +67,16 @@ module.exports = grammar({
       '>'
     ),
 
-    _raw_start_tag: $ => seq(
+    script_start_tag: $ => seq(
       '<',
-      alias($._start_raw_tag_name, $.tag_name),
+      alias($._script_start_tag_name, $.tag_name),
+      repeat($.attribute),
+      '>'
+    ),
+
+    style_start_tag: $ => seq(
+      '<',
+      alias($._style_start_tag_name, $.tag_name),
       repeat($.attribute),
       '>'
     ),

package.json

@@ -16,10 +16,20 @@
     "nan": "^2.10.0"
   },
   "devDependencies": {
-    "tree-sitter-cli": "^0.13.8"
+    "tree-sitter-cli": "^0.14.4"
   },
   "scripts": {
     "build": "tree-sitter generate && node-gyp build",
-    "test": "tree-sitter test && tree-sitter parse examples --quiet --time"
+    "test": "tree-sitter test && tree-sitter parse examples/*.html --quiet --time",
-  }
+    "test-windows": "tree-sitter test"
+  },
+  "tree-sitter": [
+    {
+      "name": "HTML",
+      "textmate-scope": "text.html.basic",
+      "file-types": ["html"],
+      "highlights": "src/highlights.json",
+      "injection-regex": "html"
+    }
+  ]
 }

properties/highlights.css

@@ -0,0 +1,26 @@
+@import "./injections.css";
+
+tag_name {
+  scope: 'tag';
+}
+
+/* TODO - highlight as an error? */
+erroneous_end_tag_name {
+  scope: 'tag';
+}
+
+doctype {
+  scope: 'constant';
+}
+
+attribute_name {
+  scope: 'attribute';
+}
+
+attribute_value {
+  scope: 'string';
+}
+
+comment {
+  scope: 'comment';
+}

properties/injections.css

@@ -0,0 +1,9 @@
+script_element {
+  injection-language: 'javascript';
+  injection-content: child(this(), 1);
+}
+
+style_element {
+  injection-language: 'css';
+  injection-content: child(this(), 1);
+}

src/grammar.json

@@ -55,11 +55,15 @@
         },
         {
           "type": "SYMBOL",
-          "name": "erroneous_end_tag"
+          "name": "script_element"
         },
         {
           "type": "SYMBOL",
-          "name": "raw_element"
+          "name": "style_element"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "erroneous_end_tag"
         }
       ]
     },
@@ -101,14 +105,44 @@
         }
       ]
     },
-    "raw_element": {
+    "script_element": {
       "type": "SEQ",
       "members": [
         {
           "type": "ALIAS",
           "content": {
             "type": "SYMBOL",
-            "name": "_raw_start_tag"
+            "name": "script_start_tag"
+          },
+          "named": true,
+          "value": "start_tag"
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "raw_text"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "SYMBOL",
+          "name": "end_tag"
+        }
+      ]
+    },
+    "style_element": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "ALIAS",
+          "content": {
+            "type": "SYMBOL",
+            "name": "style_start_tag"
           },
           "named": true,
           "value": "start_tag"
@@ -160,7 +194,7 @@
         }
       ]
     },
-    "_raw_start_tag": {
+    "script_start_tag": {
       "type": "SEQ",
       "members": [
         {
@@ -171,7 +205,36 @@
           "type": "ALIAS",
           "content": {
             "type": "SYMBOL",
-            "name": "_start_raw_tag_name"
+            "name": "_script_start_tag_name"
+          },
+          "named": true,
+          "value": "tag_name"
+        },
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "SYMBOL",
+            "name": "attribute"
+          }
+        },
+        {
+          "type": "STRING",
+          "value": ">"
+        }
+      ]
+    },
+    "style_start_tag": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "<"
+        },
+        {
+          "type": "ALIAS",
+          "content": {
+            "type": "SYMBOL",
+            "name": "_style_start_tag_name"
           },
           "named": true,
           "value": "tag_name"
@@ -392,7 +455,11 @@
     },
     {
       "type": "SYMBOL",
-      "name": "_start_raw_tag_name"
+      "name": "_script_start_tag_name"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_style_start_tag_name"
     },
     {
       "type": "SYMBOL",
@@ -420,4 +487,4 @@
     }
   ],
   "inline": []
-}
+}

src/highlights.json

@@ -0,0 +1,424 @@
+{
+  "states": [
+    {
+      "id": 0,
+      "property_set_id": 0,
+      "transitions": [
+        {
+          "type": "attribute_name",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "attribute_value",
+          "named": true,
+          "state_id": 2
+        },
+        {
+          "type": "comment",
+          "named": true,
+          "state_id": 3
+        },
+        {
+          "type": "doctype",
+          "named": true,
+          "state_id": 4
+        },
+        {
+          "type": "erroneous_end_tag_name",
+          "named": true,
+          "state_id": 5
+        },
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 6
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 7
+        },
+        {
+          "type": "tag_name",
+          "named": true,
+          "state_id": 5
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 1,
+      "property_set_id": 1,
+      "transitions": [
+        {
+          "type": "attribute_name",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "attribute_value",
+          "named": true,
+          "state_id": 2
+        },
+        {
+          "type": "comment",
+          "named": true,
+          "state_id": 3
+        },
+        {
+          "type": "doctype",
+          "named": true,
+          "state_id": 4
+        },
+        {
+          "type": "erroneous_end_tag_name",
+          "named": true,
+          "state_id": 5
+        },
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 6
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 7
+        },
+        {
+          "type": "tag_name",
+          "named": true,
+          "state_id": 5
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 2,
+      "property_set_id": 2,
+      "transitions": [
+        {
+          "type": "attribute_name",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "attribute_value",
+          "named": true,
+          "state_id": 2
+        },
+        {
+          "type": "comment",
+          "named": true,
+          "state_id": 3
+        },
+        {
+          "type": "doctype",
+          "named": true,
+          "state_id": 4
+        },
+        {
+          "type": "erroneous_end_tag_name",
+          "named": true,
+          "state_id": 5
+        },
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 6
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 7
+        },
+        {
+          "type": "tag_name",
+          "named": true,
+          "state_id": 5
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 3,
+      "property_set_id": 3,
+      "transitions": [
+        {
+          "type": "attribute_name",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "attribute_value",
+          "named": true,
+          "state_id": 2
+        },
+        {
+          "type": "comment",
+          "named": true,
+          "state_id": 3
+        },
+        {
+          "type": "doctype",
+          "named": true,
+          "state_id": 4
+        },
+        {
+          "type": "erroneous_end_tag_name",
+          "named": true,
+          "state_id": 5
+        },
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 6
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 7
+        },
+        {
+          "type": "tag_name",
+          "named": true,
+          "state_id": 5
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 4,
+      "property_set_id": 4,
+      "transitions": [
+        {
+          "type": "attribute_name",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "attribute_value",
+          "named": true,
+          "state_id": 2
+        },
+        {
+          "type": "comment",
+          "named": true,
+          "state_id": 3
+        },
+        {
+          "type": "doctype",
+          "named": true,
+          "state_id": 4
+        },
+        {
+          "type": "erroneous_end_tag_name",
+          "named": true,
+          "state_id": 5
+        },
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 6
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 7
+        },
+        {
+          "type": "tag_name",
+          "named": true,
+          "state_id": 5
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 5,
+      "property_set_id": 5,
+      "transitions": [
+        {
+          "type": "attribute_name",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "attribute_value",
+          "named": true,
+          "state_id": 2
+        },
+        {
+          "type": "comment",
+          "named": true,
+          "state_id": 3
+        },
+        {
+          "type": "doctype",
+          "named": true,
+          "state_id": 4
+        },
+        {
+          "type": "erroneous_end_tag_name",
+          "named": true,
+          "state_id": 5
+        },
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 6
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 7
+        },
+        {
+          "type": "tag_name",
+          "named": true,
+          "state_id": 5
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 6,
+      "property_set_id": 6,
+      "transitions": [
+        {
+          "type": "attribute_name",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "attribute_value",
+          "named": true,
+          "state_id": 2
+        },
+        {
+          "type": "comment",
+          "named": true,
+          "state_id": 3
+        },
+        {
+          "type": "doctype",
+          "named": true,
+          "state_id": 4
+        },
+        {
+          "type": "erroneous_end_tag_name",
+          "named": true,
+          "state_id": 5
+        },
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 6
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 7
+        },
+        {
+          "type": "tag_name",
+          "named": true,
+          "state_id": 5
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 7,
+      "property_set_id": 7,
+      "transitions": [
+        {
+          "type": "attribute_name",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "attribute_value",
+          "named": true,
+          "state_id": 2
+        },
+        {
+          "type": "comment",
+          "named": true,
+          "state_id": 3
+        },
+        {
+          "type": "doctype",
+          "named": true,
+          "state_id": 4
+        },
+        {
+          "type": "erroneous_end_tag_name",
+          "named": true,
+          "state_id": 5
+        },
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 6
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 7
+        },
+        {
+          "type": "tag_name",
+          "named": true,
+          "state_id": 5
+        }
+      ],
+      "default_next_state_id": 0
+    }
+  ],
+  "property_sets": [
+    {},
+    {
+      "scope": "attribute"
+    },
+    {
+      "scope": "string"
+    },
+    {
+      "scope": "comment"
+    },
+    {
+      "scope": "constant"
+    },
+    {
+      "scope": "tag"
+    },
+    {
+      "injection-content": {
+        "args": [
+          {
+            "args": [],
+            "name": "this"
+          },
+          1
+        ],
+        "name": "child"
+      },
+      "injection-language": "javascript"
+    },
+    {
+      "injection-content": {
+        "args": [
+          {
+            "args": [],
+            "name": "this"
+          },
+          1
+        ],
+        "name": "child"
+      },
+      "injection-language": "css"
+    }
+  ]
+}

src/injections.json

@@ -0,0 +1,84 @@
+{
+  "states": [
+    {
+      "id": 0,
+      "property_set_id": 0,
+      "transitions": [
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 2
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 1,
+      "property_set_id": 1,
+      "transitions": [
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 2
+        }
+      ],
+      "default_next_state_id": 0
+    },
+    {
+      "id": 2,
+      "property_set_id": 2,
+      "transitions": [
+        {
+          "type": "script_element",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "style_element",
+          "named": true,
+          "state_id": 2
+        }
+      ],
+      "default_next_state_id": 0
+    }
+  ],
+  "property_sets": [
+    {},
+    {
+      "injection-content": {
+        "args": [
+          {
+            "args": [],
+            "name": "this"
+          },
+          1
+        ],
+        "name": "child"
+      },
+      "injection-language": "javascript"
+    },
+    {
+      "injection-content": {
+        "args": [
+          {
+            "args": [],
+            "name": "this"
+          },
+          1
+        ],
+        "name": "child"
+      },
+      "injection-language": "css"
+    }
+  ]
+}

src/scanner.cc

@@ -13,7 +13,8 @@ using std::string;
 
 enum TokenType {
   START_TAG_NAME,
-  START_RAW_TAG_NAME,
+  SCRIPT_START_TAG_NAME,
+  STYLE_START_TAG_NAME,
   END_TAG_NAME,
   ERRONEOUS_END_TAG_NAME,
   SELF_CLOSING_TAG_DELIMITER,
@@ -187,10 +188,16 @@ struct Scanner {
     if (tag_name.empty()) return false;
     Tag tag = Tag::for_name(tag_name);
     tags.push_back(tag);
-    if (tag.is_raw()) {
+    switch (tag.type) {
-      lexer->result_symbol = START_RAW_TAG_NAME;
+      case SCRIPT:
-    } else {
+        lexer->result_symbol = SCRIPT_START_TAG_NAME;
-      lexer->result_symbol = START_TAG_NAME;
+        break;
+      case STYLE:
+        lexer->result_symbol = STYLE_START_TAG_NAME;
+        break;
+      default:
+        lexer->result_symbol = START_TAG_NAME;
+        break;
     }
     return true;
   }

src/tag.h

@@ -329,10 +329,6 @@ struct Tag {
     return type < END_OF_VOID_TAGS;
   }
 
-  inline bool is_raw() const {
-    return type == SCRIPT || type == STYLE;
-  }
-
   inline bool can_contain(const Tag &tag) {
     TagType child = tag.type;
 

src/tree_sitter/parser.h

@@ -13,7 +13,7 @@ extern "C" {
 #define ts_builtin_sym_end 0
 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
 
-#ifndef TREE_SITTER_RUNTIME_H_
+#ifndef TREE_SITTER_API_H_
 typedef uint16_t TSSymbol;
 typedef struct TSLanguage TSLanguage;
 #endif