⬆️ tree-sitter-cli to 0.16

package.json

@@ -10,28 +10,19 @@
   "author": "Max Brunsfeld",
   "license": "MIT",
   "dependencies": {
-    "nan": "^2.10.0",
+    "nan": "^2.14.0",
     "prebuild-install": "^5.0.0"
   },
   "devDependencies": {
-    "nodemon": "^1.18.3",
     "prebuild": "^7.6.1",
-    "tree-sitter-cli": "^0.15.3"
+    "tree-sitter-cli": "^0.16.1"
   },
   "scripts": {
-    "build": "tree-sitter generate && node-gyp build",
     "install": "prebuild-install || node-gyp rebuild",
     "pre-build": "prebuild --all --strip --verbose",
     "pre-build:upload": "prebuild --upload-all",
     "test": "tree-sitter test && script/parse-examples.sh",
-    "test:watch": "nodemon --exec 'npm run build && npm run test' --ext js,txt,sh",
     "test-windows": "tree-sitter test"
   },
-  "nodemonConfig": {
-    "ignore": [
-      "build/",
-      "src/"
-    ]
-  },
   "repository": "https://github.com/tree-sitter/tree-sitter-bash"
 }

script/known-failures.txt

@@ -1,15 +1,15 @@
-examples/bash-it/plugins/available/git.plugin.bash
-examples/bash-it/plugins/available/extract.plugin.bash
-examples/bash-it/plugins/available/go.plugin.bash
-examples/bash-it/completion/available/svn.completion.bash
 examples/bash-it/completion/available/docker-compose.completion.bash
-examples/bash-it/completion/available/drush.completion.bash
 examples/bash-it/completion/available/docker-machine.completion.bash
-examples/bash-it/completion/available/git.completion.bash
 examples/bash-it/completion/available/docker.completion.bash
+examples/bash-it/completion/available/drush.completion.bash
+examples/bash-it/completion/available/git.completion.bash
+examples/bash-it/completion/available/svn.completion.bash
 examples/bash-it/completion/available/tmux.completion.bash
-examples/bash-it/themes/hawaii50/hawaii50.theme.bash
+examples/bash-it/plugins/available/extract.plugin.bash
+examples/bash-it/plugins/available/git.plugin.bash
+examples/bash-it/plugins/available/go.plugin.bash
 examples/bash-it/themes/colors.theme.bash
-examples/bash-it/themes/powerline/powerline.base.bash
-examples/bash-it/themes/nwinkler_random_colors/nwinkler_random_colors.theme.bash
 examples/bash-it/themes/doubletime/doubletime.theme.bash
+examples/bash-it/themes/hawaii50/hawaii50.theme.bash
+examples/bash-it/themes/nwinkler_random_colors/nwinkler_random_colors.theme.bash
+examples/bash-it/themes/powerline/powerline.base.bash

script/parse-examples.sh

@@ -1,15 +1,8 @@
 #!/bin/bash
 
-known_failures=$(cat script/known-failures.txt)
+known_failures="$(cat script/known-failures.txt)"
 
-example_files=$(find examples -name '*.sh' -or -name '*.bash')
+tree-sitter parse -q -t \
-
+  examples/**/*.bash \
-examples_to_parse=$(
+  examples/**/*.sh   \
-  for example in $example_files; do
+  $(for failure in $known_failures; do echo "!${failure}"; done)
-    if [[ ! $known_failures == *$example* ]]; then
-      echo $example
-    fi
-  done
-)
-
-tree-sitter parse $examples_to_parse -q -t

script/update-known-failures.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
-tree-sitter parse $(find examples/bash-it -name '*.bash' -or -name '*.sh') -q -t \
+tree-sitter parse -q -t   \
+  examples/**/*.sh        \
+  examples/**/*.bash      \
   | egrep 'ERROR|MISSING' \
-  | tee >(cut -d' ' -f1 > script/known-failures.txt)
+  | tee >(cut -d' ' -f1 | sort > script/known-failures.txt)

src/node-types.json

@@ -56,7 +56,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -131,7 +131,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -206,7 +206,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -329,7 +329,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -384,7 +384,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -450,8 +450,8 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -502,7 +502,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -761,6 +761,10 @@
           "type": "variable_assignment",
           "named": true
         },
+        {
+          "type": "variable_name",
+          "named": true
+        },
         {
           "type": "word",
           "named": true
@@ -857,7 +861,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -1023,7 +1027,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -1090,7 +1094,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -1145,7 +1149,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -1187,6 +1191,10 @@
           "type": "string_expansion",
           "named": true
         },
+        {
+          "type": "variable_name",
+          "named": true
+        },
         {
           "type": "word",
           "named": true
@@ -1200,7 +1208,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "compound_statement",
@@ -1241,8 +1249,8 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "heredoc_start",
@@ -1256,8 +1264,8 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -1308,7 +1316,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -1399,7 +1407,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -1477,8 +1485,8 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "command",
@@ -1500,8 +1508,8 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -1568,7 +1576,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -1646,8 +1654,8 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -1714,7 +1722,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -1880,7 +1888,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -1970,12 +1978,16 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "special_variable_name",
           "named": true
+        },
+        {
+          "type": "variable_name",
+          "named": true
         }
       ]
     }
@@ -2008,8 +2020,8 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "raw_string",
@@ -2028,7 +2040,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -2083,7 +2095,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -2165,8 +2177,8 @@
     "named": true,
     "fields": {},
     "children": {
-      "multiple": true,
+      "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -2233,7 +2245,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -2342,6 +2354,10 @@
           "type": "string_expansion",
           "named": true
         },
+        {
+          "type": "variable_name",
+          "named": true
+        },
         {
           "type": "word",
           "named": true
@@ -2355,7 +2371,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "ansii_c_string",
@@ -2418,7 +2434,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": false,
+      "required": true,
       "types": [
         {
           "type": "c_style_for_statement",
@@ -2499,180 +2515,44 @@
     "type": "\n",
     "named": false
   },
-  {
-    "type": "for",
-    "named": false
-  },
-  {
-    "type": "in",
-    "named": false
-  },
-  {
-    "type": "((",
-    "named": false
-  },
-  {
-    "type": "))",
-    "named": false
-  },
-  {
-    "type": ";",
-    "named": false
-  },
-  {
-    "type": "while",
-    "named": false
-  },
-  {
-    "type": "do",
-    "named": false
-  },
-  {
-    "type": "done",
-    "named": false
-  },
-  {
-    "type": "if",
-    "named": false
-  },
-  {
-    "type": "then",
-    "named": false
-  },
-  {
-    "type": "fi",
-    "named": false
-  },
-  {
-    "type": "elif",
-    "named": false
-  },
-  {
-    "type": "else",
-    "named": false
-  },
-  {
-    "type": "case",
-    "named": false
-  },
-  {
-    "type": "esac",
-    "named": false
-  },
-  {
-    "type": "|",
-    "named": false
-  },
-  {
-    "type": ")",
-    "named": false
-  },
-  {
-    "type": ";;",
-    "named": false
-  },
-  {
-    "type": "function",
-    "named": false
-  },
-  {
-    "type": "(",
-    "named": false
-  },
-  {
-    "type": "{",
-    "named": false
-  },
-  {
-    "type": "}",
-    "named": false
-  },
-  {
-    "type": "|&",
-    "named": false
-  },
-  {
-    "type": "&&",
-    "named": false
-  },
-  {
-    "type": "||",
-    "named": false
-  },
   {
     "type": "!",
     "named": false
   },
   {
-    "type": "[",
+    "type": "!=",
     "named": false
   },
   {
-    "type": "]",
+    "type": "\"",
     "named": false
   },
   {
-    "type": "[[",
+    "type": "#",
     "named": false
   },
   {
-    "type": "]]",
+    "type": "$",
     "named": false
   },
   {
-    "type": "declare",
+    "type": "$(",
     "named": false
   },
   {
-    "type": "typeset",
+    "type": "${",
     "named": false
   },
   {
-    "type": "export",
+    "type": "%",
     "named": false
   },
   {
-    "type": "readonly",
+    "type": "&",
     "named": false
   },
   {
-    "type": "local",
+    "type": "&&",
-    "named": false
-  },
-  {
-    "type": "unset",
-    "named": false
-  },
-  {
-    "type": "unsetenv",
-    "named": false
-  },
-  {
-    "type": "=~",
-    "named": false
-  },
-  {
-    "type": "==",
-    "named": false
-  },
-  {
-    "type": "=",
-    "named": false
-  },
-  {
-    "type": "+=",
-    "named": false
-  },
-  {
-    "type": "<",
-    "named": false
-  },
-  {
-    "type": ">",
-    "named": false
-  },
-  {
-    "type": ">>",
     "named": false
   },
   {
@@ -2683,12 +2563,80 @@
     "type": "&>>",
     "named": false
   },
+  {
+    "type": "(",
+    "named": false
+  },
+  {
+    "type": "((",
+    "named": false
+  },
+  {
+    "type": ")",
+    "named": false
+  },
+  {
+    "type": "))",
+    "named": false
+  },
+  {
+    "type": "+",
+    "named": false
+  },
+  {
+    "type": "++",
+    "named": false
+  },
+  {
+    "type": "+=",
+    "named": false
+  },
+  {
+    "type": "-",
+    "named": false
+  },
+  {
+    "type": "--",
+    "named": false
+  },
+  {
+    "type": "-=",
+    "named": false
+  },
+  {
+    "type": "/",
+    "named": false
+  },
+  {
+    "type": ":",
+    "named": false
+  },
+  {
+    "type": ":-",
+    "named": false
+  },
+  {
+    "type": ":?",
+    "named": false
+  },
+  {
+    "type": ";",
+    "named": false
+  },
+  {
+    "type": ";;",
+    "named": false
+  },
+  {
+    "type": "<",
+    "named": false
+  },
   {
     "type": "<&",
     "named": false
   },
   {
-    "type": ">&",
+    "type": "<(",
     "named": false
   },
   {
@@ -2703,92 +2651,28 @@
     "type": "<<<",
     "named": false
   },
-  {
-    "type": "!=",
-    "named": false
-  },
-  {
-    "type": "+",
-    "named": false
-  },
-  {
-    "type": "-",
-    "named": false
-  },
-  {
-    "type": "-=",
-    "named": false
-  },
   {
     "type": "<=",
     "named": false
   },
   {
-    "type": ">=",
+    "type": "=",
     "named": false
   },
   {
-    "type": "++",
+    "type": "==",
     "named": false
   },
   {
-    "type": "--",
+    "type": "=~",
     "named": false
   },
   {
-    "type": "$",
+    "type": ">",
     "named": false
   },
   {
-    "type": "\"",
+    "type": ">&",
-    "named": false
-  },
-  {
-    "type": "raw_string",
-    "named": true
-  },
-  {
-    "type": "ansii_c_string",
-    "named": true
-  },
-  {
-    "type": "#",
-    "named": false
-  },
-  {
-    "type": "${",
-    "named": false
-  },
-  {
-    "type": "/",
-    "named": false
-  },
-  {
-    "type": ":",
-    "named": false
-  },
-  {
-    "type": ":?",
-    "named": false
-  },
-  {
-    "type": ":-",
-    "named": false
-  },
-  {
-    "type": "%",
-    "named": false
-  },
-  {
-    "type": "$(",
-    "named": false
-  },
-  {
-    "type": "`",
-    "named": false
-  },
-  {
-    "type": "<(",
     "named": false
   },
   {
@@ -2796,31 +2680,103 @@
     "named": false
   },
   {
-    "type": "comment",
+    "type": ">=",
+    "named": false
+  },
+  {
+    "type": ">>",
+    "named": false
+  },
+  {
+    "type": "[",
+    "named": false
+  },
+  {
+    "type": "[[",
+    "named": false
+  },
+  {
+    "type": "]",
+    "named": false
+  },
+  {
+    "type": "]]",
+    "named": false
+  },
+  {
+    "type": "`",
+    "named": false
+  },
+  {
+    "type": "ansii_c_string",
     "named": true
   },
   {
-    "type": "*",
+    "type": "case",
     "named": false
   },
   {
-    "type": "@",
+    "type": "declare",
     "named": false
   },
   {
-    "type": "?",
+    "type": "do",
     "named": false
   },
   {
-    "type": "0",
+    "type": "done",
     "named": false
   },
   {
-    "type": "_",
+    "type": "elif",
     "named": false
   },
   {
-    "type": "word",
+    "type": "else",
+    "named": false
+  },
+  {
+    "type": "esac",
+    "named": false
+  },
+  {
+    "type": "export",
+    "named": false
+  },
+  {
+    "type": "fi",
+    "named": false
+  },
+  {
+    "type": "for",
+    "named": false
+  },
+  {
+    "type": "function",
+    "named": false
+  },
+  {
+    "type": "if",
+    "named": false
+  },
+  {
+    "type": "in",
+    "named": false
+  },
+  {
+    "type": "local",
+    "named": false
+  },
+  {
+    "type": "raw_string",
+    "named": true
+  },
+  {
+    "type": "readonly",
+    "named": false
+  },
+  {
+    "type": "special_variable_name",
     "named": true
   },
   {
@@ -2828,7 +2784,51 @@
     "named": true
   },
   {
-    "type": "&",
+    "type": "then",
+    "named": false
+  },
+  {
+    "type": "typeset",
+    "named": false
+  },
+  {
+    "type": "unset",
+    "named": false
+  },
+  {
+    "type": "unsetenv",
+    "named": false
+  },
+  {
+    "type": "variable_name",
+    "named": true
+  },
+  {
+    "type": "while",
+    "named": false
+  },
+  {
+    "type": "word",
+    "named": true
+  },
+  {
+    "type": "{",
+    "named": false
+  },
+  {
+    "type": "|",
+    "named": false
+  },
+  {
+    "type": "|&",
+    "named": false
+  },
+  {
+    "type": "||",
+    "named": false
+  },
+  {
+    "type": "}",
     "named": false
   }
 ]

src/tree_sitter/parser.h

@@ -45,7 +45,8 @@ struct TSLexer {
   void (*advance)(TSLexer *, bool);
   void (*mark_end)(TSLexer *);
   uint32_t (*get_column)(TSLexer *);
-  bool (*is_at_included_range_start)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
 };
 
 typedef enum {
@@ -114,6 +115,10 @@ struct TSLanguage {
   const TSFieldMapSlice *field_map_slices;
   const TSFieldMapEntry *field_map_entries;
   const char **field_names;
+  uint32_t large_state_count;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSSymbol *public_symbol_map;
 };
 
 /*
@@ -123,6 +128,7 @@ struct TSLanguage {
 #define START_LEXER()           \
   bool result = false;          \
   bool skip = false;            \
+  bool eof = false;             \
   int32_t lookahead;            \
   goto start;                   \
   next_state:                   \
@@ -155,6 +161,8 @@ struct TSLanguage {
  *  Parse Table Macros
  */
 
+#define SMALL_STATE(id) id - LARGE_STATE_COUNT
+
 #define STATE(id) id
 
 #define ACTIONS(id) id