Allow other redirects between a heredoc's delimiter and body

Fixes #16

.gitignore

@@ -2,3 +2,4 @@ node_modules
 build
 *.log
 package-lock.json
+test.sh

corpus/commands.txt

@@ -171,11 +171,13 @@ JS
 (program
   (command
     (command_name (word))
-    (heredoc_redirect (heredoc)))
+    (heredoc_redirect (heredoc_start))
+    (heredoc_body))
   (command
     (command_name (word))
     (word)
-    (heredoc_redirect (heredoc))))
+    (heredoc_redirect (heredoc_start))
+    (heredoc_body)))
 
 ===============================
 Heredocs with variables
@@ -192,7 +194,33 @@ exit
 (program
   (command
     (command_name (word))
-    (heredoc_redirect (heredoc
+    (heredoc_redirect (heredoc_start))
+    (heredoc_body
       (simple_expansion (variable_name))
-      (expansion (variable_name)))))
+      (expansion (variable_name))))
   (command (command_name (word))))
+
+=================================
+Heredocs with file redirects
+================================
+
+cat <<EOF > $tmpfile
+a $B ${C}
+EOF
+
+wc -l $tmpfile
+
+---
+
+(program
+  (command
+    (command_name (word))
+    (heredoc_redirect (heredoc_start))
+    (heredoc_body
+      (simple_expansion (variable_name))
+      (simple_expansion (variable_name))
+      (expansion (variable_name))))
+  (command
+    (command_name (word))
+    (word)
+    (simple_expansion (variable_name))))

grammar.js

@@ -24,10 +24,11 @@ module.exports = grammar({
   ],
 
   externals: $ => [
-    $._simple_heredoc,
+    $.heredoc_start,
-    $._heredoc_beginning,
+    $._simple_heredoc_body,
-    $._heredoc_middle,
+    $._heredoc_body_beginning,
-    $._heredoc_end,
+    $._heredoc_body_middle,
+    $._heredoc_body_end,
     $.file_descriptor,
     $._empty_value,
     $._concat,
@@ -87,7 +88,8 @@ module.exports = grammar({
         $.file_redirect,
         $.heredoc_redirect,
         $.herestring_redirect
-      ))
+      )),
+      optional($.heredoc_body)
     ),
 
     do_group: $ => seq(
@@ -207,7 +209,8 @@ module.exports = grammar({
         $.file_redirect,
         $.heredoc_redirect,
         $.herestring_redirect
-      ))
+      )),
+      optional($.heredoc_body)
     )),
 
     command_name: $ => $._expression,
@@ -288,19 +291,19 @@ module.exports = grammar({
 
     heredoc_redirect: $ => seq(
       choice('<<', '<<-'),
-      $.heredoc
+      $.heredoc_start
     ),
 
-    heredoc: $ => choice(
+    heredoc_body: $ => choice(
-      $._simple_heredoc,
+      $._simple_heredoc_body,
       seq(
-        $._heredoc_beginning,
+        $._heredoc_body_beginning,
         repeat(choice(
           $.expansion,
           $.simple_expansion,
-          $._heredoc_middle
+          $._heredoc_body_middle
         )),
-        $._heredoc_end
+        $._heredoc_body_end
       )
     ),
 

script/known-failures.txt

@@ -3,8 +3,6 @@ examples/bash-it/plugins/available/extract.plugin.bash
 examples/bash-it/plugins/available/z_autoenv.plugin.bash
 examples/bash-it/plugins/available/sshagent.plugin.bash
 examples/bash-it/plugins/available/go.plugin.bash
-examples/bash-it/plugins/available/proxy.plugin.bash
-examples/bash-it/plugins/available/browser.plugin.bash
 examples/bash-it/install.sh
 examples/bash-it/completion/available/terraform.completion.bash
 examples/bash-it/completion/available/go.completion.bash

script/update-known-failures.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
 tree-sitter parse $(find examples/bash-it -name '*.bash' -or -name '*.sh') -q -t \
-  | egrep 'ERROR|undefined' \
+  | egrep 'ERROR|MISSING' \
   | tee >(cut -d' ' -f1 > script/known-failures.txt)

src/grammar.json

@@ -157,6 +157,18 @@
               }
             ]
           }
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "heredoc_body"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         }
       ]
     },
@@ -760,6 +772,18 @@
                 }
               ]
             }
+          },
+          {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "heredoc_body"
+              },
+              {
+                "type": "BLANK"
+              }
+            ]
           }
         ]
       }
@@ -1180,23 +1204,23 @@
         },
         {
           "type": "SYMBOL",
-          "name": "heredoc"
+          "name": "heredoc_start"
         }
       ]
     },
-    "heredoc": {
+    "heredoc_body": {
       "type": "CHOICE",
       "members": [
         {
           "type": "SYMBOL",
-          "name": "_simple_heredoc"
+          "name": "_simple_heredoc_body"
         },
         {
           "type": "SEQ",
           "members": [
             {
               "type": "SYMBOL",
-              "name": "_heredoc_beginning"
+              "name": "_heredoc_body_beginning"
             },
             {
               "type": "REPEAT",
@@ -1213,14 +1237,14 @@
                   },
                   {
                     "type": "SYMBOL",
-                    "name": "_heredoc_middle"
+                    "name": "_heredoc_body_middle"
                   }
                 ]
               }
             },
             {
               "type": "SYMBOL",
-              "name": "_heredoc_end"
+              "name": "_heredoc_body_end"
             }
           ]
         }
@@ -1904,19 +1928,23 @@
   "externals": [
     {
       "type": "SYMBOL",
-      "name": "_simple_heredoc"
+      "name": "heredoc_start"
     },
     {
       "type": "SYMBOL",
-      "name": "_heredoc_beginning"
+      "name": "_simple_heredoc_body"
     },
     {
       "type": "SYMBOL",
-      "name": "_heredoc_middle"
+      "name": "_heredoc_body_beginning"
     },
     {
       "type": "SYMBOL",
-      "name": "_heredoc_end"
+      "name": "_heredoc_body_middle"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_heredoc_body_end"
     },
     {
       "type": "SYMBOL",

src/scanner.cc

@@ -7,10 +7,11 @@ namespace {
 using std::string;
 
 enum TokenType {
-  SIMPLE_HEREDOC,
+  HEREDOC_START,
-  HEREDOC_BEGINNING,
+  SIMPLE_HEREDOC_BODY,
-  HEREDOC_MIDDLE,
+  HEREDOC_BODY_BEGINNING,
-  HEREDOC_END,
+  HEREDOC_BODY_MIDDLE,
+  HEREDOC_BODY_END,
   FILE_DESCRIPTOR,
   EMPTY_VALUE,
   CONCAT,
@@ -68,6 +69,7 @@ struct Scanner {
           did_advance = true;
           advance(lexer);
           if (scan_heredoc_end_identifier(lexer)) {
+            heredoc_delimiter.clear();
             lexer->result_symbol = end_type;
             return true;
           }
@@ -111,26 +113,25 @@ struct Scanner {
       }
     }
 
-    if (valid_symbols[HEREDOC_MIDDLE] && !heredoc_delimiter.empty()) {
+    if (valid_symbols[HEREDOC_BODY_BEGINNING] && !heredoc_delimiter.empty()) {
-      return scan_heredoc_content(lexer, HEREDOC_MIDDLE, HEREDOC_END);
+      return scan_heredoc_content(lexer, HEREDOC_BODY_BEGINNING, SIMPLE_HEREDOC_BODY);
     }
 
-    if (valid_symbols[HEREDOC_BEGINNING]) {
+    if (valid_symbols[HEREDOC_BODY_MIDDLE] && !heredoc_delimiter.empty()) {
+      return scan_heredoc_content(lexer, HEREDOC_BODY_MIDDLE, HEREDOC_BODY_END);
+    }
+
+    if (valid_symbols[HEREDOC_START]) {
+      while (iswspace(lexer->lookahead)) skip(lexer);
+
+      lexer->result_symbol = HEREDOC_START;
       heredoc_delimiter.clear();
       while (iswalpha(lexer->lookahead)) {
         heredoc_delimiter += lexer->lookahead;
         advance(lexer);
       }
 
-      if (lexer->lookahead != '\n') return false;
+      return !heredoc_delimiter.empty();
-      advance(lexer);
-
-      if (scan_heredoc_end_identifier(lexer)) {
-        lexer->result_symbol = SIMPLE_HEREDOC;
-        return true;
-      }
-
-      return scan_heredoc_content(lexer, HEREDOC_BEGINNING, SIMPLE_HEREDOC);
     }
 
     if (valid_symbols[VARIABLE_NAME] || valid_symbols[FILE_DESCRIPTOR]) {