Add raw (quoted) heredocs

Fixes #31

corpus/commands.txt

@@ -181,3 +181,25 @@ EOF
 ---
 
 (program (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body))
+
+======================================
+Quoted Heredocs
+======================================
+
+cat << 'EOF'
+a=$b
+EOF
+
+cat << "EOF"
+a=$b
+EOF
+
+cat << \EOF
+EOF
+
+---
+
+(program
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body)
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body (simple_expansion (variable_name)))
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body))

grammar.js

@@ -318,6 +318,7 @@ module.exports = grammar({
         repeat(choice(
           $.expansion,
           $.simple_expansion,
+          $.command_substitution,
           $._heredoc_body_middle
         )),
         $._heredoc_body_end

script/known-failures.txt

@@ -10,7 +10,6 @@ examples/bash-it/completion/available/defaults.completion.bash
 examples/bash-it/completion/available/docker.completion.bash
 examples/bash-it/completion/available/tmux.completion.bash
 examples/bash-it/lib/preexec.bash
-examples/bash-it/lib/composure.bash
 examples/bash-it/themes/hawaii50/hawaii50.theme.bash
 examples/bash-it/themes/colors.theme.bash
 examples/bash-it/themes/morris/morris.theme.bash

src/grammar.json

@@ -1269,6 +1269,10 @@
                     "type": "SYMBOL",
                     "name": "simple_expansion"
                   },
+                  {
+                    "type": "SYMBOL",
+                    "name": "command_substitution"
+                  },
                   {
                     "type": "SYMBOL",
                     "name": "_heredoc_body_middle"

src/scanner.cc

@@ -32,14 +32,53 @@ struct Scanner {
   }
 
   unsigned serialize(char *buffer) {
-    if (heredoc_delimiter.size() >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) return 0;
-    heredoc_delimiter.copy(buffer, heredoc_delimiter.length());
-    return heredoc_delimiter.length();
+    if (heredoc_delimiter.length() + 2 >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) return 0;
+    buffer[0] = heredoc_is_raw;
+    buffer[1] = started_heredoc;
+    heredoc_delimiter.copy(&buffer[2], heredoc_delimiter.length());
+    return heredoc_delimiter.length() + 2;
   }
 
   void deserialize(const char *buffer, unsigned length) {
-    if (length == 0) heredoc_delimiter.clear();
-    else heredoc_delimiter.assign(buffer, buffer + length);
+    if (length == 0) {
+      heredoc_is_raw = false;
+      started_heredoc = false;
+      heredoc_delimiter.clear();
+    } else {
+      heredoc_is_raw = buffer[0];
+      started_heredoc = buffer[1];
+      heredoc_delimiter.assign(&buffer[2], &buffer[length]);
+    }
+  }
+
+  bool scan_heredoc_start(TSLexer *lexer) {
+    while (iswspace(lexer->lookahead)) skip(lexer);
+
+    lexer->result_symbol = HEREDOC_START;
+    heredoc_is_raw = lexer->lookahead == '\'';
+    started_heredoc = false;
+    heredoc_delimiter.clear();
+
+    if (lexer->lookahead == '\\') {
+      advance(lexer);
+    }
+
+    int32_t quote = 0;
+    if (heredoc_is_raw || lexer->lookahead == '"') {
+      quote = lexer->lookahead;
+      advance(lexer);
+    }
+
+    while (iswalpha(lexer->lookahead)) {
+      heredoc_delimiter += lexer->lookahead;
+      advance(lexer);
+    }
+
+    if (lexer->lookahead == quote) {
+      advance(lexer);
+    }
+
+    return !heredoc_delimiter.empty();
   }
 
   bool scan_heredoc_end_identifier(TSLexer *lexer) {
@@ -57,8 +96,15 @@ struct Scanner {
     for (;;) {
       switch (lexer->lookahead) {
         case '\0': {
-          lexer->result_symbol = end_type;
-          return did_advance;
+          if (did_advance) {
+            heredoc_is_raw = false;
+            started_heredoc = false;
+            heredoc_delimiter.clear();
+            lexer->result_symbol = end_type;
+            return true;
+          } else {
+            return false;
+          }
         }
 
         case '\\': {
@@ -69,14 +115,25 @@ struct Scanner {
         }
 
         case '$': {
-          lexer->result_symbol = middle_type;
-          return did_advance;
+          if (heredoc_is_raw) {
+            did_advance = true;
+            advance(lexer);
+            break;
+          } else if (did_advance) {
+            lexer->result_symbol = middle_type;
+            started_heredoc = true;
+            return true;
+          } else {
+            return false;
+          }
         }
 
         case '\n': {
           did_advance = true;
           advance(lexer);
           if (scan_heredoc_end_identifier(lexer)) {
+            heredoc_is_raw = false;
+            started_heredoc = false;
             heredoc_delimiter.clear();
             lexer->result_symbol = end_type;
             return true;
@@ -122,25 +179,16 @@ struct Scanner {
       }
     }
 
-    if (valid_symbols[HEREDOC_BODY_BEGINNING] && !heredoc_delimiter.empty()) {
+    if (valid_symbols[HEREDOC_BODY_BEGINNING] && !heredoc_delimiter.empty() && !started_heredoc) {
       return scan_heredoc_content(lexer, HEREDOC_BODY_BEGINNING, SIMPLE_HEREDOC_BODY);
     }
 
-    if (valid_symbols[HEREDOC_BODY_MIDDLE] && !heredoc_delimiter.empty()) {
+    if (valid_symbols[HEREDOC_BODY_MIDDLE] && !heredoc_delimiter.empty() && started_heredoc) {
       return scan_heredoc_content(lexer, HEREDOC_BODY_MIDDLE, HEREDOC_BODY_END);
     }
 
     if (valid_symbols[HEREDOC_START]) {
-      while (iswspace(lexer->lookahead)) skip(lexer);
-
-      lexer->result_symbol = HEREDOC_START;
-      heredoc_delimiter.clear();
-      while (iswalpha(lexer->lookahead)) {
-        heredoc_delimiter += lexer->lookahead;
-        advance(lexer);
-      }
-
-      return !heredoc_delimiter.empty();
+      return scan_heredoc_start(lexer);
     }
 
     if (valid_symbols[VARIABLE_NAME] || valid_symbols[FILE_DESCRIPTOR]) {
@@ -271,6 +319,8 @@ struct Scanner {
   }
 
   string heredoc_delimiter;
+  bool heredoc_is_raw;
+  bool started_heredoc;
   string current_leading_word;
 };