Always lex braces and brackets as separate tokens

The lexer needs to always return braces and brackets separately so that
the parser can decide if they are part of some construct like an array
subscript or a variable expansion. This means that there was no point
in moving bare-word tokenization into the external scanner. I've moved
it back into the normal scanner.

The tricky part is how to deal with the separate '}' and ']' tokens
in the case where they are *not* part of a subscript or an expansion.
For example, in code like `echo {hi}`, the syntax tree should still
clearly indicate that only *one* argument is being passed to echo.
For now, we achieve this by grouping the '{', hi, and '}' tokens into
a single `concatenation` node, which is a bit odd, but it's the best
we can do.

corpus/control_flow.txt

@@ -139,12 +139,12 @@ function do_yet_another_thing {
 
 (program
   (function_definition
-    (variable_name)
+    (word)
     (compound_statement (command (command_name (word)) (word))))
   (function_definition
-    (variable_name)
+    (word)
     (compound_statement (command (command_name (word)) (word))))
   (function_definition
-    (variable_name)
+    (word)
     (compound_statement (command (command_name (word)) (word)))
     (file_redirect (file_descriptor) (word))))

corpus/expressions.txt

@@ -15,6 +15,7 @@ echo a b
 Words with special characters
 =============================
 
+echo {o[k]}
 echo }}}
 echo ]]] ===
 [[ "35d8b" =~ ^[0-9a-fA-F] ]] || echo {nomatch}
@@ -22,11 +23,19 @@ echo ]]] ===
 ---
 
 (program
+  (command (command_name (word)) (concatenation (word) (word)))
   (command (command_name (word)) (word))
   (command (command_name (word)) (word) (word))
   (list
-    (command (command_name (word)) (string) (word) (word) (word))
+    (command
-    (command (command_name (word)) (word))))
+      (command_name (word))
+      (string)
+      (word)
+      (concatenation (word) (word))
+      (word))
+    (command
+      (command_name (word))
+      (concatenation (word)))))
 
 =============================
 Simple variable expansions
@@ -57,6 +66,32 @@ echo ${abc:
   (command (command_name (word)) (expansion (variable_name)))
   (command (command_name (word)) (expansion (variable_name))))
 
+===================================
+Variable expansions in strings
+===================================
+
+A="${A:-$B/c}"
+A="${b/$c/$d}"
+
+---
+
+(program
+  (variable_assignment
+    (variable_name)
+    (string
+      (expansion
+        (variable_name)
+        (concatenation (simple_expansion (variable_name)) (word)))))
+  (variable_assignment
+    (variable_name)
+    (string
+      (expansion
+        (variable_name)
+        (concatenation
+          (simple_expansion (variable_name))
+          (word)
+          (simple_expansion (variable_name)))))))
+
 ===================================
 Other variable expansion operators
 ===================================
@@ -248,8 +283,8 @@ a+=(foo "bar" $(baz))
 (program
   (variable_assignment (variable_name) (array))
   (variable_assignment (variable_name) (array (word) (word) (word)))
-  (command (command_name (word)) (expansion (variable_name)))
+  (command (command_name (word)) (expansion (subscript (variable_name) (word))))
-  (command (command_name (word)) (expansion (variable_name)))
+  (command (command_name (word)) (expansion (subscript (variable_name) (word))))
   (variable_assignment
     (subscript (variable_name) (simple_expansion (variable_name)))
     (word))

grammar.js

@@ -1,3 +1,16 @@
+const SPECIAL_CHARACTERS = [
+  "'", '"',
+  '<', '>',
+  '{', '}',
+  '\\[', '\\]',
+  '(', ')',
+  '`', '$',
+  '&', ';',
+  '\\',
+  '\\s',
+  '#',
+];
+
 module.exports = grammar({
   name: 'bash',
 
@@ -6,9 +19,8 @@ module.exports = grammar({
     $._terminator,
     $._expression,
     $._primary_expression,
-    $._variable_name,
     $._simple_variable_name,
-    $._simple_word,
+    $._special_variable_name,
   ],
 
   externals: $ => [
@@ -17,13 +29,12 @@ module.exports = grammar({
     $._heredoc_middle,
     $._heredoc_end,
     $.file_descriptor,
-    $.word,
     $._empty_value,
     $._concat,
     $.variable_name, // Variable name followed by an operator like '=' or '+='
-    '\n',
-    ']',
     '}',
+    ']',
+    '\n',
   ],
 
   extras: $ => [
@@ -118,8 +129,8 @@ module.exports = grammar({
 
     function_definition: $ => seq(
       choice(
-        seq('function', $._simple_variable_name, optional(seq('(', ')'))),
+        seq('function', $.word, optional(seq('(', ')'))),
-        seq($._simple_variable_name, '(', ')')
+        seq($.word, '(', ')')
       ),
       $.compound_statement,
       optional($.file_redirect)
@@ -162,7 +173,8 @@ module.exports = grammar({
       repeat($._expression),
       repeat(choice(
         $.file_redirect,
-        $.heredoc_redirect
+        $.heredoc_redirect,
+        $.herestring_redirect
       ))
     )),
 
@@ -201,7 +213,9 @@ module.exports = grammar({
       $.variable_name,
       '[',
       $._expression,
-      ']'
+      optional($._concat),
+      ']',
+      optional($._concat)
     ),
 
     file_redirect: $ => prec.left(seq(
@@ -228,16 +242,21 @@ module.exports = grammar({
       )
     ),
 
+    herestring_redirect: $ => seq(
+      '<<<',
+      $._expression
+    ),
+
     // Expressions
 
     _expression: $ => choice(
       $.concatenation,
-      $._primary_expression
+      $._primary_expression,
+      alias(prec(-2, $._special_characters), $.word)
     ),
 
     _primary_expression: $ => choice(
       $.word,
-      $._simple_word,
       $.string,
       $.raw_string,
       $.expansion,
@@ -247,22 +266,36 @@ module.exports = grammar({
     ),
 
     concatenation: $ => prec(-1, seq(
-      $._primary_expression,
+      choice(
-      repeat1(seq($._concat, $._primary_expression))
+        $._primary_expression,
+        $._special_characters,
+      ),
+      repeat1(prec(-1, seq(
+        $._concat,
+        choice(
+          $._primary_expression,
+          $._special_characters,
+        )
+      ))),
     )),
 
+    _special_characters: $ => token(prec(-1, repeat1(choice('{', '}', '[', ']')))),
+
     string: $ => seq(
       '"',
-      repeat(choice(
+      repeat(seq(
-        $._string_content,
+        choice(
-        $.expansion,
+          $._string_content,
-        $.simple_expansion,
+          $.expansion,
-        $.command_substitution
+          $.simple_expansion,
+          $.command_substitution
+        ),
+        optional($._concat)
       )),
       '"'
     ),
 
-    _string_content: $ => /([^"`$]|\\.)*/,
+    _string_content: $ => /([^"`$]|\\.)+/,
 
     array: $ => seq(
       '(',
@@ -274,30 +307,37 @@ module.exports = grammar({
 
     simple_expansion: $ => seq(
       '$',
-      $._variable_name
+      choice($._simple_variable_name, $._special_variable_name)
     ),
 
     expansion: $ => seq(
       '${',
+      optional('#'),
       choice(
-        $._variable_name,
-        seq('#', $._variable_name),
-        seq('#', $._variable_name, '[', '@', ']'),
-        seq($._variable_name, '[', '@', ']'),
         seq(
-          $._variable_name,
+          $.variable_name,
-          choice(':', ':?', '=', ':-', '%', '/'),
+          '=',
-          optional(seq($._expression, optional($._concat)))
+          optional(seq(
-        )
+            $._expression
+          ))
+        ),
+        seq(
+          choice(
+            $.subscript,
+            $._simple_variable_name,
+            $._special_variable_name
+          ),
+          optional(seq(
+            choice(':', ':?', '=', ':-', '%', '/', '-'),
+            optional(seq(
+              $._expression
+            ))
+          ))
+        ),
       ),
       '}'
     ),
 
-    _variable_name: $ => choice(
-      $._simple_variable_name,
-      $.special_variable_name
-    ),
-
     command_substitution: $ => choice(
       seq('$(', $._statement, ')'),
       prec(1, seq('`', $._statement, '`'))
@@ -311,14 +351,20 @@ module.exports = grammar({
 
     comment: $ => token(prec(-1, /#.*/)),
 
-    _simple_variable_name: $ => alias($.identifier, $.variable_name),
+    _simple_variable_name: $ => alias(/\w+/, $.variable_name),
 
-    _simple_word: $ => alias($.identifier, $.word),
+    _special_variable_name: $ => alias(choice('*', '@', '?', '-', '$', '0', '_'), $.special_variable_name),
 
-    identifier: $ => /\w+/,
+    word: $ => token(repeat1(choice(
-
+      noneOf(...SPECIAL_CHARACTERS),
-    special_variable_name: $ => choice('*', '@', '#', '?', '-', '$', '!', '0', '_'),
+      seq('\\', noneOf('\\s'))
+    ))),
 
     _terminator: $ => choice(';', ';;', '\n', '&')
   }
 });
+
+function noneOf(...characters) {
+  const negatedString = characters.map(c => c == '\\' ? '\\\\' : c).join('')
+  return new RegExp('[^' + negatedString + ']')
+}

src/grammar.json

@@ -325,7 +325,7 @@
                 },
                 {
                   "type": "SYMBOL",
-                  "name": "_simple_variable_name"
+                  "name": "word"
                 },
                 {
                   "type": "CHOICE",
@@ -355,7 +355,7 @@
               "members": [
                 {
                   "type": "SYMBOL",
-                  "name": "_simple_variable_name"
+                  "name": "word"
                 },
                 {
                   "type": "STRING",
@@ -548,6 +548,10 @@
                 {
                   "type": "SYMBOL",
                   "name": "heredoc_redirect"
+                },
+                {
+                  "type": "SYMBOL",
+                  "name": "herestring_redirect"
                 }
               ]
             }
@@ -681,9 +685,33 @@
           "type": "SYMBOL",
           "name": "_expression"
         },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_concat"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
         {
           "type": "STRING",
           "value": "]"
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_concat"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         }
       ]
     },
@@ -809,6 +837,19 @@
         }
       ]
     },
+    "herestring_redirect": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "<<<"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "_expression"
+        }
+      ]
+    },
     "_expression": {
       "type": "CHOICE",
       "members": [
@@ -819,6 +860,19 @@
         {
           "type": "SYMBOL",
           "name": "_primary_expression"
+        },
+        {
+          "type": "ALIAS",
+          "content": {
+            "type": "PREC",
+            "value": -2,
+            "content": {
+              "type": "SYMBOL",
+              "name": "_special_characters"
+            }
+          },
+          "named": true,
+          "value": "word"
         }
       ]
     },
@@ -829,10 +883,6 @@
           "type": "SYMBOL",
           "name": "word"
         },
-        {
-          "type": "SYMBOL",
-          "name": "_simple_word"
-        },
         {
           "type": "SYMBOL",
           "name": "string"
@@ -866,28 +916,81 @@
         "type": "SEQ",
         "members": [
           {
-            "type": "SYMBOL",
+            "type": "CHOICE",
-            "name": "_primary_expression"
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "_primary_expression"
+              },
+              {
+                "type": "SYMBOL",
+                "name": "_special_characters"
+              }
+            ]
           },
           {
             "type": "REPEAT1",
             "content": {
-              "type": "SEQ",
+              "type": "PREC",
-              "members": [
+              "value": -1,
-                {
+              "content": {
-                  "type": "SYMBOL",
+                "type": "SEQ",
-                  "name": "_concat"
+                "members": [
-                },
+                  {
-                {
+                    "type": "SYMBOL",
-                  "type": "SYMBOL",
+                    "name": "_concat"
-                  "name": "_primary_expression"
+                  },
-                }
+                  {
-              ]
+                    "type": "CHOICE",
+                    "members": [
+                      {
+                        "type": "SYMBOL",
+                        "name": "_primary_expression"
+                      },
+                      {
+                        "type": "SYMBOL",
+                        "name": "_special_characters"
+                      }
+                    ]
+                  }
+                ]
+              }
             }
           }
         ]
       }
     },
+    "_special_characters": {
+      "type": "TOKEN",
+      "content": {
+        "type": "PREC",
+        "value": -1,
+        "content": {
+          "type": "REPEAT1",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "STRING",
+                "value": "{"
+              },
+              {
+                "type": "STRING",
+                "value": "}"
+              },
+              {
+                "type": "STRING",
+                "value": "["
+              },
+              {
+                "type": "STRING",
+                "value": "]"
+              }
+            ]
+          }
+        }
+      }
+    },
     "string": {
       "type": "SEQ",
       "members": [
@@ -898,23 +1001,40 @@
         {
           "type": "REPEAT",
           "content": {
-            "type": "CHOICE",
+            "type": "SEQ",
             "members": [
               {
-                "type": "SYMBOL",
+                "type": "CHOICE",
-                "name": "_string_content"
+                "members": [
+                  {
+                    "type": "SYMBOL",
+                    "name": "_string_content"
+                  },
+                  {
+                    "type": "SYMBOL",
+                    "name": "expansion"
+                  },
+                  {
+                    "type": "SYMBOL",
+                    "name": "simple_expansion"
+                  },
+                  {
+                    "type": "SYMBOL",
+                    "name": "command_substitution"
+                  }
+                ]
               },
               {
-                "type": "SYMBOL",
+                "type": "CHOICE",
-                "name": "expansion"
+                "members": [
-              },
+                  {
-              {
+                    "type": "SYMBOL",
-                "type": "SYMBOL",
+                    "name": "_concat"
-                "name": "simple_expansion"
+                  },
-              },
+                  {
-              {
+                    "type": "BLANK"
-                "type": "SYMBOL",
+                  }
-                "name": "command_substitution"
+                ]
               }
             ]
           }
@@ -927,7 +1047,7 @@
     },
     "_string_content": {
       "type": "PATTERN",
-      "value": "([^\"`$]|\\\\.)*"
+      "value": "([^\"`$]|\\\\.)+"
     },
     "array": {
       "type": "SEQ",
@@ -961,8 +1081,17 @@
           "value": "$"
         },
         {
-          "type": "SYMBOL",
+          "type": "CHOICE",
-          "name": "_variable_name"
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_simple_variable_name"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "_special_variable_name"
+            }
+          ]
         }
       ]
     },
@@ -977,103 +1106,27 @@
           "type": "CHOICE",
           "members": [
             {
-              "type": "SYMBOL",
+              "type": "STRING",
-              "name": "_variable_name"
+              "value": "#"
             },
             {
-              "type": "SEQ",
+              "type": "BLANK"
-              "members": [
+            }
-                {
+          ]
-                  "type": "STRING",
+        },
-                  "value": "#"
+        {
-                },
+          "type": "CHOICE",
-                {
+          "members": [
-                  "type": "SYMBOL",
-                  "name": "_variable_name"
-                }
-              ]
-            },
-            {
-              "type": "SEQ",
-              "members": [
-                {
-                  "type": "STRING",
-                  "value": "#"
-                },
-                {
-                  "type": "SYMBOL",
-                  "name": "_variable_name"
-                },
-                {
-                  "type": "STRING",
-                  "value": "["
-                },
-                {
-                  "type": "STRING",
-                  "value": "@"
-                },
-                {
-                  "type": "STRING",
-                  "value": "]"
-                }
-              ]
-            },
             {
               "type": "SEQ",
               "members": [
                 {
                   "type": "SYMBOL",
-                  "name": "_variable_name"
+                  "name": "variable_name"
                 },
                 {
                   "type": "STRING",
-                  "value": "["
+                  "value": "="
-                },
-                {
-                  "type": "STRING",
-                  "value": "@"
-                },
-                {
-                  "type": "STRING",
-                  "value": "]"
-                }
-              ]
-            },
-            {
-              "type": "SEQ",
-              "members": [
-                {
-                  "type": "SYMBOL",
-                  "name": "_variable_name"
-                },
-                {
-                  "type": "CHOICE",
-                  "members": [
-                    {
-                      "type": "STRING",
-                      "value": ":"
-                    },
-                    {
-                      "type": "STRING",
-                      "value": ":?"
-                    },
-                    {
-                      "type": "STRING",
-                      "value": "="
-                    },
-                    {
-                      "type": "STRING",
-                      "value": ":-"
-                    },
-                    {
-                      "type": "STRING",
-                      "value": "%"
-                    },
-                    {
-                      "type": "STRING",
-                      "value": "/"
-                    }
-                  ]
                 },
                 {
                   "type": "CHOICE",
@@ -1084,13 +1137,86 @@
                         {
                           "type": "SYMBOL",
                           "name": "_expression"
+                        }
+                      ]
+                    },
+                    {
+                      "type": "BLANK"
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "CHOICE",
+                  "members": [
+                    {
+                      "type": "SYMBOL",
+                      "name": "subscript"
+                    },
+                    {
+                      "type": "SYMBOL",
+                      "name": "_simple_variable_name"
+                    },
+                    {
+                      "type": "SYMBOL",
+                      "name": "_special_variable_name"
+                    }
+                  ]
+                },
+                {
+                  "type": "CHOICE",
+                  "members": [
+                    {
+                      "type": "SEQ",
+                      "members": [
+                        {
+                          "type": "CHOICE",
+                          "members": [
+                            {
+                              "type": "STRING",
+                              "value": ":"
+                            },
+                            {
+                              "type": "STRING",
+                              "value": ":?"
+                            },
+                            {
+                              "type": "STRING",
+                              "value": "="
+                            },
+                            {
+                              "type": "STRING",
+                              "value": ":-"
+                            },
+                            {
+                              "type": "STRING",
+                              "value": "%"
+                            },
+                            {
+                              "type": "STRING",
+                              "value": "/"
+                            },
+                            {
+                              "type": "STRING",
+                              "value": "-"
+                            }
+                          ]
                         },
                         {
                           "type": "CHOICE",
                           "members": [
                             {
-                              "type": "SYMBOL",
+                              "type": "SEQ",
-                              "name": "_concat"
+                              "members": [
+                                {
+                                  "type": "SYMBOL",
+                                  "name": "_expression"
+                                }
+                              ]
                             },
                             {
                               "type": "BLANK"
@@ -1114,19 +1240,6 @@
         }
       ]
     },
-    "_variable_name": {
-      "type": "CHOICE",
-      "members": [
-        {
-          "type": "SYMBOL",
-          "name": "_simple_variable_name"
-        },
-        {
-          "type": "SYMBOL",
-          "name": "special_variable_name"
-        }
-      ]
-    },
     "command_substitution": {
       "type": "CHOICE",
       "members": [
@@ -1210,65 +1323,77 @@
     "_simple_variable_name": {
       "type": "ALIAS",
       "content": {
-        "type": "SYMBOL",
+        "type": "PATTERN",
-        "name": "identifier"
+        "value": "\\w+"
       },
       "named": true,
       "value": "variable_name"
     },
-    "_simple_word": {
+    "_special_variable_name": {
       "type": "ALIAS",
       "content": {
-        "type": "SYMBOL",
+        "type": "CHOICE",
-        "name": "identifier"
+        "members": [
+          {
+            "type": "STRING",
+            "value": "*"
+          },
+          {
+            "type": "STRING",
+            "value": "@"
+          },
+          {
+            "type": "STRING",
+            "value": "?"
+          },
+          {
+            "type": "STRING",
+            "value": "-"
+          },
+          {
+            "type": "STRING",
+            "value": "$"
+          },
+          {
+            "type": "STRING",
+            "value": "0"
+          },
+          {
+            "type": "STRING",
+            "value": "_"
+          }
+        ]
       },
       "named": true,
-      "value": "word"
+      "value": "special_variable_name"
     },
-    "identifier": {
+    "word": {
-      "type": "PATTERN",
+      "type": "TOKEN",
-      "value": "\\w+"
+      "content": {
-    },
+        "type": "REPEAT1",
-    "special_variable_name": {
+        "content": {
-      "type": "CHOICE",
+          "type": "CHOICE",
-      "members": [
+          "members": [
-        {
+            {
-          "type": "STRING",
+              "type": "PATTERN",
-          "value": "*"
+              "value": "[^'\"<>{}\\[\\]()`$&;\\\\\\s#]"
-        },
+            },
-        {
+            {
-          "type": "STRING",
+              "type": "SEQ",
-          "value": "@"
+              "members": [
-        },
+                {
-        {
+                  "type": "STRING",
-          "type": "STRING",
+                  "value": "\\"
-          "value": "#"
+                },
-        },
+                {
-        {
+                  "type": "PATTERN",
-          "type": "STRING",
+                  "value": "[^\\s]"
-          "value": "?"
+                }
-        },
+              ]
-        {
+            }
-          "type": "STRING",
+          ]
-          "value": "-"
-        },
-        {
-          "type": "STRING",
-          "value": "$"
-        },
-        {
-          "type": "STRING",
-          "value": "!"
-        },
-        {
-          "type": "STRING",
-          "value": "0"
-        },
-        {
-          "type": "STRING",
-          "value": "_"
         }
-      ]
+      }
     },
     "_terminator": {
       "type": "CHOICE",
@@ -1336,10 +1461,6 @@
       "type": "SYMBOL",
       "name": "file_descriptor"
     },
-    {
-      "type": "SYMBOL",
-      "name": "word"
-    },
     {
       "type": "SYMBOL",
       "name": "_empty_value"
@@ -1354,7 +1475,7 @@
     },
     {
       "type": "STRING",
-      "value": "\n"
+      "value": "}"
     },
     {
       "type": "STRING",
@@ -1362,7 +1483,7 @@
     },
     {
       "type": "STRING",
-      "value": "}"
+      "value": "\n"
     }
   ],
   "inline": [
@@ -1370,8 +1491,7 @@
     "_terminator",
     "_expression",
     "_primary_expression",
-    "_variable_name",
     "_simple_variable_name",
-    "_simple_word"
+    "_special_variable_name"
   ]
 }

src/scanner.cc

@@ -12,13 +12,12 @@ enum TokenType {
   HEREDOC_MIDDLE,
   HEREDOC_END,
   FILE_DESCRIPTOR,
-  WORD,
   EMPTY_VALUE,
   CONCAT,
   VARIABLE_NAME,
-  NEWLINE,
-  CLOSING_BRACKET,
   CLOSING_BRACE,
+  CLOSING_BRACKET,
+  NEWLINE,
 };
 
 struct Scanner {
@@ -92,13 +91,12 @@ struct Scanner {
         lexer->lookahead == '<' ||
         lexer->lookahead == ')' ||
         lexer->lookahead == '(' ||
-        lexer->lookahead == '[' ||
-        lexer->lookahead == '|' ||
-        lexer->lookahead == ']' ||
-        lexer->lookahead == '}' ||
         lexer->lookahead == ';' ||
         lexer->lookahead == '&' ||
-        lexer->lookahead == '`'
+        lexer->lookahead == '`' ||
+        lexer->lookahead == 0 ||
+        (lexer->lookahead == '}' && valid_symbols[CLOSING_BRACE]) ||
+        (lexer->lookahead == ']' && valid_symbols[CLOSING_BRACKET])
       )) {
         lexer->result_symbol = CONCAT;
         return true;
@@ -134,9 +132,7 @@ struct Scanner {
       return scan_heredoc_content(lexer, HEREDOC_BEGINNING, SIMPLE_HEREDOC);
     }
 
-    if (valid_symbols[VARIABLE_NAME] || valid_symbols[FILE_DESCRIPTOR] || valid_symbols[WORD]) {
+    if (valid_symbols[VARIABLE_NAME] || valid_symbols[FILE_DESCRIPTOR]) {
-      unsigned length = 0;
-
       for (;;) {
         if (
           lexer->lookahead == ' ' ||
@@ -145,98 +141,62 @@ struct Scanner {
         ) {
           skip(lexer);
         } else if (lexer->lookahead == '\\') {
-          advance(lexer);
+          skip(lexer);
           if (lexer->lookahead == '\n') {
             skip(lexer);
           } else {
-            length++;
+            return false;
-            break;
           }
         } else {
           break;
         }
       }
 
-      bool is_numeric = iswdigit(lexer->lookahead);
+      bool is_number = true;
-      bool is_alphanumeric = iswalpha(lexer->lookahead);
+      if (iswdigit(lexer->lookahead)) {
+        advance(lexer);
+      } else if (iswalpha(lexer->lookahead) || lexer->lookahead == '_') {
+        is_number = false;
+        advance(lexer);
+      } else {
+        return false;
+      }
 
       for (;;) {
-        // These characters are not allowed in unquoted arguments
+        if (iswdigit(lexer->lookahead)) {
-        // or environment variable names
+          advance(lexer);
-        if (
+        } else if (iswalpha(lexer->lookahead) || lexer->lookahead == '_') {
-          lexer->lookahead == 0 ||
+          is_number = false;
-          lexer->lookahead == ';' ||
+          advance(lexer);
-          lexer->lookahead == '"' ||
+        } else {
-          lexer->lookahead == '(' ||
-          lexer->lookahead == ')' ||
-          lexer->lookahead == '\'' ||
-          lexer->lookahead == '&' ||
-          lexer->lookahead == '#' ||
-          lexer->lookahead == '`' ||
-          lexer->lookahead == '|' ||
-          lexer->lookahead == '$' ||
-          iswspace(lexer->lookahead)
-        ) break;
-
-        // Curly braces are not allowed in unquoted arguments within curly braces
-        // (e.g. inside of a variable expansion like `${key:arg}`).
-        if (
-          lexer->lookahead == '}' &&
-          valid_symbols[CLOSING_BRACE]
-        ) break;
-
-        // Square brackets are not allowed in unquoted arguments within square brackets
-        // (e.g. inside of an array subscript like `a[arg]`).
-        if (
-          lexer->lookahead == ']' &&
-          valid_symbols[CLOSING_BRACKET]
-        ) break;
-
-        // Numbers followed by '<' and '>' at the beginning of commands
-        // are parsed as file descriptors.
-        if (lexer->lookahead == '<' || lexer->lookahead == '>') {
-          if (is_numeric && valid_symbols[FILE_DESCRIPTOR]) {
-            lexer->result_symbol = FILE_DESCRIPTOR;
-            return true;
-          }
           break;
         }
-
-        if (!iswdigit(lexer->lookahead)) is_numeric = false;
-
-        if (!iswalnum(lexer->lookahead) && lexer->lookahead != '_') {
-
-          // Alphanumeric strings followed by '=', '[', or '+=' are treated
-          // as environment variable names.
-          if (is_alphanumeric && valid_symbols[VARIABLE_NAME] && length > 0) {
-            if (lexer->lookahead == '+') {
-              lexer->mark_end(lexer);
-              advance(lexer);
-              if (lexer->lookahead == '=') {
-                lexer->result_symbol = VARIABLE_NAME;
-                return true;
-              } else {
-                return false;
-              }
-            } else if (lexer->lookahead == '=' || lexer->lookahead == '[') {
-              lexer->result_symbol = VARIABLE_NAME;
-              return true;
-            }
-          }
-
-          is_alphanumeric = false;
-        }
-
-        advance(lexer);
-        length++;
       }
 
-      // Do not handle strings containing only letters, because those
+      if (is_number &&
-      // might be keywords. Let the normal lexer handle those.
+          valid_symbols[FILE_DESCRIPTOR] &&
-      if (length > 0 && valid_symbols[WORD] && !is_alphanumeric) {
+          (lexer->lookahead == '>' || lexer->lookahead == '<')) {
-        lexer->result_symbol = WORD;
+        lexer->result_symbol = FILE_DESCRIPTOR;
         return true;
       }
+
+      if (valid_symbols[VARIABLE_NAME]) {
+        if (lexer->lookahead == '+') {
+          lexer->mark_end(lexer);
+          advance(lexer);
+          if (lexer->lookahead == '=') {
+            lexer->result_symbol = VARIABLE_NAME;
+            return true;
+          } else {
+            return false;
+          }
+        } else if (lexer->lookahead == '=' || lexer->lookahead == '[') {
+          lexer->result_symbol = VARIABLE_NAME;
+          return true;
+        }
+      }
+
+      return false;
     }
 
     return false;