shadowfacts
/
tree-sitter-bash
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Handle words concatenated with strings and expansions

This commit is contained in:
Max Brunsfeld 2017-07-17 10:19:35 -07:00
parent 403361626d
commit dce4ee6937
14 changed files with 35268 additions and 34864 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
corpus/commands.txt
View File

@ -7,7 +7,7 @@ whoami
---
(program
(command (command_name)))
(command (command_name (word))))
===============================
Commands with arguments
@ -19,8 +19,8 @@ git diff --word-diff=color -- file1.txt file2.txt
---
(program
(command (command_name) (word))
(command (command_name) (word) (word) (word) (word) (word)))
(command (command_name (word)) (word))
(command (command_name (word)) (word) (word) (word) (word) (word)))
===============================
Commands with quoted arguments
@ -32,10 +32,10 @@ echo 'hi'
---
(program
(command (command_name) (string
(command_substitution (command (command_name)))
(command_substitution (command (command_name)))))
(command (command_name) (raw_string)))
(command (command_name (word)) (string
(command_substitution (command (command_name (word))))
(command_substitution (command (command_name (word))))))
(command (command_name (word)) (raw_string)))
===============================
Quoted command names
@ -46,7 +46,9 @@ Quoted command names
---
(program
(command (string (simple_expansion (variable_name)) (simple_expansion (variable_name))) (word)))
(command
(command_name (string (simple_expansion (variable_name)) (simple_expansion (variable_name))))
(word)))
===============================
Commands with numeric arguments
@ -57,7 +59,7 @@ exit 1
---
(program
(command (command_name) (word)))
(command (command_name (word)) (word)))
===================================
Commands with environment variables
@ -71,11 +73,11 @@ VAR1=a VAR2="ok" git diff --word-diff=color
(program
(command
(environment_variable_assignment (variable_name) (word))
(command_name))
(command_name (word)))
(command
(environment_variable_assignment (variable_name) (word))
(environment_variable_assignment (variable_name) (string))
(command_name)
(command_name (word))
(word)
(word)))
@ -90,7 +92,7 @@ VAR2= echo
(program
(environment_variable_assignment (variable_name))
(command (environment_variable_assignment (variable_name)) (command_name)))
(command (environment_variable_assignment (variable_name)) (command_name (word))))
===================================
Pipelines
@ -103,11 +105,11 @@ cat foo | grep -v bar
(program
(pipeline
(command (command_name))
(command (command_name)))
(command (command_name (word)))
(command (command_name (word))))
(pipeline
(command (command_name) (word))
(command (command_name) (word) (word))))
(command (command_name (word)) (word))
(command (command_name (word)) (word) (word))))
===================================
Lists
@ -121,13 +123,13 @@ a | b && c && d; d e f || e g
(list
(list
(pipeline
(command (command_name))
(command (command_name)))
(command (command_name)))
(command (command_name)))
(command (command_name (word)))
(command (command_name (word))))
(command (command_name (word))))
(command (command_name (word))))
(list
(command (command_name) (word) (word))
(command (command_name) (word))))
(command (command_name (word)) (word) (word))
(command (command_name (word)) (word))))
===============================
File redirects
@ -141,16 +143,16 @@ cat a b > /dev/null
(program
(command
(command_name)
(command_name (word))
(file_redirect (word)))
(command
(command_name)
(command_name (word))
(word)
(word)
(file_redirect (word)))
(command
(file_redirect (file_descriptor) (word))
(command_name)))
(command_name (word))))
===============================
Heredoc redirects
@ -168,10 +170,10 @@ JS
(program
(command
(command_name)
(command_name (word))
(heredoc_redirect (heredoc)))
(command
(command_name)
(command_name (word))
(word)
(heredoc_redirect (heredoc))))
@ -189,8 +191,8 @@ exit
(program
(command
(command_name)
(command_name (word))
(heredoc_redirect (heredoc
(simple_expansion (variable_name))
(expansion (variable_name)))))
(command (command_name)))
(command (command_name (word))))

42
corpus/control_flow.txt
View File

@ -11,16 +11,16 @@ done
(program
(while_statement
(command (command_name) (word))
(command (command_name (word)) (word))
(do_group
(command (command_name) (word))
(command (command_name) (word)))))
(command (command_name (word)) (word))
(command (command_name (word)) (word)))))
====================================
For statements
====================================
for a in $(seq 1 10); do
for a in 1 2 $(seq 5 10); do
echo $a
done
@ -29,9 +29,11 @@ done
(program
(for_statement
(word)
(command (command_substitution (command (command_name) (word) (word))))
(word)
(word)
(command_substitution (command (command_name (word)) (word) (word)))
(do_group
(command (command_name) (simple_expansion (variable_name))))))
(command (command_name (word)) (simple_expansion (variable_name))))))
====================================
If statements
@ -50,16 +52,16 @@ fi
(program
(if_statement
(pipeline
(command (command_name) (word))
(command (command_name) (word) (word)))
(command (command_name) (word))
(command (command_name (word)) (word))
(command (command_name (word)) (word) (word)))
(command (command_name (word)) (word))
(elif_clause
(pipeline
(command (command_name) (word))
(command (command_name) (word) (word)))
(command (command_name) (word)))
(command (command_name (word)) (word))
(command (command_name (word)) (word) (word)))
(command (command_name (word)) (word)))
(else_clause
(command (command_name)))))
(command (command_name (word))))))
====================================
If statements with conditional expressions
@ -74,10 +76,10 @@ fi
(program
(if_statement
(bracket_command
(string (command_substitution (command (command_name))))
(string (command_substitution (command (command_name (word)))))
(word)
(raw_string))
(command (command_name) (word))))
(command (command_name (word)) (word))))
====================================
Case statements
@ -98,9 +100,9 @@ esac
(program
(case_statement (string)
(case_item (word)
(command (command_name) (word)))
(command (command_name (word)) (word)))
(case_item (word)
(command (command_name) (word)))))
(command (command_name (word)) (word)))))
===============================
Subshells
@ -113,7 +115,7 @@ Subshells
---
(program
(subshell (command (command_name) (word))))
(subshell (command (command_name (word)) (word))))
===============================
Function definitions
@ -130,5 +132,5 @@ function do_something_else() {
---
(program
(function_definition (command_name) (compound_statement (command (command_name) (word))))
(function_definition (command_name) (compound_statement (command (command_name) (word)))))
(function_definition (word) (compound_statement (command (command_name (word)) (word))))
(function_definition (word) (compound_statement (command (command_name (word)) (word)))))

56
corpus/expressions.txt
View File

@ -8,8 +8,8 @@ echo a b
---
(program
(command (command_name) (word))
(command (command_name) (word) (word)))
(command (command_name (word)) (word))
(command (command_name (word)) (word) (word)))
=============================
Simple variable expansions
@ -20,7 +20,7 @@ echo $abc
---
(program
(command (command_name) (simple_expansion (variable_name))))
(command (command_name (word)) (simple_expansion (variable_name))))
=============================
Variable expansions
@ -32,8 +32,8 @@ echo ${abc:-def}
---
(program
(command (command_name) (expansion (variable_name)))
(command (command_name) (expansion (variable_name) (word))))
(command (command_name (word)) (expansion (variable_name)))
(command (command_name (word)) (expansion (variable_name) (word))))
===================================
Other variable expansion operators
@ -45,7 +45,7 @@ cat ${BAR} ${ABC=def} ${GHI:?jkl}
(program
(command
(command_name)
(command_name (word))
(expansion (variable_name))
(expansion (variable_name) (word))
(expansion (variable_name) (word))))
@ -61,14 +61,14 @@ echo $(echo $(echo hi))
(program
(command
(command_name)
(command_substitution (command (command_name) (word))))
(command_name (word))
(command_substitution (command (command_name (word)) (word))))
(command
(command_name)
(command_name (word))
(command_substitution (command
(command_name)
(command_name (word))
(command_substitution (command
(command_name)
(command_name (word))
(word)))))))
=============================
@ -82,16 +82,16 @@ echo abc > >(wc -c)
(program
(command
(command_name)
(command_name (word))
(word)
(process_substitution (list
(command (command_name) (word))
(command (command_name) (word)))))
(command (command_name (word)) (word))
(command (command_name (word)) (word)))))
(command
(command_name)
(command_name (word))
(word)
(file_redirect (process_substitution
(command (command_name) (word))))))
(command (command_name (word)) (word))))))
=============================
Single quoted strings
@ -102,7 +102,7 @@ echo 'a b' 'c d'
---
(program
(command (command_name) (raw_string) (raw_string)))
(command (command_name (word)) (raw_string) (raw_string)))
=============================
Double quoted strings
@ -114,10 +114,10 @@ echo "a ${b} c" "d $e"
---
(program
(command (command_name)
(command (command_name (word))
(string)
(string))
(command (command_name)
(command (command_name (word))
(string (expansion (variable_name)))
(string (simple_expansion (variable_name)))))
@ -131,11 +131,12 @@ find "`dirname $file`" -name "$base"'*'
(program
(command
(command_name)
(string (command_substitution (command (command_name) (simple_expansion (variable_name)))))
(command_name (word))
(string (command_substitution (command (command_name (word)) (simple_expansion (variable_name)))))
(word)
(string (simple_expansion (variable_name)))
(raw_string)))
(concatenation
(string (simple_expansion (variable_name)))
(raw_string))))
=========================================
Arrays and array expansions
@ -147,10 +148,15 @@ b=(1 2 3)
echo ${a[@]}
echo ${#b[@]}
a[$i]=50
---
(program
(environment_variable_assignment (variable_name) (array))
(environment_variable_assignment (variable_name) (array (word) (word) (word)))
(command (command_name) (expansion (variable_name)))
(command (command_name) (expansion (variable_name))))
(command (command_name (word)) (expansion (variable_name)))
(command (command_name (word)) (expansion (variable_name)))
(environment_variable_assignment
(subscript (variable_name) (simple_expansion (variable_name)))
(word)))

4
corpus/programs.txt
View File

@ -28,7 +28,7 @@ f=g \
(program
(command
(command_name)
(command_name (word))
(word)
(word))
(command
@ -38,5 +38,5 @@ f=g \
(environment_variable_assignment
(variable_name)
(word))
(command_name)
(command_name (word))
(word)))

165
examples/clean-old.sh vendored Executable file
View File

@ -0,0 +1,165 @@
#!/bin/bash
# look for old 0.x cruft, and get rid of it.
# Should already be sitting in the npm folder.
# This doesn't have to be quite as cross-platform as install.sh.
# There are some bash-isms, because maintaining *two*
# fully-portable posix/bourne sh scripts is too much for
# one project with a sane maintainer.
# If readlink isn't available, then this is just too tricky.
# However, greadlink is fine, so Solaris can join the party, too.
readlink="readlink"
which $readlink >/dev/null 2>/dev/null
if [ $? -ne 0 ]; then
readlink="greadlink"
which $readlink >/dev/null 2>/dev/null
if [ $? -ne 0 ]; then
echo "Can't find the readlink or greadlink command. Aborting."
exit 1
fi
fi
if [ "x$npm_config_prefix" != "x" ]; then
PREFIXES=$npm_config_prefix
else
node="$NODE"
if [ "x$node" = "x" ]; then
node=`which node`
fi
if [ "x$node" = "x" ]; then
echo "Can't find node to determine prefix. Aborting."
exit 1
fi
PREFIX=`dirname $node`
PREFIX=`dirname $PREFIX`
echo "cleanup prefix=$PREFIX"
PREFIXES=$PREFIX
altprefix=`"$node" -e process.installPrefix`
if [ "x$altprefix" != "x" ] && [ "x$altprefix" != "x$PREFIX" ]; then
echo "altprefix=$altprefix"
PREFIXES="$PREFIX $altprefix"
fi
fi
# now prefix is where npm would be rooted by default
# go hunting.
packages=
for prefix in $PREFIXES; do
packages="$packages
"`ls "$prefix"/lib/node/.npm 2>/dev/null | grep -v .cache`
done
packages=`echo $packages`
filelist=()
fid=0
for prefix in $PREFIXES; do
# remove any links into the .npm dir, or links to
# version-named shims/symlinks.
for folder in share/man bin lib/node; do
find $prefix/$folder -type l | while read file; do
target=`$readlink $file | grep '/\.npm/'`
if [ "x$target" != "x" ]; then
# found one!
filelist[$fid]="$file"
let 'fid++'
# also remove any symlinks to this file.
base=`basename "$file"`
base=`echo "$base" | awk -F@ '{print $1}'`
if [ "x$base" != "x" ]; then
find "`dirname $file`" -type l -name "$base"'*' \
| while read l; do
target=`$readlink "$l" | grep "$base"`
if [ "x$target" != "x" ]; then
filelist[$fid]="$1"
let 'fid++'
fi
done
fi
fi
done
# Scour for shim files. These are relics of 0.2 npm installs.
# note: grep -r is not portable.
find $prefix/$folder -type f \
| xargs grep -sl '// generated by npm' \
| while read file; do
filelist[$fid]="$file"
let 'fid++'
done
done
# now remove the package modules, and the .npm folder itself.
if [ "x$packages" != "x" ]; then
for pkg in $packages; do
filelist[$fid]="$prefix/lib/node/$pkg"
let 'fid++'
for i in $prefix/lib/node/$pkg\@*; do
filelist[$fid]="$i"
let 'fid++'
done
done
fi
for folder in lib/node/.npm lib/npm share/npm; do
if [ -d $prefix/$folder ]; then
filelist[$fid]="$prefix/$folder"
let 'fid++'
fi
done
done
# now actually clean, but only if there's anything TO clean
if [ "${#filelist[@]}" -gt 0 ]; then
echo ""
echo "This script will find and eliminate any shims, symbolic"
echo "links, and other cruft that was installed by npm 0.x."
echo ""
if [ "x$packages" != "x" ]; then
echo "The following packages appear to have been installed with"
echo "an old version of npm, and will be removed forcibly:"
for pkg in $packages; do
echo " $pkg"
done
echo "Make a note of these. You may want to install them"
echo "with npm 1.0 when this process is completed."
echo ""
fi
OK=
if [ "x$1" = "x-y" ]; then
OK="yes"
fi
while [ "$OK" != "y" ] && [ "$OK" != "yes" ] && [ "$OK" != "no" ]; do
echo "Is this OK?"
echo " enter 'yes' or 'no'"
echo " or 'show' to see a list of files "
read OK
if [ "x$OK" = "xshow" ] || [ "x$OK" = "xs" ]; then
for i in "${filelist[@]}"; do
echo "$i"
done
fi
done
if [ "$OK" = "no" ]; then
echo "Aborting"
exit 1
fi
for i in "${filelist[@]}"; do
rm -rf "$i"
done
fi
echo ""
echo 'All clean!'
exit 0

119
examples/doc-build.sh vendored Executable file
View File

@ -0,0 +1,119 @@
#!/usr/bin/env bash
if [[ $DEBUG != "" ]]; then
set -x
fi
set -o errexit
set -o pipefail
if ! [ -x node_modules/.bin/marked-man ]; then
ps=0
if [ -f .building_marked-man ]; then
pid=$(cat .building_marked-man)
ps=$(ps -p $pid | grep $pid | wc -l) || true
fi
if [ -f .building_marked-man ] && [ $ps != 0 ]; then
while [ -f .building_marked-man ]; do
sleep 1
done
else
# a race to see which make process will be the one to install marked-man
echo $$ > .building_marked-man
sleep 1
if [ $(cat .building_marked-man) == $$ ]; then
make node_modules/.bin/marked-man
rm .building_marked-man
else
while [ -f .building_marked-man ]; do
sleep 1
done
fi
fi
fi
if ! [ -x node_modules/.bin/marked ]; then
ps=0
if [ -f .building_marked ]; then
pid=$(cat .building_marked)
ps=$(ps -p $pid | grep $pid | wc -l) || true
fi
if [ -f .building_marked ] && [ $ps != 0 ]; then
while [ -f .building_marked ]; do
sleep 1
done
else
# a race to see which make process will be the one to install marked
echo $$ > .building_marked
sleep 1
if [ $(cat .building_marked) == $$ ]; then
make node_modules/.bin/marked
rm .building_marked
else
while [ -f .building_marked ]; do
sleep 1
done
fi
fi
fi
src=$1
dest=$2
name=$(basename ${src%.*})
date=$(date -u +'%Y-%m-%d %H:%M:%S')
version=$(node cli.js -v)
mkdir -p $(dirname $dest)
html_replace_tokens () {
local url=$1
sed "s|@NAME@|$name|g" \
| sed "s|@DATE@|$date|g" \
| sed "s|@URL@|$url|g" \
| sed "s|@VERSION@|$version|g" \
| perl -p -e 's/<h1([^>]*)>([^\(]*\([0-9]\)) -- (.*?)<\/h1>/<h1>\2<\/h1> <p>\3<\/p>/g' \
| perl -p -e 's/npm-npm/npm/g' \
| perl -p -e 's/([^"-])(npm-)?README(?!\.html)(\(1\))?/\1<a href="..\/..\/doc\/README.html">README<\/a>/g' \
| perl -p -e 's/<title><a href="[^"]+README.html">README<\/a><\/title>/<title>README<\/title>/g' \
| perl -p -e 's/([^"-])([^\(> ]+)(\(1\))/\1<a href="..\/cli\/\2.html">\2\3<\/a>/g' \
| perl -p -e 's/([^"-])([^\(> ]+)(\(3\))/\1<a href="..\/api\/\2.html">\2\3<\/a>/g' \
| perl -p -e 's/([^"-])([^\(> ]+)(\(5\))/\1<a href="..\/files\/\2.html">\2\3<\/a>/g' \
| perl -p -e 's/([^"-])([^\(> ]+)(\(7\))/\1<a href="..\/misc\/\2.html">\2\3<\/a>/g' \
| perl -p -e 's/\([1357]\)<\/a><\/h1>/<\/a><\/h1>/g' \
| (if [ $(basename $(dirname $dest)) == "doc" ]; then
perl -p -e 's/ href="\.\.\// href="/g'
else
cat
fi)
}
man_replace_tokens () {
sed "s|@VERSION@|$version|g" \
| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(1\)/npm help \2/g' \
| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(([57])\)/npm help \3 \2/g' \
| perl -p -e 's/(npm\\-)?([a-zA-Z\\\.\-]*)\(3\)/npm apihelp \2/g' \
| perl -p -e 's/npm\(1\)/npm help npm/g' \
| perl -p -e 's/npm\(3\)/npm apihelp npm/g'
}
case $dest in
*.[1357])
./node_modules/.bin/marked-man --roff $src \
| man_replace_tokens > $dest
exit $?
;;
*.html)
url=${dest/html\//}
(cat html/dochead.html && \
cat $src | ./node_modules/.bin/marked &&
cat html/docfoot.html)\
| html_replace_tokens $url \
> $dest
exit $?
;;
*)
echo "Invalid destination type: $dest" >&2
exit 1
;;
esac

270
examples/install.sh vendored Executable file
View File

@ -0,0 +1,270 @@
#!/bin/sh
# A word about this shell script:
#
# It must work everywhere, including on systems that lack
# a /bin/bash, map 'sh' to ksh, ksh97, bash, ash, or zsh,
# and potentially have either a posix shell or bourne
# shell living at /bin/sh.
#
# See this helpful document on writing portable shell scripts:
# http://www.gnu.org/s/hello/manual/autoconf/Portable-Shell.html
#
# The only shell it won't ever work on is cmd.exe.
if [ "x$0" = "xsh" ]; then
# run as curl | sh
# on some systems, you can just do cat>npm-install.sh
# which is a bit cuter. But on others, &1 is already closed,
# so catting to another script file won't do anything.
# Follow Location: headers, and fail on errors
curl -f -L -s https://www.npmjs.org/install.sh > npm-install-$$.sh
ret=$?
if [ $ret -eq 0 ]; then
(exit 0)
else
rm npm-install-$$.sh
echo "Failed to download script" >&2
exit $ret
fi
sh npm-install-$$.sh
ret=$?
rm npm-install-$$.sh
exit $ret
fi
# See what "npm_config_*" things there are in the env,
# and make them permanent.
# If this fails, it's not such a big deal.
configures="`env | grep 'npm_config_' | sed -e 's|^npm_config_||g'`"
npm_config_loglevel="error"
if [ "x$npm_debug" = "x" ]; then
(exit 0)
else
echo "Running in debug mode."
echo "Note that this requires bash or zsh."
set -o xtrace
set -o pipefail
npm_config_loglevel="verbose"
fi
export npm_config_loglevel
# make sure that node exists
node=`which node 2>&1`
ret=$?
if [ $ret -eq 0 ] && [ -x "$node" ]; then
(exit 0)
else
echo "npm cannot be installed without node.js." >&2
echo "Install node first, and then try again." >&2
echo "" >&2
echo "Maybe node is installed, but not in the PATH?" >&2
echo "Note that running as sudo can change envs." >&2
echo ""
echo "PATH=$PATH" >&2
exit $ret
fi
# set the temp dir
TMP="${TMPDIR}"
if [ "x$TMP" = "x" ]; then
TMP="/tmp"
fi
TMP="${TMP}/npm.$$"
rm -rf "$TMP" || true
mkdir "$TMP"
if [ $? -ne 0 ]; then
echo "failed to mkdir $TMP" >&2
exit 1
fi
BACK="$PWD"
ret=0
tar="${TAR}"
if [ -z "$tar" ]; then
tar="${npm_config_tar}"
fi
if [ -z "$tar" ]; then
tar=`which tar 2>&1`
ret=$?
fi
if [ $ret -eq 0 ] && [ -x "$tar" ]; then
echo "tar=$tar"
echo "version:"
$tar --version
ret=$?
fi
if [ $ret -eq 0 ]; then
(exit 0)
else
echo "No suitable tar program found."
exit 1
fi
# Try to find a suitable make
# If the MAKE environment var is set, use that.
# otherwise, try to find gmake, and then make.
# If no make is found, then just execute the necessary commands.
# XXX For some reason, make is building all the docs every time. This
# is an annoying source of bugs. Figure out why this happens.
MAKE=NOMAKE
if [ "x$MAKE" = "x" ]; then
make=`which gmake 2>&1`
if [ $? -eq 0 ] && [ -x "$make" ]; then
(exit 0)
else
make=`which make 2>&1`
if [ $? -eq 0 ] && [ -x "$make" ]; then
(exit 0)
else
make=NOMAKE
fi
fi
else
make="$MAKE"
fi
if [ -x "$make" ]; then
(exit 0)
else
# echo "Installing without make. This may fail." >&2
make=NOMAKE
fi
# If there's no bash, then don't even try to clean
if [ -x "/bin/bash" ]; then
(exit 0)
else
clean="no"
fi
node_version=`"$node" --version 2>&1`
ret=$?
if [ $ret -ne 0 ]; then
echo "You need node to run this program." >&2
echo "node --version reports: $node_version" >&2
echo "with exit code = $ret" >&2
echo "Please install node before continuing." >&2
exit $ret
fi
t="${npm_install}"
if [ -z "$t" ]; then
# switch based on node version.
# note that we can only use strict sh-compatible patterns here.
case $node_version in
0.[01234567].* | v0.[01234567].*)
echo "You are using an outdated and unsupported version of" >&2
echo "node ($node_version). Please update node and try again." >&2
exit 99
;;
*)
echo "install npm@latest"
t="latest"
;;
esac
fi
# need to echo "" after, because Posix sed doesn't treat EOF
# as an implied end of line.
url=`(curl -SsL https://registry.npmjs.org/npm/$t; echo "") \
| sed -e 's/^.*tarball":"//' \
| sed -e 's/".*$//'`
ret=$?
if [ "x$url" = "x" ]; then
ret=125
# try without the -e arg to sed.
url=`(curl -SsL https://registry.npmjs.org/npm/$t; echo "") \
| sed 's/^.*tarball":"//' \
| sed 's/".*$//'`
ret=$?
if [ "x$url" = "x" ]; then
ret=125
fi
fi
if [ $ret -ne 0 ]; then
echo "Failed to get tarball url for npm/$t" >&2
exit $ret
fi
echo "fetching: $url" >&2
cd "$TMP" \
&& curl -SsL "$url" \
| $tar -xzf - \
&& cd "$TMP"/* \
&& (ver=`"$node" bin/read-package-json.js package.json version`
isnpm10=0
if [ $ret -eq 0 ]; then
if [ -d node_modules ]; then
if "$node" node_modules/semver/bin/semver -v "$ver" -r "1"
then
isnpm10=1
fi
else
if "$node" bin/semver -v "$ver" -r ">=1.0"; then
isnpm10=1
fi
fi
fi
ret=0
if [ $isnpm10 -eq 1 ] && [ -f "scripts/clean-old.sh" ]; then
if [ "x$skipclean" = "x" ]; then
(exit 0)
else
clean=no
fi
if [ "x$clean" = "xno" ] \
|| [ "x$clean" = "xn" ]; then
echo "Skipping 0.x cruft clean" >&2
ret=0
elif [ "x$clean" = "xy" ] || [ "x$clean" = "xyes" ]; then
NODE="$node" /bin/bash "scripts/clean-old.sh" "-y"
ret=$?
else
NODE="$node" /bin/bash "scripts/clean-old.sh" </dev/tty
ret=$?
fi
fi
if [ $ret -ne 0 ]; then
echo "Aborted 0.x cleanup. Exiting." >&2
exit $ret
fi) \
&& (if [ "x$configures" = "x" ]; then
(exit 0)
else
echo "./configure $configures"
echo "$configures" > npmrc
fi) \
&& (if [ "$make" = "NOMAKE" ]; then
(exit 0)
elif "$make" uninstall install; then
(exit 0)
else
make="NOMAKE"
fi
if [ "$make" = "NOMAKE" ]; then
"$node" cli.js rm npm -gf
"$node" cli.js install -gf
fi) \
&& cd "$BACK" \
&& rm -rf "$TMP" \
&& echo "It worked"
ret=$?
if [ $ret -ne 0 ]; then
echo "It failed" >&2
fi
exit $ret

36
examples/release.sh vendored Normal file
View File

@ -0,0 +1,36 @@
#!/bin/bash
# script for creating a zip and tarball for inclusion in node
unset CDPATH
set -e
rm -rf release *.tgz || true
mkdir release
node ./cli.js pack --loglevel error >/dev/null
mv *.tgz release
cd release
tar xzf *.tgz
mkdir node_modules
mv package node_modules/npm
# make the zip for windows users
cp node_modules/npm/bin/*.cmd .
zipname=npm-$(node ../cli.js -v).zip
zip -q -9 -r -X "$zipname" *.cmd node_modules
# make the tar for node's deps
cd node_modules
tarname=npm-$(node ../../cli.js -v).tgz
tar czf "$tarname" npm
cd ..
mv "node_modules/$tarname" .
rm -rf *.cmd
rm -rf node_modules
echo "release/$tarname"
echo "release/$zipname"

26
examples/relocate.sh vendored Executable file
View File

@ -0,0 +1,26 @@
#!/bin/bash
# Change the cli shebang to point at the specified node
# Useful for when the program is moved around after install.
# Also used by the default 'make install' in node to point
# npm at the newly installed node, rather than the first one
# in the PATH, which would be the default otherwise.
# bash /path/to/npm/scripts/relocate.sh $nodepath
# If $nodepath is blank, then it'll use /usr/bin/env
dir="$(dirname "$(dirname "$0")")"
cli="$dir"/bin/npm-cli.js
tmp="$cli".tmp
node="$1"
if [ "x$node" = "x" ]; then
node="/usr/bin/env node"
fi
node="#!$node"
sed -e 1d "$cli" > "$tmp"
echo "$node" > "$cli"
cat "$tmp" >> "$cli"
rm "$tmp"
chmod ogu+x $cli

9
examples/update-authors.sh vendored Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
git log --reverse --format='%aN <%aE>' | perl -wnE '
BEGIN {
say "# Authors sorted by whether or not they\x27re me";
}
print $seen{$_} = $_ unless $seen{$_}
' > AUTHORS

115
grammar.js
View File

@ -1,3 +1,14 @@
const SPECIAL_CHARACTERS = [
"'", '"',
'<', '>',
'{', '}',
'(', ')',
'`', '$',
'&', ';',
'\\',
'\\s',
];
module.exports = grammar({
name: 'bash',
@ -5,7 +16,7 @@ module.exports = grammar({
$._statement,
$._terminator,
$._expression,
$._variable_name
$._variable_name,
],
externals: $ => [
@ -15,7 +26,9 @@ module.exports = grammar({
$._heredoc_end,
$.file_descriptor,
$._empty_value,
'#'
$._concat,
$.variable_name,
'\n',
],
extras: $ => [
@ -51,7 +64,8 @@ module.exports = grammar({
'for',
$.word,
'in',
$._terminated_statement,
repeat1($._expression),
$._terminator,
$.do_group
),
@ -101,6 +115,7 @@ module.exports = grammar({
case_item: $ => seq(
$._expression,
repeat(seq('|', $._expression)),
')',
repeat($._terminated_statement),
';;'
@ -108,7 +123,7 @@ module.exports = grammar({
function_definition: $ => seq(
optional('function'),
rename($.leading_word, 'command_name'),
$.word,
'(',
')',
$.compound_statement
@ -123,6 +138,8 @@ module.exports = grammar({
subshell: $ => seq(
'(',
repeat($._terminated_statement),
$._statement,
optional($._terminator),
')'
),
@ -150,37 +167,41 @@ module.exports = grammar({
$.environment_variable_assignment,
$.file_redirect
)),
choice(
rename(choice($.leading_word), 'command_name'),
':',
$.string,
$.raw_string,
$.command_substitution
),
optional(seq(
/\s+/,
repeat($._expression)
)),
$.command_name,
repeat($._expression),
repeat(choice(
$.file_redirect,
$.heredoc_redirect
))
)),
command_name: $ => $._expression,
environment_variable_assignment: $ => seq(
rename($.leading_word, 'variable_name'),
choice(
$.variable_name,
$.subscript
),
'=',
choice(
$._expression,
$.array,
$._empty_value
)
),
file_redirect: $ => seq(
subscript: $ => seq(
$.variable_name,
'[',
$._expression,
']'
),
file_redirect: $ => prec.left(seq(
optional($.file_descriptor),
choice('<', '>', '>>', '&>', '&>>', '<&', '>&'),
$._expression
),
)),
heredoc_redirect: $ => seq(
choice('<<', '<<-'),
@ -205,14 +226,38 @@ module.exports = grammar({
_expression: $ => choice(
$.word,
$.string,
$.array,
$.raw_string,
$.expansion,
$.simple_expansion,
$.command_substitution,
$.process_substitution
$.process_substitution,
$.concatenation
),
concatenation: $ => prec(-1, seq(
choice(
$.word,
$.string,
$.raw_string,
$.expansion,
$.simple_expansion,
$.command_substitution,
$.process_substitution
),
repeat1(seq(
$._concat,
choice(
$.word,
$.string,
$.raw_string,
$.expansion,
$.simple_expansion,
$.command_substitution,
$.process_substitution
)
))
)),
string: $ => seq(
'"',
repeat(choice(
@ -234,10 +279,7 @@ module.exports = grammar({
simple_expansion: $ => seq(
'$',
choice(
rename($.simple_variable_name, 'variable_name'),
$.special_variable_name
)
$._variable_name
),
expansion: $ => seq(
@ -245,9 +287,11 @@ module.exports = grammar({
choice(
$._variable_name,
seq('#', $._variable_name),
seq('#', $._variable_name, '[', '@', ']'),
seq($._variable_name, '[', '@', ']'),
seq(
$._variable_name,
choice(':', ':?', '=', ':-'),
choice(':', ':?', '=', ':-', '%', '/'),
$._expression
)
),
@ -255,7 +299,7 @@ module.exports = grammar({
),
_variable_name: $ => choice(
rename($.leading_word, 'variable_name'),
rename($.simple_variable_name, 'variable_name'),
$.special_variable_name
),
@ -265,22 +309,27 @@ module.exports = grammar({
),
process_substitution: $ => seq(
choice('<', '>'),
'(',
choice('<(', '>('),
$._statement,
')'
),
leading_word: $ => /[^`"\\\s#=|;:{}()]+/,
word: $ => token(repeat1(choice(
noneOf('#', ...SPECIAL_CHARACTERS),
seq('\\', noneOf('\\s'))
))),
word: $ => /[^"`#\\\s$<>{}&;()]+/,
comment: $ => /#.*/,
comment: $ => token(prec(-1, /#.*/)),
simple_variable_name: $ => /\w+/,
special_variable_name: $ => choice('*', '@', '#', '?', '-', '$', '!', '0', '_'),
_terminator: $ => choice(';', ';;', '\n', '&'),
_terminator: $ => choice(';', ';;', '\n', '&')
}
});
function noneOf(...characters) {
const negatedString = characters.map(c => c == '\\' ? '\\\\' : c).join('')
return new RegExp('[^' + negatedString + ']')
}

473
src/grammar.json vendored
View File

@ -85,9 +85,16 @@
"type": "STRING",
"value": "in"
},
{
"type": "REPEAT1",
"content": {
"type": "SYMBOL",
"name": "_expression"
}
},
{
"type": "SYMBOL",
"name": "_terminated_statement"
"name": "_terminator"
},
{
"type": "SYMBOL",
@ -270,6 +277,22 @@
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "|"
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
},
{
"type": "STRING",
"value": ")"
@ -303,12 +326,8 @@
]
},
{
"type": "RENAME",
"content": {
"type": "SYMBOL",
"name": "leading_word"
},
"value": "command_name"
"type": "SYMBOL",
"name": "word"
},
{
"type": "STRING",
@ -358,6 +377,22 @@
"name": "_terminated_statement"
}
},
{
"type": "SYMBOL",
"name": "_statement"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_terminator"
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": ")"
@ -492,62 +527,15 @@
}
},
{
"type": "CHOICE",
"members": [
{
"type": "RENAME",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "leading_word"
}
]
},
"value": "command_name"
},
{
"type": "STRING",
"value": ":"
},
{
"type": "SYMBOL",
"name": "string"
},
{
"type": "SYMBOL",
"name": "raw_string"
},
{
"type": "SYMBOL",
"name": "command_substitution"
}
]
"type": "SYMBOL",
"name": "command_name"
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "PATTERN",
"value": "\\s+"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_expression"
}
}
]
},
{
"type": "BLANK"
}
]
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_expression"
}
},
{
"type": "REPEAT",
@ -568,16 +556,25 @@
]
}
},
"command_name": {
"type": "SYMBOL",
"name": "_expression"
},
"environment_variable_assignment": {
"type": "SEQ",
"members": [
{
"type": "RENAME",
"content": {
"type": "SYMBOL",
"name": "leading_word"
},
"value": "variable_name"
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "variable_name"
},
{
"type": "SYMBOL",
"name": "subscript"
}
]
},
{
"type": "STRING",
@ -590,6 +587,10 @@
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "SYMBOL",
"name": "array"
},
{
"type": "SYMBOL",
"name": "_empty_value"
@ -598,60 +599,85 @@
}
]
},
"file_redirect": {
"subscript": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "file_descriptor"
},
{
"type": "BLANK"
}
]
"type": "SYMBOL",
"name": "variable_name"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "<"
},
{
"type": "STRING",
"value": ">"
},
{
"type": "STRING",
"value": ">>"
},
{
"type": "STRING",
"value": "&>"
},
{
"type": "STRING",
"value": "&>>"
},
{
"type": "STRING",
"value": "<&"
},
{
"type": "STRING",
"value": ">&"
}
]
"type": "STRING",
"value": "["
},
{
"type": "SYMBOL",
"name": "_expression"
},
{
"type": "STRING",
"value": "]"
}
]
},
"file_redirect": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "file_descriptor"
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "<"
},
{
"type": "STRING",
"value": ">"
},
{
"type": "STRING",
"value": ">>"
},
{
"type": "STRING",
"value": "&>"
},
{
"type": "STRING",
"value": "&>>"
},
{
"type": "STRING",
"value": "<&"
},
{
"type": "STRING",
"value": ">&"
}
]
},
{
"type": "SYMBOL",
"name": "_expression"
}
]
}
},
"heredoc_redirect": {
"type": "SEQ",
"members": [
@ -727,10 +753,6 @@
"type": "SYMBOL",
"name": "string"
},
{
"type": "SYMBOL",
"name": "array"
},
{
"type": "SYMBOL",
"name": "raw_string"
@ -750,9 +772,100 @@
{
"type": "SYMBOL",
"name": "process_substitution"
},
{
"type": "SYMBOL",
"name": "concatenation"
}
]
},
"concatenation": {
"type": "PREC",
"value": -1,
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "word"
},
{
"type": "SYMBOL",
"name": "string"
},
{
"type": "SYMBOL",
"name": "raw_string"
},
{
"type": "SYMBOL",
"name": "expansion"
},
{
"type": "SYMBOL",
"name": "simple_expansion"
},
{
"type": "SYMBOL",
"name": "command_substitution"
},
{
"type": "SYMBOL",
"name": "process_substitution"
}
]
},
{
"type": "REPEAT1",
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_concat"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "word"
},
{
"type": "SYMBOL",
"name": "string"
},
{
"type": "SYMBOL",
"name": "raw_string"
},
{
"type": "SYMBOL",
"name": "expansion"
},
{
"type": "SYMBOL",
"name": "simple_expansion"
},
{
"type": "SYMBOL",
"name": "command_substitution"
},
{
"type": "SYMBOL",
"name": "process_substitution"
}
]
}
]
}
}
]
}
},
"string": {
"type": "SEQ",
"members": [
@ -822,21 +935,8 @@
"value": "$"
},
{
"type": "CHOICE",
"members": [
{
"type": "RENAME",
"content": {
"type": "SYMBOL",
"name": "simple_variable_name"
},
"value": "variable_name"
},
{
"type": "SYMBOL",
"name": "special_variable_name"
}
]
"type": "SYMBOL",
"name": "_variable_name"
}
]
},
@ -867,6 +967,52 @@
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "#"
},
{
"type": "SYMBOL",
"name": "_variable_name"
},
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "@"
},
{
"type": "STRING",
"value": "]"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_variable_name"
},
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "@"
},
{
"type": "STRING",
"value": "]"
}
]
},
{
"type": "SEQ",
"members": [
@ -892,6 +1038,14 @@
{
"type": "STRING",
"value": ":-"
},
{
"type": "STRING",
"value": "%"
},
{
"type": "STRING",
"value": "/"
}
]
},
@ -916,7 +1070,7 @@
"type": "RENAME",
"content": {
"type": "SYMBOL",
"name": "leading_word"
"name": "simple_variable_name"
},
"value": "variable_name"
},
@ -977,18 +1131,14 @@
"members": [
{
"type": "STRING",
"value": "<"
"value": "<("
},
{
"type": "STRING",
"value": ">"
"value": ">("
}
]
},
{
"type": "STRING",
"value": "("
},
{
"type": "SYMBOL",
"name": "_statement"
@ -999,17 +1149,44 @@
}
]
},
"leading_word": {
"type": "PATTERN",
"value": "[^`\"\\\\\\s#=|;:{}()]+"
},
"word": {
"type": "PATTERN",
"value": "[^\"`#\\\\\\s$<>{}&;()]+"
"type": "TOKEN",
"content": {
"type": "REPEAT1",
"content": {
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^#'\"<>{}()`$&;\\\\\\s]"
},
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\\"
},
{
"type": "PATTERN",
"value": "[^\\s]"
}
]
}
]
}
}
},
"comment": {
"type": "PATTERN",
"value": "#.*"
"type": "TOKEN",
"content": {
"type": "PREC",
"value": -1,
"content": {
"type": "PATTERN",
"value": "#.*"
}
}
},
"simple_variable_name": {
"type": "PATTERN",
@ -1126,9 +1303,17 @@
"type": "SYMBOL",
"name": "_empty_value"
},
{
"type": "SYMBOL",
"name": "_concat"
},
{
"type": "SYMBOL",
"name": "variable_name"
},
{
"type": "STRING",
"value": "#"
"value": "\n"
}
],
"inline": [

68666
src/parser.c vendored
View File

File diff suppressed because it is too large Load Diff

91
src/scanner.cc vendored
View File

@ -13,7 +13,9 @@ enum TokenType {
HEREDOC_END,
FILE_DESCRIPTOR,
EMPTY_VALUE,
LENGTH_OPERATOR
CONCAT,
VARIABLE_NAME,
NEWLINE,
};
struct Scanner {
@ -75,6 +77,32 @@ struct Scanner {
}
bool scan(TSLexer *lexer, const bool *valid_symbols) {
if (valid_symbols[CONCAT]) {
if (!(
iswspace(lexer->lookahead) ||
lexer->lookahead == '>' ||
lexer->lookahead == '<' ||
lexer->lookahead == ')' ||
lexer->lookahead == '(' ||
lexer->lookahead == '[' ||
lexer->lookahead == ']' ||
lexer->lookahead == '}' ||
lexer->lookahead == ';' ||
lexer->lookahead == '&' ||
lexer->lookahead == '`'
)) {
lexer->result_symbol = CONCAT;
return true;
}
}
if (valid_symbols[EMPTY_VALUE]) {
if (iswspace(lexer->lookahead)) {
lexer->result_symbol = EMPTY_VALUE;
return true;
}
}
if (valid_symbols[HEREDOC_MIDDLE] && !heredoc_delimiter.empty()) {
return scan_heredoc_content(lexer, HEREDOC_MIDDLE, HEREDOC_END);
}
@ -97,33 +125,58 @@ struct Scanner {
return scan_heredoc_content(lexer, HEREDOC_BEGINNING, SIMPLE_HEREDOC);
}
if (valid_symbols[FILE_DESCRIPTOR]) {
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') skip(lexer);
if (valid_symbols[VARIABLE_NAME] || valid_symbols[FILE_DESCRIPTOR]) {
for (;;) {
if (
lexer->lookahead == ' ' ||
lexer->lookahead == '\t' ||
(lexer->lookahead == '\n' && !valid_symbols[NEWLINE])
) {
skip(lexer);
} else if (lexer->lookahead == '\\') {
skip(lexer);
if (lexer->lookahead == '\n') {
skip(lexer);
} else {
return false;
}
} else {
break;
}
}
bool is_number = true;
if (iswdigit(lexer->lookahead)) {
advance(lexer);
while (iswdigit(lexer->lookahead)) advance(lexer);
if (lexer->lookahead == '>' || lexer->lookahead == '<') {
lexer->result_symbol = FILE_DESCRIPTOR;
return true;
} else if (iswalpha(lexer->lookahead) || lexer->lookahead == '_') {
is_number = false;
advance(lexer);
} else {
return false;
}
for (;;) {
if (iswdigit(lexer->lookahead)) {
advance(lexer);
} else if (iswalpha(lexer->lookahead) || lexer->lookahead == '_') {
is_number = false;
advance(lexer);
} else {
break;
}
}
}
if (valid_symbols[EMPTY_VALUE]) {
if (iswspace(lexer->lookahead)) {
lexer->result_symbol = EMPTY_VALUE;
if (is_number && valid_symbols[FILE_DESCRIPTOR] && (lexer->lookahead == '>' || lexer->lookahead == '<')) {
lexer->result_symbol = FILE_DESCRIPTOR;
return true;
}
}
if (valid_symbols[LENGTH_OPERATOR]) {
if (lexer->lookahead == '#') {
advance(lexer);
if (iswalpha(lexer->lookahead)) {
lexer->result_symbol = LENGTH_OPERATOR;
return true;
}
if (valid_symbols[VARIABLE_NAME] && (lexer->lookahead == '=' || lexer->lookahead == '[')) {
lexer->result_symbol = VARIABLE_NAME;
return true;
}
return false;
}
return false;