Skip to content

Repository navigation

Commit b67f199

Browse files
sindresorhussindresorhus
committed
Fix template string tokenization of escape sequences
1 parent 7c1a631 commit b67f199

5 files changed

Lines changed: 86 additions & 5 deletions

File tree

‎lib/arguments/escape.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ const COMMON_ESCAPES = {
6767
'\t': '\\t',
6868
};
6969

70-
// Up until that codepoint, \u notation can be used instead of \U
71-
const ASTRAL_START = 65_535;
70+
// Up until that code point, a character is a single UTF-16 code unit, so the \u notation can be used instead of \U
71+
export const ASTRAL_START = 0xFF_FF;
7272

7373
// Some characters are shell-specific, i.e. need to be escaped when the command is copy-pasted then run.
7474
// Escaping is shell-specific. We cannot know which shell is used: `process.platform` detection is not enough.

‎lib/methods/template.js‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {ChildProcess} from 'node:child_process';
22
import isPlainObject from 'is-plain-obj';
3+
import {ASTRAL_START} from '../arguments/escape.js';
34
import {isUint8Array, uint8ArrayToString} from '../utils/uint-array.js';
45

56
// Check whether the template string syntax is being used
@@ -53,13 +54,15 @@ const parseTemplate = ({templates, expressions, tokens, index, template}) => {
5354
// - the first array argument, which does not escape backslash sequences
5455
// - its `raw` property, which escapes them
5556
const splitByWhitespaces = (template, rawTemplate) => {
56-
if (rawTemplate.length === 0) {
57+
// The template is empty either between two `${expression}`, or when it only contains escaped newlines.
58+
// Neither holds any whitespace, so neither separates tokens.
59+
if (template.length === 0) {
5760
return {nextTokens: [], leadingWhitespaces: false, trailingWhitespaces: false};
5861
}
5962

6063
const nextTokens = [];
6164
let templateStart = 0;
62-
const isLeadingWhitespaces = DELIMITERS.has(rawTemplate[0]);
65+
let isLeadingWhitespaces = false;
6366

6467
for (
6568
let templateIndex = 0, rawIndex = 0;
@@ -68,6 +71,11 @@ const splitByWhitespaces = (template, rawTemplate) => {
6871
) {
6972
const rawCharacter = rawTemplate[rawIndex];
7073
if (DELIMITERS.has(rawCharacter)) {
74+
// Escaped newlines are ignored, so a whitespace after them is still a leading one
75+
if (templateIndex === 0) {
76+
isLeadingWhitespaces = true;
77+
}
78+
7179
if (templateStart !== templateIndex) {
7280
nextTokens.push(template.slice(templateStart, templateIndex));
7381
}
@@ -80,7 +88,13 @@ const splitByWhitespaces = (template, rawTemplate) => {
8088
templateIndex -= 1;
8189
rawIndex += 1;
8290
} else if (nextRawCharacter === 'u' && rawTemplate[rawIndex + 2] === '{') {
83-
rawIndex = rawTemplate.indexOf('}', rawIndex + 3);
91+
const closingBraceIndex = rawTemplate.indexOf('}', rawIndex + 3);
92+
// A code point above the BMP is a surrogate pair, i.e. two characters, in the template string
93+
if (isAstralEscape(rawTemplate.slice(rawIndex + 3, closingBraceIndex))) {
94+
templateIndex += 1;
95+
}
96+
97+
rawIndex = closingBraceIndex;
8498
} else {
8599
rawIndex += ESCAPE_LENGTH[nextRawCharacter] ?? 1;
86100
}
@@ -95,6 +109,8 @@ const splitByWhitespaces = (template, rawTemplate) => {
95109
return {nextTokens, leadingWhitespaces: isLeadingWhitespaces, trailingWhitespaces: isTrailingWhitespaces};
96110
};
97111

112+
const isAstralEscape = codePointHex => Number.parseInt(codePointHex, 16) > ASTRAL_START;
113+
98114
const DELIMITERS = new Set([' ', '\t', '\r', '\n']);
99115

100116
// Number of characters in backslash escape sequences: \0 \xXX or \uXXXX

‎test/arguments/escape.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,3 +102,32 @@ test('result.escapedCommand - \\uE000', testEscapedCommand, ['\u{E000}'], '\'\\u
102102
test('result.escapedCommand - \\U1D172', testEscapedCommand, ['\u{1D172}'], '\'\u{1D172}\'', '"\u{1D172}"');
103103
test('result.escapedCommand - \\U1D173', testEscapedCommand, ['\u{1D173}'], '\'\\U1d173\'', '"\\U1d173"', '\'\u{1D173}\'', '"\u{1D173}"');
104104
test('result.escapedCommand - \\U10FFFD', testEscapedCommand, ['\u{10FFFD}'], '\'\\U10fffd\'', '"\\U10fffd"', '\'\u{10FFFD}\'', '"\u{10FFFD}"');
105+
106+
/*
107+
`result.escapedCommand` is documented as safe to copy and paste in a terminal, so a shell must parse it back to the exact same arguments.
108+
Control characters are excluded, since those are intentionally escaped with a Bash-specific notation instead.
109+
*/
110+
if (!isWindows) {
111+
const testShellRoundTrip = async (t, commandArguments) => {
112+
const {escapedCommand} = await execa('print-arguments.js', commandArguments);
113+
const {stdout} = await execa('/bin/sh', ['-c', escapedCommand]);
114+
t.deepEqual(JSON.parse(stdout), commandArguments);
115+
};
116+
117+
testShellRoundTrip.title = (message, commandArguments) => `result.escapedCommand can be pasted in a shell: ${JSON.stringify(commandArguments)}`;
118+
119+
test(testShellRoundTrip, ['foo', 'bar']);
120+
test(testShellRoundTrip, ['foo bar']);
121+
test(testShellRoundTrip, ['\'foo\'', '"bar"']);
122+
test(testShellRoundTrip, ['it\'s', 'a\'\'b']);
123+
test(testShellRoundTrip, ['foo\\bar', '\\', '\\\\']);
124+
// eslint-disable-next-line no-template-curly-in-string
125+
test(testShellRoundTrip, ['$FOO', '${FOO}', '$(echo foo)']);
126+
test(testShellRoundTrip, ['`echo foo`']);
127+
test(testShellRoundTrip, ['foo;bar', 'foo&&bar', 'foo|bar', 'foo>bar']);
128+
test(testShellRoundTrip, ['*', '?', '[a-z]', '{a,b}']);
129+
test(testShellRoundTrip, ['~', '#foo', '!foo', '-foo', '--foo=bar']);
130+
test(testShellRoundTrip, ['', 'foo']);
131+
test(testShellRoundTrip, ['ã', '😀']);
132+
test(testShellRoundTrip, ['\'"`$\\!*&;| ']);
133+
}

‎test/fixtures/print-arguments.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env node
2+
import process from 'node:process';
3+
4+
process.stdout.write(JSON.stringify(process.argv.slice(2)));

‎test/methods/template.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,25 @@ test('$ can use escaped newlines and character escaped tab indentations', testSc
3030
\tbar`, 'foo\tbar');
3131
test('$ can use escaped newlines and character escaped newlines', testScriptStdout, () => $`echo.js foo\
3232
\n\nbar`, 'foo\n\nbar');
33+
// An escaped newline is ignored, so it must not separate tokens, nor hide the whitespaces following it.
34+
// This must behave the same whether the tokens around it are expressions or not.
35+
test('$ can use escaped newlines between tokens', testScriptStdout, () => $`echo.js foo\
36+
bar`, 'foobar');
37+
test('$ can use escaped newlines between expressions', testScriptStdout, () => $`echo.js ${'foo'}\
38+
${'bar'}`, 'foobar');
39+
test('$ can use multiple escaped newlines between expressions', testScriptStdout, () => $`echo.js ${'foo'}\
40+
\
41+
${'bar'}`, 'foobar');
42+
test('$ can use escaped newlines and space indentations between expressions', testScriptStdout, () => $`echo.js ${'foo'}\
43+
${'bar'}`, 'foo\nbar');
44+
test('$ can use escaped newlines and tab indentations between expressions', testScriptStdout, () => $`echo.js ${'foo'}\
45+
${'bar'}`, 'foo\nbar');
46+
test('$ can use escaped newlines and space indentations after an expression', testScriptStdout, () => $`echo.js ${'foo'}\
47+
bar`, 'foo\nbar');
48+
test('$ can use escaped newlines and tab indentations after an expression', testScriptStdout, () => $`echo.js ${'foo'}\
49+
bar`, 'foo\nbar');
50+
test('$ can use escaped newlines before an expression', testScriptStdout, () => $`echo.js foo\
51+
${'bar'}`, 'foobar');
3352
test('$ can use Windows newlines and tab indentations', testScriptStdout, () => escapedCall('echo.js foo\r\n\tbar'), 'foo\nbar');
3453
test('$ can use Windows newlines and space indentations', testScriptStdout, () => escapedCall('echo.js foo\r\n bar'), 'foo\nbar');
3554
test('$ does not ignore comments in expressions', testScriptStdout, () => $`echo.js foo
@@ -238,6 +257,19 @@ test('$ splits tokens - \\u{0000063}', testScriptStdout, () => $`echo.js a\u{000
238257
test('$ splits expressions - \\u{0000063}', testScriptStdout, () => $`echo.js ${'a'}\u{0000063}${'b'}`, 'acb');
239258
test('$ concatenates tokens - \\u{0000063}', testScriptStdout, () => $`echo.js \u{0000063}a\u{0000063} b`, 'cac\nb');
240259
test('$ concatenates expressions - \\u{0000063}', testScriptStdout, () => $`echo.js \u{0000063}${'a'}\u{0000063} b`, 'cac\nb');
260+
261+
// Code points above the BMP are a surrogate pair, i.e. two characters, in the template string but not in its raw version.
262+
// Miscounting them shifts every following character, which splits tokens at the wrong place and breaks the surrogate pair.
263+
test('$ handles tokens - \\u{1F600}', testScriptStdout, () => $`echo.js \u{1F600}`, '\u{1F600}');
264+
test('$ splits tokens - \\u{1F600}', testScriptStdout, () => $`echo.js a\u{1F600}b`, 'a\u{1F600}b');
265+
test('$ splits expressions - \\u{1F600}', testScriptStdout, () => $`echo.js ${'a'}\u{1F600}${'b'}`, 'a\u{1F600}b');
266+
test('$ concatenates tokens - \\u{1F600}', testScriptStdout, () => $`echo.js \u{1F600}a\u{1F600} b`, '\u{1F600}a\u{1F600}\nb');
267+
test('$ concatenates expressions - \\u{1F600}', testScriptStdout, () => $`echo.js \u{1F600}${'a'}\u{1F600} b`, '\u{1F600}a\u{1F600}\nb');
268+
test('$ handles multiple tokens - \\u{1F600}', testScriptStdout, () => $`echo.js \u{1F600}\u{1F600} b c`, '\u{1F600}\u{1F600}\nb\nc');
269+
test('$ handles the highest code point - \\u{10FFFF}', testScriptStdout, () => $`echo.js \u{10FFFF} b`, '\u{10FFFF}\nb');
270+
// The BMP boundary is the exact point where the character count changes
271+
test('$ handles the last BMP code point - \\u{FFFF}', testScriptStdout, () => $`echo.js \u{FFFF} b`, '\u{FFFF}\nb');
272+
test('$ handles the first astral code point - \\u{10000}', testScriptStdout, () => $`echo.js \u{10000} b`, '\u{10000}\nb');
241273
test('$ handles tokens - \\u{0063}}', testScriptStdout, () => $`echo.js \u{0063}}`, 'c}');
242274
test('$ splits tokens - \\u{0063}}', testScriptStdout, () => $`echo.js a\u{0063}}b`, 'ac}b');
243275
test('$ splits expressions - \\u{0063}}', testScriptStdout, () => $`echo.js ${'a'}\u{0063}}${'b'}`, 'ac}b');

0 commit comments

Comments
 (0)