Skip to content

Repository navigation

Commit a2912e5

Browse files
sindresorhussindresorhus
committed
Fix object-mode chunks being typed as text or bytes when iterating
`subprocess` iteration and `subprocess.iterable()` ignored the `objectMode` of the descriptor's transform, so their chunks were typed as `string` or `Uint8Array` instead of the original objects.
1 parent 8629132 commit a2912e5

6 files changed

Lines changed: 148 additions & 12 deletions

File tree

‎docs/api.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,8 @@ If `true`, iterates over arbitrary chunks of data. Each line is an [`Uint8Array`
468468

469469
This is always `true` when the [`encoding`](#optionsencoding) option is binary.
470470

471+
With a [transform in object mode](transform.md#object-mode), each chunk is the object itself, whatever the value of this option.
472+
471473
More info for [iterables](binary.md#iterable) and [streams](binary.md#streams).
472474

473475
#### readableOptions.preserveNewlines

‎test-d/convert/iterable.test-d.ts‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,91 @@ import {execa} from '../../index.js';
44
const subprocess = execa('unicorns');
55
const bufferSubprocess = execa('unicorns', {encoding: 'buffer', all: true});
66
const hexSubprocess = execa('unicorns', {encoding: 'hex', all: true});
7+
const objectTransform = {
8+
objectMode: true as const,
9+
* transform() {
10+
yield {value: 'unicorn'};
11+
},
12+
};
13+
const objectSubprocess = execa('unicorns', {all: true, stdout: objectTransform});
14+
15+
expectTypeOf(objectSubprocess.iterable()).toBe<AsyncIterableIterator<unknown>>();
16+
expectTypeOf(objectSubprocess[Symbol.asyncIterator]()).toBe<AsyncIterableIterator<unknown>>();
17+
18+
const wrappedObjectSubprocess = execa('unicorns', {stdout: {value: objectTransform}});
19+
expectTypeOf(wrappedObjectSubprocess.iterable()).toBe<AsyncIterableIterator<unknown>>();
20+
expectTypeOf(wrappedObjectSubprocess[Symbol.asyncIterator]()).toBe<AsyncIterableIterator<unknown>>();
21+
const wrappedObjectResult = await wrappedObjectSubprocess;
22+
expectTypeOf(wrappedObjectResult.stdout).toBe<unknown[]>();
23+
24+
const wrappedArraySubprocess = execa('unicorns', {stdout: ['pipe', {value: objectTransform, input: false}] as const});
25+
expectTypeOf(wrappedArraySubprocess.iterable({binary: true})).toBe<AsyncIterableIterator<unknown>>();
26+
const wrappedArrayResult = await wrappedArraySubprocess;
27+
expectTypeOf(wrappedArrayResult.stdout).toBe<string | unknown[]>();
28+
29+
const wrappedStderrSubprocess = execa('unicorns', {all: true, stderr: {value: objectTransform}});
30+
expectTypeOf(wrappedStderrSubprocess.iterable()).toBe<AsyncIterableIterator<string>>();
31+
expectTypeOf(wrappedStderrSubprocess.iterable({from: 'stderr'})).toBe<AsyncIterableIterator<unknown>>();
32+
expectTypeOf(wrappedStderrSubprocess.iterable({from: 'all'})).toBe<AsyncIterableIterator<unknown>>();
33+
const wrappedStderrResult = await wrappedStderrSubprocess;
34+
expectTypeOf(wrappedStderrResult.stderr).toBe<unknown[]>();
35+
36+
const wrappedFdSubprocess = execa('unicorns', {stdio: ['pipe', 'pipe', 'pipe', {value: objectTransform, input: false}] as const});
37+
expectTypeOf(wrappedFdSubprocess.iterable({from: 'fd3'})).toBe<AsyncIterableIterator<unknown>>();
38+
const wrappedFdResult = await wrappedFdSubprocess;
39+
expectTypeOf(wrappedFdResult.stdio[3]).toBe<unknown[]>();
40+
41+
const wrappedTextSubprocess = execa('unicorns', {stdout: {value: 'pipe'}});
42+
expectTypeOf(wrappedTextSubprocess.iterable()).toBe<AsyncIterableIterator<string>>();
43+
const wrappedTextResult = await wrappedTextSubprocess;
44+
expectTypeOf(wrappedTextResult.stdout).toBe<string>();
45+
46+
expectTypeOf(objectSubprocess.iterable({from: 'stdout'})).toBe<AsyncIterableIterator<unknown>>();
47+
expectTypeOf(objectSubprocess.iterable({from: 'fd1'})).toBe<AsyncIterableIterator<unknown>>();
48+
expectTypeOf(objectSubprocess.iterable({binary: true})).toBe<AsyncIterableIterator<unknown>>();
49+
expectTypeOf(objectSubprocess.iterable({binary: false, preserveNewlines: true})).toBe<AsyncIterableIterator<unknown>>();
50+
expectTypeOf(objectSubprocess.iterable({from: 'stderr'})).toBe<AsyncIterableIterator<string>>();
51+
expectTypeOf(objectSubprocess.iterable({from: 'fd2', binary: true})).toBe<AsyncIterableIterator<Uint8Array>>();
52+
expectTypeOf(objectSubprocess.iterable({from: 'all'})).toBe<AsyncIterableIterator<unknown>>();
53+
expectTypeOf(objectSubprocess.iterable({from: 'all', binary: true})).toBe<AsyncIterableIterator<unknown>>();
54+
expectTypeOf(objectSubprocess.iterable({} as {from?: 'stdout' | 'stderr'; binary?: boolean})).toBe<AsyncIterableIterator<unknown>>();
55+
56+
const objectStderrSubprocess = execa('unicorns', {all: true, stderr: objectTransform});
57+
expectTypeOf(objectStderrSubprocess.iterable()).toBe<AsyncIterableIterator<string>>();
58+
expectTypeOf(objectStderrSubprocess[Symbol.asyncIterator]()).toBe<AsyncIterableIterator<string>>();
59+
expectTypeOf(objectStderrSubprocess.iterable({from: 'stderr'})).toBe<AsyncIterableIterator<unknown>>();
60+
expectTypeOf(objectStderrSubprocess.iterable({from: 'fd2'})).toBe<AsyncIterableIterator<unknown>>();
61+
expectTypeOf(objectStderrSubprocess.iterable({from: 'all'})).toBe<AsyncIterableIterator<unknown>>();
62+
63+
const objectFdSubprocess = execa('unicorns', {stdio: ['pipe', 'pipe', 'pipe', objectTransform] as const});
64+
expectTypeOf(objectFdSubprocess.iterable()).toBe<AsyncIterableIterator<string>>();
65+
expectTypeOf(objectFdSubprocess.iterable({from: 'fd3'})).toBe<AsyncIterableIterator<unknown>>();
66+
expectTypeOf(objectFdSubprocess.iterable({from: 'fd3', binary: true})).toBe<AsyncIterableIterator<unknown>>();
67+
expectTypeOf(objectFdSubprocess.iterable({from: 'fd3' as `fd${number}`})).toBe<AsyncIterableIterator<unknown>>();
68+
expectTypeOf(objectFdSubprocess.iterable({from: 'fd3' as `fd${number}`, binary: true})).toBe<AsyncIterableIterator<unknown>>();
69+
70+
const objectBufferSubprocess = execa('unicorns', {encoding: 'buffer', stdout: objectTransform});
71+
const objectHexSubprocess = execa('unicorns', {encoding: 'hex', stdout: objectTransform});
72+
expectTypeOf(objectBufferSubprocess.iterable()).toBe<AsyncIterableIterator<unknown>>();
73+
expectTypeOf(objectBufferSubprocess[Symbol.asyncIterator]()).toBe<AsyncIterableIterator<unknown>>();
74+
expectTypeOf(objectBufferSubprocess.iterable({binary: false})).toBe<AsyncIterableIterator<unknown>>();
75+
expectTypeOf(objectBufferSubprocess.iterable({from: 'stderr'})).toBe<AsyncIterableIterator<Uint8Array>>();
76+
expectTypeOf(objectHexSubprocess.iterable({binary: true})).toBe<AsyncIterableIterator<unknown>>();
77+
78+
const dynamicObjectSubprocess = execa('unicorns', {stdout: {...objectTransform, objectMode: true as boolean}});
79+
expectTypeOf(dynamicObjectSubprocess.iterable()).toBe<AsyncIterableIterator<unknown>>();
80+
81+
const unionObjectSubprocess = execa('unicorns', {} as {stdout: typeof objectTransform} | {stdout: 'pipe'});
82+
expectTypeOf(unionObjectSubprocess.iterable()).toBe<AsyncIterableIterator<unknown>>();
83+
84+
const unionDefaultSubprocess = execa('unicorns', {} as {} | {stdout: typeof objectTransform});
85+
expectTypeOf(unionDefaultSubprocess.iterable()).toBe<AsyncIterableIterator<unknown>>();
86+
87+
const objectPipe = subprocess.pipe('unicorns', {stdout: objectTransform});
88+
expectTypeOf(objectPipe.iterable()).toBe<AsyncIterableIterator<unknown>>();
89+
expectTypeOf(objectPipe[Symbol.asyncIterator]()).toBe<AsyncIterableIterator<unknown>>();
90+
expectTypeOf(subprocess.pipe(objectSubprocess).iterable()).toBe<AsyncIterableIterator<unknown>>();
91+
expectTypeOf(subprocess.pipe(objectSubprocess).iterable({from: 'stderr'})).toBe<AsyncIterableIterator<string>>();
792

893
const asyncIteration = async () => {
994
for await (const line of subprocess) {

‎test/convert/iterable.js‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {assertRejects} from '../helpers/assert.js';
44
import {execa} from '../../index.js';
55
import {setFixtureDirectory} from '../helpers/fixtures-directory.js';
66
import {foobarString} from '../helpers/input.js';
7-
import {fullStdio, assertEpipe} from '../helpers/stdio.js';
7+
import {fullStdio, assertEpipe, getStdio} from '../helpers/stdio.js';
88
import {
99
arrayFromAsync,
1010
assertWritableAborted,
@@ -70,6 +70,33 @@ test('Can iterate using Symbol.asyncIterator', async () => {
7070
assert.deepEqual(lines, noNewlinesChunks);
7171
});
7272

73+
const testObjectChunks = async (fdNumber, from, binary) => {
74+
const chunks = [{value: 'object'}, [1, [2]], new Uint8Array([97, 98]), 'first\nsecond\n', false, 0];
75+
const subprocess = execa('noop-fd.js', [`${fdNumber}`, 'x'], {
76+
...getStdio(fdNumber, {
77+
objectMode: true,
78+
binary: true,
79+
* transform() {
80+
yield * chunks;
81+
},
82+
}),
83+
all: true,
84+
});
85+
const iterable = from === undefined ? subprocess : subprocess.iterable({from, binary});
86+
const output = await arrayFromAsync(iterable);
87+
assert.deepEqual(output, chunks);
88+
for (const [index, chunk] of chunks.entries()) {
89+
assert.equal(output[index], chunk);
90+
}
91+
};
92+
93+
test('Default iteration preserves arbitrary object-mode chunks', () => testObjectChunks(1));
94+
test('Text iteration preserves arbitrary object-mode chunks', () => testObjectChunks(1, 'stdout', false));
95+
test('Binary iteration preserves arbitrary object-mode chunks', () => testObjectChunks(1, 'stdout', true));
96+
test('Iteration preserves arbitrary stderr object-mode chunks', () => testObjectChunks(2, 'stderr', false));
97+
test('Iteration preserves arbitrary fd3 object-mode chunks', () => testObjectChunks(3, 'fd3', true));
98+
test('Iteration preserves arbitrary merged object-mode chunks', () => testObjectChunks(1, 'all', false));
99+
73100
const assertMultipleCalls = async (iterable, iterableTwo) => {
74101
assert.notEqual(iterable, iterableTwo);
75102
const lines = await arrayFromAsync(iterable);

‎types/convert.d.ts‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type {BinaryEncodingOption} from './arguments/encoding-option.js';
22
import type {Options} from './arguments/options.js';
33
import type {FromOption, ToOption} from './arguments/fd-options.js';
4+
import type {IsObjectFd} from './transform/object-mode.js';
45

56
// `subprocess.readable|duplex|iterable()` options
67
export type ReadableOptions = {
@@ -20,6 +21,8 @@ export type ReadableOptions = {
2021
2122
This is always `true` when the `encoding` option is binary.
2223
24+
With a transform in object mode, each chunk is the object itself, whatever the value of this option.
25+
2326
@default `false` with `subprocess.iterable()`, `true` otherwise
2427
*/
2528
readonly binary?: boolean;
@@ -49,12 +52,27 @@ export type DuplexOptions = ReadableOptions & WritableOptions;
4952
export type SubprocessAsyncIterable<
5053
BinaryOption extends boolean | undefined,
5154
EncodingOption extends Options['encoding'],
55+
ObjectMode extends boolean,
5256
> = AsyncIterableIterator<
53-
EncodingOption extends BinaryEncodingOption
54-
? Uint8Array
55-
: BinaryOption extends true
57+
// Object-mode chunks are passed through unchanged, even with binary options.
58+
true extends ObjectMode
59+
? unknown
60+
: EncodingOption extends BinaryEncodingOption
5661
? Uint8Array
57-
: string
62+
: BinaryOption extends true
63+
? Uint8Array
64+
: string
5865
>;
5966

67+
// `all` can yield objects from either stdout or stderr.
68+
export type IsObjectIterable<
69+
From extends ReadableOptions['from'],
70+
OptionsType extends Options,
71+
> = OptionsType extends Options
72+
? From extends 'all' ? IsObjectFd<'1' | '2', OptionsType>
73+
: From extends 'stderr' ? IsObjectFd<'2', OptionsType>
74+
: From extends `fd${infer FdNumber}` ? IsObjectFd<`${number}` extends FdNumber ? string : FdNumber, OptionsType>
75+
: IsObjectFd<'1', OptionsType>
76+
: never;
77+
6078
export {};

‎types/subprocess/subprocess.d.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import type {
1010
WritableOptions,
1111
DuplexOptions,
1212
SubprocessAsyncIterable,
13+
IsObjectIterable,
1314
} from '../convert.js';
1415
import type {IpcMethods, HasIpc} from '../ipc.js';
1516
import type {SubprocessStdioStream} from './stdout.js';
@@ -32,12 +33,12 @@ export type SubprocessResultMethods<OptionsType extends Options = Options> = {
3233
/**
3334
Subprocesses are [async iterables](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol/asyncIterator). They iterate over each output line.
3435
*/
35-
[Symbol.asyncIterator](): SubprocessAsyncIterable<undefined, OptionsType['encoding']>;
36+
[Symbol.asyncIterator](): SubprocessAsyncIterable<undefined, OptionsType['encoding'], IsObjectIterable<undefined, OptionsType>>;
3637

3738
/**
3839
Same as `subprocess[Symbol.asyncIterator]` except options can be provided.
3940
*/
40-
iterable<IterableOptions extends ReadableOptions = {}>(readableOptions?: IterableOptions): SubprocessAsyncIterable<IterableOptions['binary'], OptionsType['encoding']>;
41+
iterable<IterableOptions extends ReadableOptions = {}>(readableOptions?: IterableOptions): SubprocessAsyncIterable<IterableOptions['binary'], OptionsType['encoding'], IsObjectIterable<IterableOptions['from'], OptionsType>>;
4142

4243
/**
4344
Converts the subprocess to a readable stream.

‎types/transform/object-mode.d.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,14 @@ export type IsObjectFd<
1212

1313
type IsObjectStdioOption<StdioOptionType> = IsObjectStdioSingleOption<StdioSingleOptionItems<StdioOptionType>>;
1414

15-
type IsObjectStdioSingleOption<StdioSingleOptionType> = StdioSingleOptionType extends TransformCommon
16-
? BooleanObjectMode<StdioSingleOptionType['objectMode']>
17-
: StdioSingleOptionType extends DuplexTransform
18-
? StdioSingleOptionType['transform']['readableObjectMode']
19-
: false;
15+
// Direction wrappers do not change the transform's object mode.
16+
type IsObjectStdioSingleOption<StdioSingleOptionType> = StdioSingleOptionType extends {readonly value: infer Value}
17+
? IsObjectStdioSingleOption<Value>
18+
: StdioSingleOptionType extends TransformCommon
19+
? BooleanObjectMode<StdioSingleOptionType['objectMode']>
20+
: StdioSingleOptionType extends DuplexTransform
21+
? StdioSingleOptionType['transform']['readableObjectMode']
22+
: false;
2023

2124
type BooleanObjectMode<ObjectModeOption extends boolean | undefined> = ObjectModeOption extends true ? true : false;
2225

0 commit comments

Comments
 (0)