Previously, each row was JSON serialized and deserialized on each `.pipeThrough`. The intent of this serialization was to normalize the data of each row, with an escape hatch in the `rawInput` and `rawOutput` options.
However, this approach is counterintuitive because most users would expect a CSV row to simply be an `Array<string>`, NOT a `string` that can be `JSON.parse`-ed into an `Array<string>`. Furthermore, this excessive serialization has a significant performance impact. Lastly, the `rawInput` and `rawOutput` options effectively made `CSVTransformer` an overly complex and difficult-to-use JSON parser on top of a CSV transformer.
So, simply removing the JSON serialization and directly enqueuing the actual object (i.e., the `Array<string>`) resolves all these issues with minimal impact to the user-facing API.
for await (const row of readableStream) {
console.log(row);
}
-// ["columnA","columnB"]
-// ["1","1"]
-// ["100","100"]
-// ["223423","455947"]
-// ["348553","692708"]
-// ["536368","676147"]
+// [ 'columnA', 'columnB' ]
+// [ '1', '1' ]
+// [ '100', '100' ]
+// [ '223423', '455947' ]
+// [ '348553', '692708' ]
+// [ '536368', '676147' ]
```
### `CSVTransformer`
const transformStream = new CSVTransformer(edit);
const rows = ReadableStream.from((function* () {
- yield '["header"]';
- yield '["1"]';
- yield '["2"]';
- yield '["3"]';
+ yield ['header'];
+ yield ['1'];
+ yield ['2'];
+ yield ['3'];
})()).pipeThrough(transformStream);
for await (const row of rows) {
console.log(row);
}
-// ["header"]
-// ["1: transformed!"]
-// ["2: transformed!"]
-// ["3: transformed!"]
+// [ 'header' ]
+// [ '1: transformed!' ]
+// [ '2: transformed!' ]
+// [ '3: transformed!' ]
```
### `CSVWriter`
const writableStream = new CSVWriter('./example.csv');
await ReadableStream.from((function* () {
- yield '["header1","header2"]';
- yield '["1","1"]';
- yield '["2","2"]';
- yield '["3","3"]';
+ yield ['header1', 'header2'];
+ yield ['1', '1'];
+ yield ['2', '2'];
+ yield ['3', '3'];
})()).pipeTo(writableStream);
// A new ./example.csv file is created with the CSV data
```
]
const rows = ReadableStream.from((function* () {
- yield '["numberCol","dateCol","",""]'; // Empty columns inserted
- yield '["1E+18","45292","",""]'; // Number mangled to scientific notation; date mangled to number
- yield '["","","",""]'; // Empty rows inserted
- yield '["","","",""]';
- yield '["","","",""]';
+ yield ['numberCol', 'dateCol', '', '']; // Empty columns inserted
+ yield ['1E+18', '45292', '', '']; // Number mangled to scientific notation; date mangled to number
+ yield ['', '', '', '']; // Empty rows inserted
+ yield ['', '', '', ''];
+ yield ['', '', '', ''];
})())
.pipeThrough(new CSVNormalizer(headers))
.pipeThrough(new CSVDenormalizer());
for await (const row of rows) {
console.log(row);
}
-// ["Number Column","Date Column"]
-// [1000000000000000000,"2024-01-01T00:00:00.000Z"]
+// [ 'Number Column', 'Date Column' ]
+// [ 1000000000000000000, '2024-01-01T00:00:00.000Z' ]
```
## Tests
// ## Example 1.3: Chaining multiple transformations
//
-// Set `rawInput: true` and/or `rawOutput: true` to skip JSON deserialization (`rawInput`) and/or JSON serialization
-// (`rawOutput`) when handling the transformation function input/output. These options are intended for advanced
-// transformations (e.g., transformations spread across multiple functions or lower-level transformations directly
-// on byte arrays).
+// For advanced transformations (e.g., transformations spread across multiple functions or lower-level transformations
+// directly on byte arrays), manually serialize and deserialize your data (e.g., using `JSON.stringify` and `JSON.parse`).
+// A raw `string` type will be "passed through" `CSVTransformer` for your transformation function to handle directly.
//
// Set `handleHeaders` to `true` to pass the header row to the transformation function as a normal row and manually
// handle the header output processing. This setting is required if the input CSV does not have a header row.
function parse(row) {
const [name, id] = row[0].split('_');
const [first, last] = name.split('.')
- return JSON.stringify({ first, last, id }); // Manually handling serialization here, so use `rawOutput` to turn off automatic serialization
+ return JSON.stringify({ first, last, id }); // Manually handle serialization here
}
// Contains the logic for the transformation
function calculate(obj) {
- const { first, last, id } = JSON.parse(obj); // Manually handling deserialization here, so use `rawInput` to turn off automatic deserialization
+ const { first, last, id } = JSON.parse(obj); // Manually handle deserialization here
const name = `${last}, ${first}`;
const paddedId = id.padStart(6, '0');
return [name, paddedId];
}
await new CSVReader(new URL('./data/ex1_3-in.csv', import.meta.url))
- .pipeThrough(new CSVTransformer(parse, { handleHeaders: true, rawOutput: true }))
- .pipeThrough(new CSVTransformer(calculate, { handleHeaders: true, rawInput: true }))
+ .pipeThrough(new CSVTransformer(parse, { handleHeaders: true }))
+ .pipeThrough(new CSVTransformer(calculate, { handleHeaders: true }))
.pipeTo(new CSVWriter(new URL('./data/ex1_3-out.csv', import.meta.url)));
\ No newline at end of file
/**
* A simple streaming parser for CSV files.
*
- * Each chunk is one row of the CSV file, in the form of a string that may be deserialized using `JSON.parse()`.
+ * Each chunk is one row of the CSV file as an `Array<string>`.
*/
export class CSVReader {
/** @type {Array<string>} */
* @param {PathLike} path A `string`, `Buffer`, or `URL` representing a path to a local CSV file or
* `null` to create a `TransformStream`.
* @param {CSVReaderOptions} [options] Object containing flags to configure the reader.
- * @returns {CSVReader&ReadableStream<string>}
+ * @returns {CSVReader&ReadableStream<Array<string>>}
*/
/**
* @overload
* @param {null} [path] A `string`, `Buffer`, or `URL` representing a path to a local CSV file or
* `null` to create a `TransformStream`.
- * @returns {CSVReader&TransformStream<string,string>}
+ * @returns {CSVReader&TransformStream<string,Array<string>>}
*/
constructor(path = null, options = {}) {
- /** @type {ReadableStream<string>|TransformStream<string,string>} */
+ /** @type {ReadableStream<Array<string>>|TransformStream<string,Array<string>>} */
let stream;
stream = new TransformStream({
transform: (chunk, controller) => this.#transform(chunk, controller),
return createProxy(this, stream);
}
- /** @type {import('node:stream/web').TransformerTransformCallback<string,string>} */
+ /** @type {import('node:stream/web').TransformerTransformCallback<string,Array<string>>} */
#transform(chunk, controller) {
for (const char of chunk) {
/* Escape mode logic */
else if (char === '\n') { // Terminate the field and row and do not add char to field
this.#row.push(this.#field);
this.#field = '';
- controller.enqueue(JSON.stringify(this.#row));
+ controller.enqueue(this.#row.slice());
this.#row.length = 0;
}
else { // No special cases match, add the literal char
}
}
- /** @type {import('node:stream/web').TransformerFlushCallback<string>} */
+ /** @type {import('node:stream/web').TransformerFlushCallback<Array<string>>} */
#flush(controller) {
if (this.#field !== '' || this.#row.length !== 0) { // CSV terminated without a trailing CRLF, leaving a row in the queue
this.#row.push(this.#field); // Last field is always un-pushed if CSV terminated with nothing
- controller.enqueue(JSON.stringify(this.#row));
+ controller.enqueue(this.#row.slice());
}
else if (this.#lastChar === null) { // CSV is blank
- controller.enqueue(JSON.stringify(['']));
+ controller.enqueue(['']);
}
}
}
* @returns {TransformationOutput|Promise<TransformationOutput>}
*/
-/**
- * A function to process a row or rows of raw CSV data from `CSVReader`.
- * @callback TransformationFunctionRaw
- * @param {string} row A row or rows of input CSV data represented as a raw `string`.
- * @returns {TransformationOutput|Promise<TransformationOutput>}
- */
-
/**
* A function to process multiple rows of CSV data from `CSVReader`.
* @callback TransformationFunctionBatch
* @returns {TransformationOutput|Promise<TransformationOutput>}
*/
-/**
- * A function to handle errors thrown by a transformation function.
- * @callback TransformationErrorFunctionRaw
- * @param {string} row A row or rows of raw input CSV data passed to the transformation function which threw the error.
- * @param {Error} error The error thrown by the transformation function.
- * @param {TransformationFunctionRaw} fn The transformation function itself. This argument can be used to retry a transformation.
- * @returns {TransformationOutput|Promise<TransformationOutput>}
- */
-
/**
* A function to handle errors thrown by a transformation function.
* @callback TransformationErrorFunctionBatch
* to the `TransformationFunction`. The default value is `false`.
*
* NOTE: If the CSV has no header row, set `handleHeaders` to `true` and process the first row normally in `fn()`.
- * @property {false} [rawInput] Set to `true` to send the raw CSV row to `fn()` as a `string`. Otherwise, the CSV row will be deserialized
- * using `JSON.parse()` before being sent to `fn()`. The default value is `false`.
- * @property {boolean} [rawOutput=false] Set to `true` to send the raw return value of `fn()` to the next stream. Otherwise, the return value of
- * `fn()` will be serialized using `JSON.stringify()` before being sent to the next stream. The default value is `false`.
* @property {null|TransformationErrorFunction} [onError=null] Set to a function to catch errors thrown by the transformation function. The
* input data, the error that was thrown, and the transformation function itself will be passed to the `onError` function. The default value is
* `null` (errors will not be caught).
* resolved). The default value is `1`.
*/
-/**
- * Options to configure `CSVTransformer`.
- * @typedef {Object} CSVTransformerOptionsRaw
- * @property {boolean|Array<any>|TransformationFunctionRaw} [handleHeaders=false] A `boolean`, `Array<any>`, or `TransformationFunctionRaw` to
- * process the header row (first row) of the CSV. If `false`, the header row will pass through the stream. If `true`, the header row will be passed
- * to `fn()`. If `Array<any>`, the header row will be replaced with the `Array<any>`. If `TransformationFunctionRaw`, the header row will be passed
- * to the `TransformationFunctionRaw` as a `string`. The default value is `false`.
- *
- * NOTE: If the CSV has no header row, set `handleHeaders` to `true` and process the first row normally in `fn()`.
- * @property {true} rawInput Set to `true` to send the raw CSV row to `fn()` as a `string`. Otherwise, the CSV row will be deserialized
- * using `JSON.parse()` before being sent to `fn()`. The default value is `false`.
- * @property {boolean} [rawOutput=false] Set to `true` to send the raw return value of `fn()` to the next stream. Otherwise, the return value of
- * `fn()` will be serialized using `JSON.stringify()` before being sent to the next stream. The default value is `false`.
- * @property {null|TransformationErrorFunctionRaw} [onError=null] Set to a function to catch errors thrown by the transformation function. The
- * input data, the error that was thrown, and the transformation function itself will be passed to the `onError` function. The default value is
- * `null` (errors will not be caught).
- * @property {number} [maxBatchSize=1] Set to the maximum number of rows that will be passed to the transformation function per function call
- * (greedy).
- * @property {number} [maxConcurrent=1] The maximum concurrent executions of the transformation function (greedy). The transformation function
- * will be automatically turned into a promise if it isn't already async. Execution is blocked until all promises in a concurrent group settle
- * (i.e., if one transformation in a group is hanging, further rows will NOT be processed even if all other transformations in the group are
- * resolved). The default value is `1`.
- */
-
/**
* Options to configure `CSVTransformer`.
* @typedef {Object} CSVTransformerOptionsBatch
* to the `TransformationFunction`. The default value is `false`.
*
* NOTE: If the CSV has no header row, set `handleHeaders` to `true` and process the first row normally in `fn()`.
- * @property {false} [rawInput] Set to `true` to send the raw CSV row to `fn()` as a `string`. Otherwise, the CSV row will be deserialized
- * using `JSON.parse()` before being sent to `fn()`. The default value is `false`.
- * @property {boolean} [rawOutput=false] Set to `true` to send the raw return value of `fn()` to the next stream. Otherwise, the return value of
- * `fn()` will be serialized using `JSON.stringify()` before being sent to the next stream. The default value is `false`.
* @property {null|TransformationErrorFunctionBatch} [onError=null] Set to a function
* to catch errors thrown by the transformation function. The input data, the error that was thrown, and the transformation function itself will
* be passed to the `onError` function. The default value is `null` (errors will not be caught).
#firstChunk = true;
#handleHeaders;
- #rawInput;
- #rawOutput;
#onError;
#maxBatchSize;
#maxConcurrent;
* @param {TransformationFunction} fn A function to process a row of CSV data.
* @param {CSVTransformerOptions} options Object containing flags to configure the stream logic.
*/
- /**
- * @overload
- * @param {TransformationFunctionRaw} fn A function to process a row or rows of CSV data represented as a raw `string`.
- * @param {CSVTransformerOptionsRaw} options Object containing flags to configure the stream logic.
- */
/**
* @overload
* @param {TransformationFunctionBatch} fn A function to process multiple rows of CSV data from `CSVReader`.
});
this.#handleHeaders = options.handleHeaders ?? false;
- this.#rawInput = options.rawInput ?? false;
- this.#rawOutput = options.rawOutput ?? false;
this.#onError = options.onError ?? null;
this.#maxBatchSize = options.maxBatchSize ?? null;
this.#maxConcurrent = options.maxConcurrent ?? 1;
if (row === null || row === undefined) { // Input row is consumed without emitting any output row
return;
}
- else if (this.#rawOutput || typeof row === 'string') {
+ else if (typeof row === 'string') {
controller.enqueue(row);
}
else if (Array.isArray(row)) {
if (Array.isArray(row[0])) { // Multiple rows returned, enqueue each row separately
- row.forEach(r => controller.enqueue(JSON.stringify(r)));
+ row.forEach(r => controller.enqueue(r.slice()));
}
else { // One row returned
- controller.enqueue(JSON.stringify(row));
+ controller.enqueue(row.slice());
}
}
else { // Assume ArrayLike output
- controller.enqueue(JSON.stringify(Array.from(row)));
+ controller.enqueue(Array.from(row));
}
}
}
}
- /** @type {import('node:stream/web').TransformerTransformCallback<string,string>} */
+ /** @type {import('node:stream/web').TransformerTransformCallback<Array<string>,Array<string>>} */
async #transform(chunk, controller) {
- const row = this.#rawInput ? chunk : JSON.parse(chunk);
-
// If row is an array of objects, then the transformation has "normalized" the data and there is no header row.
// Instead, the header data has been incorporated into the object keys. The transformation is responsible
// for "denormalizing" the object back into an (extra) header row in the output. Ignore `handleHeaders`.
- if (typeof row[0] === 'object') {
+ if (typeof chunk[0] === 'object') {
this.#firstChunk = false;
}
if (this.#firstChunk) {
this.#firstChunk = false;
- const out = await this.#wrappedHeaderFn(row);
+ const out = await this.#wrappedHeaderFn(chunk);
this.#enqueueRow(out, controller);
return;
}
if (this.#maxBatchSize !== null) {
- this.#batch.push(row);
+ this.#batch.push(chunk);
if (this.#batch.length === this.#maxBatchSize) {
this.#concurrent.push(this.#wrappedFn(this.#batch));
this.#batch.length = 0;
}
}
else {
- this.#concurrent.push(this.#wrappedFn(row));
+ this.#concurrent.push(this.#wrappedFn(chunk));
}
if (this.#concurrent.length === this.#maxConcurrent) {
}
}
- /** @type {import('node:stream/web').TransformerFlushCallback<string>} */
+ /** @type {import('node:stream/web').TransformerFlushCallback<Array<string>>} */
async #flush(controller) {
if (this.#batch.length > 0) {
this.#concurrent.push(this.#wrappedFn(this.#batch));
},
async write(chunk) {
let data;
- try {
- const parsed = JSON.parse(chunk);
- data = CSVWriter.arrayToCSVString(parsed);
- status.rows += parsed.length;
+ if (Array.isArray(chunk)) {
+ data = CSVWriter.arrayToCSVString(chunk);
+ status.rows += chunk.length;
}
- catch {
+ else {
data = chunk;
status.rows++;
}
/** @typedef {import('./core.js').CSVReaderOptions} CSVReaderOptions */
/** @typedef {import('./core.js').TransformationOutput} TransformationOutput */
/** @typedef {import('./core.js').TransformationFunction} TransformationFunction */
-/** @typedef {import('./core.js').TransformationFunctionRaw} TransformationFunctionRaw */
/** @typedef {import('./core.js').TransformationFunctionBatch} TransformationFunctionBatch */
/** @typedef {import('./core.js').TransformationErrorFunction} TransformationErrorFunction */
-/** @typedef {import('./core.js').TransformationErrorFunctionRaw} TransformationErrorFunctionRaw */
/** @typedef {import('./core.js').TransformationErrorFunctionBatch} TransformationErrorFunctionBatch */
/** @typedef {import('./core.js').CSVTransformerOptions} CSVTransformerOptions */
-/** @typedef {import('./core.js').CSVTransformerOptionsRaw} CSVTransformerOptionsRaw */
/** @typedef {import('./core.js').CSVTransformerOptionsBatch} CSVTransformerOptionsBatch */
/** @typedef {import('./core.js').CSVWriterStatus} CSVWriterStatus */
/** @typedef {import('./normalization.js').CSVNormalizerHeader} CSVNormalizerHeader */
}
}
- /** @type {import('node:stream/web').TransformerTransformCallback<string,string>} */
+ /** @type {import('node:stream/web').TransformerTransformCallback<Array<string>,Array<CSVNormalizerRow>>} */
#transform(chunk, controller) {
- const row = JSON.parse(chunk);
-
// Assume first row is headers and use it to prepare the columns object
// Note: the headers row is NOT forwarded downstream
if (this.#firstChunk) {
- const normalizedRow = this.#useLiteralNames ? row : row.map(f => toCamelCase(f));
+ const normalizedRow = this.#useLiteralNames ? chunk : chunk.map(f => toCamelCase(f));
let i = 0;
for (const header of normalizedRow) {
const col = this.#columns.find(c => c.name === header);
/** @type {Array<CSVNormalizerRow>} */
const out = [];
for (const { name, type, displayName, defaultValue, index } of this.#columns) {
- let value = row[index] ?? '';
+ let value = chunk[index] ?? '';
const emptyField = value === '' ? true : false;
switch (type) {
case 'string': break;
if (!this.#passthroughEmptyRows && out.every(f => f.emptyField)) { return; }
- controller.enqueue(JSON.stringify(out));
+ controller.enqueue(out);
}
/**
constructor() {
let firstRow = true;
super({
- /** @type {import('node:stream/web').TransformerTransformCallback<string,string>} */
+ /** @type {import('node:stream/web').TransformerTransformCallback<Array<CSVNormalizerRow>,Array<string>>} */
transform(chunk, controller) {
- const row = JSON.parse(chunk);
if (firstRow) {
firstRow = false;
- controller.enqueue(JSON.stringify(row.map(f => f?.displayName ?? f.name)));
+ controller.enqueue(chunk.map(f => f?.displayName ?? f.name));
}
- controller.enqueue(JSON.stringify(row.map(f => f.value)));
+ controller.enqueue(chunk.map(f => f.value instanceof Date ? f.value.toISOString() : f.value));
}
});
}
.pipeThrough(new CSVTransformer(r => r))
.pipeTo(csvStreamEqualWritable(csv));
});
- test('passes through raw input', { concurrency: true }, async () => {
- await ReadableStream.from((function* () {
- yield '["headerA","headerB"]';
- yield '["111","222"]';
- })())
- .pipeThrough(new CSVTransformer(r => r, { handleHeaders: true, rawInput: true }))
- .pipeTo(csvStreamEqualWritable([
- ['headerA', 'headerB'],
- ['111', '222']
- ]));
- });
- test('passes through raw output', { concurrency: true }, async () => {
- await createCSVMockStream([
- ['columnA', 'columnB'],
- ['a', 'b']
- ])
- .pipeThrough(new CSVTransformer(() => '["abc", "def"]', { handleHeaders: true, rawOutput: true }))
- .pipeTo(csvStreamEqualWritable([
- ['abc', 'def'],
- ['abc', 'def']
- ]));
- });
test('handles ArrayLike outputs', { concurrency: true }, async () => {
await createCSVMockStream([
['columnA', 'columnB'],
const temp = await createTempFile();
t.after(async () => await unlink(temp));
await createCSVMockStream([['']])
- .pipeThrough(new CSVTransformer(() => 'row 1,a\r\nrow 2,b\r\nrow 3,c', { handleHeaders: true, rawOutput: true }))
+ .pipeThrough(new CSVTransformer(() => 'row 1,a\r\nrow 2,b\r\nrow 3,c', { handleHeaders: true }))
.pipeTo(new CSVWriter(temp));
await new CSVReader(temp).pipeTo(
csvStreamEqualWritable([
const writer = new CSVWriter(temp);
await rejects(async () => {
await ReadableStream.from((function* () {
- yield '["header1","header2"]';
- yield '["1","1"]';
+ yield ['header1', 'header2'];
+ yield ['1', '1'];
throw new Error();
})()).pipeTo(writer)
});
t.after(async () => await unlink(temp));
const writer = new CSVWriter(temp);
await createCSVMockStream([['']])
- .pipeThrough(new CSVTransformer(() => 'row 1,a\r\nrow 2,b\r\nrow 3,c', { handleHeaders: true, rawOutput: true }))
+ .pipeThrough(new CSVTransformer(() => 'row 1,a\r\nrow 2,b\r\nrow 3,c', { handleHeaders: true }))
.pipeTo(writer);
deepStrictEqual(writer.status.rows, 1);
await new CSVReader(temp)
const reader = new CSVReader(temp);
let rowCount = 0;
for await (const row of reader) {
- const parsed = JSON.parse(row);
if (rowCount === 0) {
- deepStrictEqual(parsed[0], '1717370683'); // A1
- deepStrictEqual(parsed[49], '1159187275'); // AX1
- deepStrictEqual(parsed[99], '398856332'); // CV1
+ deepStrictEqual(row[0], '1717370683'); // A1
+ deepStrictEqual(row[49], '1159187275'); // AX1
+ deepStrictEqual(row[99], '398856332'); // CV1
}
else if (rowCount === 49_999) {
- deepStrictEqual(parsed[0], '1950971124'); // A50000
- deepStrictEqual(parsed[49], '1212575531'); // AX50000
- deepStrictEqual(parsed[99], '135758415'); // CV50000
+ deepStrictEqual(row[0], '1950971124'); // A50000
+ deepStrictEqual(row[49], '1212575531'); // AX50000
+ deepStrictEqual(row[99], '135758415'); // CV50000
}
else if (rowCount === 99_999) {
- deepStrictEqual(parsed[0], '737583361'); // A100000
- deepStrictEqual(parsed[49], '793036957'); // AX100000
- deepStrictEqual(parsed[99], '1923608001'); // CV100000
+ deepStrictEqual(row[0], '737583361'); // A100000
+ deepStrictEqual(row[49], '793036957'); // AX100000
+ deepStrictEqual(row[99], '1923608001'); // CV100000
}
rowCount++;
}
.pipeThrough(new CSVTransformer(r => r.map(f => Number(f) + 1), { handleHeaders: true }));
let rowCount = 0;
for await (const row of reader) {
- const parsed = JSON.parse(row);
if (rowCount === 0) {
- deepStrictEqual(parsed[0], 1717370684);
- deepStrictEqual(parsed[49], 1159187276);
- deepStrictEqual(parsed[99], 398856333);
+ deepStrictEqual(row[0], 1717370684);
+ deepStrictEqual(row[49], 1159187276);
+ deepStrictEqual(row[99], 398856333);
}
else if (rowCount === 49_999) {
- deepStrictEqual(parsed[0], 1950971125);
- deepStrictEqual(parsed[49], 1212575532);
- deepStrictEqual(parsed[99], 135758416);
+ deepStrictEqual(row[0], 1950971125);
+ deepStrictEqual(row[49], 1212575532);
+ deepStrictEqual(row[99], 135758416);
}
else if (rowCount === 99_999) {
- deepStrictEqual(parsed[0], 737583362);
- deepStrictEqual(parsed[49], 793036958);
- deepStrictEqual(parsed[99], 1923608002);
+ deepStrictEqual(row[0], 737583362);
+ deepStrictEqual(row[49], 793036958);
+ deepStrictEqual(row[99], 1923608002);
}
rowCount++;
}
const reader = new CSVReader(temp);
let rowCount = 0;
for await (const row of reader) {
- const parsed = JSON.parse(row);
if (rowCount === 0) {
- deepStrictEqual(parsed[0], '1717370683');
- deepStrictEqual(parsed[49_999], '689610881');
- deepStrictEqual(parsed[99_999], '183082010');
+ deepStrictEqual(row[0], '1717370683');
+ deepStrictEqual(row[49_999], '689610881');
+ deepStrictEqual(row[99_999], '183082010');
}
else if (rowCount === 49) {
- deepStrictEqual(parsed[0], '1521260985');
- deepStrictEqual(parsed[49_999], '1397216552');
- deepStrictEqual(parsed[99_999], '817708515');
+ deepStrictEqual(row[0], '1521260985');
+ deepStrictEqual(row[49_999], '1397216552');
+ deepStrictEqual(row[99_999], '817708515');
}
else if (rowCount === 99) {
- deepStrictEqual(parsed[0], '1749947568');
- deepStrictEqual(parsed[49_999], '2119460449');
- deepStrictEqual(parsed[99_999], '127114681');
+ deepStrictEqual(row[0], '1749947568');
+ deepStrictEqual(row[49_999], '2119460449');
+ deepStrictEqual(row[99_999], '127114681');
}
rowCount++;
}
.pipeThrough(new CSVTransformer(r => r.map(f => Number(f) + 1), { handleHeaders: true }));
let rowCount = 0;
for await (const row of reader) {
- const parsed = JSON.parse(row);
if (rowCount === 0) {
- deepStrictEqual(parsed[0], 1717370684);
- deepStrictEqual(parsed[49_999], 689610882);
- deepStrictEqual(parsed[99_999], 183082011);
+ deepStrictEqual(row[0], 1717370684);
+ deepStrictEqual(row[49_999], 689610882);
+ deepStrictEqual(row[99_999], 183082011);
}
else if (rowCount === 49) {
- deepStrictEqual(parsed[0], 1521260986);
- deepStrictEqual(parsed[49_999], 1397216553);
- deepStrictEqual(parsed[99_999], 817708516);
+ deepStrictEqual(row[0], 1521260986);
+ deepStrictEqual(row[49_999], 1397216553);
+ deepStrictEqual(row[99_999], 817708516);
}
else if (rowCount === 99) {
- deepStrictEqual(parsed[0], 1749947569);
- deepStrictEqual(parsed[49_999], 2119460450);
- deepStrictEqual(parsed[99_999], 127114682);
+ deepStrictEqual(row[0], 1749947569);
+ deepStrictEqual(row[49_999], 2119460450);
+ deepStrictEqual(row[99_999], 127114682);
}
rowCount++;
}
let i = 0;
return new WritableStream({
write(chunk) {
- const row = JSON.parse(chunk);
- deepStrictEqual(row.length, csv[i].length);
+ deepStrictEqual(chunk.length, csv[i].length);
for (let j = 0; j < csv[i].length; j++) {
- deepStrictEqual(row[j], csv[i][j]);
+ deepStrictEqual(chunk[j], csv[i][j]);
}
i++;
}
let i = 0;
return new WritableStream({
write(chunk) {
- const row = JSON.parse(chunk);
for (let j = 0; j < csv[i].length; j++) {
- notStrictEqual(row[j], csv[i][j]);
+ notStrictEqual(chunk[j], csv[i][j]);
}
i++;
}
controller.close();
}
else {
- controller.enqueue(JSON.stringify(value));
+ controller.enqueue(value.slice());
}
}
});
const data = new Uint32Array(columns);
getPseudoRandomValues(data, seed);
seed = data[columns - 1];
- yield JSON.stringify(Array.from(data));
+ yield Array.from(data);
}
})();
return ReadableStream.from(chunks);
git.ardis.lu / csvio / commitdiff