From: Ardis Lu Date: Mon, 4 Aug 2025 06:06:34 +0000 (-0700) Subject: Remove intermediate JSON serialization and `rawInput`/`rawOutput` options X-Git-Tag: v0.1.5~12 X-Git-Url: https://git.ardis.lu/?a=commitdiff_plain;h=81c11c17ebcb6d01cca152d578997567eef5a556;p=csvio Remove intermediate JSON serialization and `rawInput`/`rawOutput` options Previously, each row was JSON serialized and deserialized on each `.pipeThrough`. The intent of this serialization was to normalize the data of each row, with an escape hatch in the `rawInput` and `rawOutput` options. However, this approach is counterintuitive because most users would expect a CSV row to simply be an `Array`, NOT a `string` that can be `JSON.parse`-ed into an `Array`. Furthermore, this excessive serialization has a significant performance impact. Lastly, the `rawInput` and `rawOutput` options effectively made `CSVTransformer` an overly complex and difficult-to-use JSON parser on top of a CSV transformer. So, simply removing the JSON serialization and directly enqueuing the actual object (i.e., the `Array`) resolves all these issues with minimal impact to the user-facing API. --- diff --git a/README.md b/README.md index b527b1a..1815a70 100644 --- a/README.md +++ b/README.md @@ -97,12 +97,12 @@ const readableStream = new CSVReader('./examples/data/ex1_1-in.csv'); for await (const row of readableStream) { console.log(row); } -// ["columnA","columnB"] -// ["1","1"] -// ["100","100"] -// ["223423","455947"] -// ["348553","692708"] -// ["536368","676147"] +// [ 'columnA', 'columnB' ] +// [ '1', '1' ] +// [ '100', '100' ] +// [ '223423', '455947' ] +// [ '348553', '692708' ] +// [ '536368', '676147' ] ``` ### `CSVTransformer` @@ -119,19 +119,19 @@ function edit(row) { const transformStream = new CSVTransformer(edit); const rows = ReadableStream.from((function* () { - yield '["header"]'; - yield '["1"]'; - yield '["2"]'; - yield '["3"]'; + yield ['header']; + yield ['1']; + yield ['2']; + yield ['3']; })()).pipeThrough(transformStream); for await (const row of rows) { console.log(row); } -// ["header"] -// ["1: transformed!"] -// ["2: transformed!"] -// ["3: transformed!"] +// [ 'header' ] +// [ '1: transformed!' ] +// [ '2: transformed!' ] +// [ '3: transformed!' ] ``` ### `CSVWriter` @@ -144,10 +144,10 @@ import { CSVWriter } from '@ardislu/csvio'; const writableStream = new CSVWriter('./example.csv'); await ReadableStream.from((function* () { - yield '["header1","header2"]'; - yield '["1","1"]'; - yield '["2","2"]'; - yield '["3","3"]'; + yield ['header1', 'header2']; + yield ['1', '1']; + yield ['2', '2']; + yield ['3', '3']; })()).pipeTo(writableStream); // A new ./example.csv file is created with the CSV data ``` @@ -176,11 +176,11 @@ const headers = [ ] const rows = ReadableStream.from((function* () { - yield '["numberCol","dateCol","",""]'; // Empty columns inserted - yield '["1E+18","45292","",""]'; // Number mangled to scientific notation; date mangled to number - yield '["","","",""]'; // Empty rows inserted - yield '["","","",""]'; - yield '["","","",""]'; + yield ['numberCol', 'dateCol', '', '']; // Empty columns inserted + yield ['1E+18', '45292', '', '']; // Number mangled to scientific notation; date mangled to number + yield ['', '', '', '']; // Empty rows inserted + yield ['', '', '', '']; + yield ['', '', '', '']; })()) .pipeThrough(new CSVNormalizer(headers)) .pipeThrough(new CSVDenormalizer()); @@ -188,8 +188,8 @@ const rows = ReadableStream.from((function* () { for await (const row of rows) { console.log(row); } -// ["Number Column","Date Column"] -// [1000000000000000000,"2024-01-01T00:00:00.000Z"] +// [ 'Number Column', 'Date Column' ] +// [ 1000000000000000000, '2024-01-01T00:00:00.000Z' ] ``` ## Tests diff --git a/examples/ex1_3.js b/examples/ex1_3.js index c19486f..f325c39 100644 --- a/examples/ex1_3.js +++ b/examples/ex1_3.js @@ -1,9 +1,8 @@ // ## Example 1.3: Chaining multiple transformations // -// Set `rawInput: true` and/or `rawOutput: true` to skip JSON deserialization (`rawInput`) and/or JSON serialization -// (`rawOutput`) when handling the transformation function input/output. These options are intended for advanced -// transformations (e.g., transformations spread across multiple functions or lower-level transformations directly -// on byte arrays). +// For advanced transformations (e.g., transformations spread across multiple functions or lower-level transformations +// directly on byte arrays), manually serialize and deserialize your data (e.g., using `JSON.stringify` and `JSON.parse`). +// A raw `string` type will be "passed through" `CSVTransformer` for your transformation function to handle directly. // // Set `handleHeaders` to `true` to pass the header row to the transformation function as a normal row and manually // handle the header output processing. This setting is required if the input CSV does not have a header row. @@ -14,18 +13,18 @@ import { CSVReader, CSVTransformer, CSVWriter } from '../src/index.js'; function parse(row) { const [name, id] = row[0].split('_'); const [first, last] = name.split('.') - return JSON.stringify({ first, last, id }); // Manually handling serialization here, so use `rawOutput` to turn off automatic serialization + return JSON.stringify({ first, last, id }); // Manually handle serialization here } // Contains the logic for the transformation function calculate(obj) { - const { first, last, id } = JSON.parse(obj); // Manually handling deserialization here, so use `rawInput` to turn off automatic deserialization + const { first, last, id } = JSON.parse(obj); // Manually handle deserialization here const name = `${last}, ${first}`; const paddedId = id.padStart(6, '0'); return [name, paddedId]; } await new CSVReader(new URL('./data/ex1_3-in.csv', import.meta.url)) - .pipeThrough(new CSVTransformer(parse, { handleHeaders: true, rawOutput: true })) - .pipeThrough(new CSVTransformer(calculate, { handleHeaders: true, rawInput: true })) + .pipeThrough(new CSVTransformer(parse, { handleHeaders: true })) + .pipeThrough(new CSVTransformer(calculate, { handleHeaders: true })) .pipeTo(new CSVWriter(new URL('./data/ex1_3-out.csv', import.meta.url))); \ No newline at end of file diff --git a/src/core.js b/src/core.js index d620724..e42d5a5 100644 --- a/src/core.js +++ b/src/core.js @@ -112,7 +112,7 @@ function createProxy(primary, underlying) { /** * A simple streaming parser for CSV files. * - * Each chunk is one row of the CSV file, in the form of a string that may be deserialized using `JSON.parse()`. + * Each chunk is one row of the CSV file as an `Array`. */ export class CSVReader { /** @type {Array} */ @@ -128,16 +128,16 @@ export class CSVReader { * @param {PathLike} path A `string`, `Buffer`, or `URL` representing a path to a local CSV file or * `null` to create a `TransformStream`. * @param {CSVReaderOptions} [options] Object containing flags to configure the reader. - * @returns {CSVReader&ReadableStream} + * @returns {CSVReader&ReadableStream>} */ /** * @overload * @param {null} [path] A `string`, `Buffer`, or `URL` representing a path to a local CSV file or * `null` to create a `TransformStream`. - * @returns {CSVReader&TransformStream} + * @returns {CSVReader&TransformStream>} */ constructor(path = null, options = {}) { - /** @type {ReadableStream|TransformStream} */ + /** @type {ReadableStream>|TransformStream>} */ let stream; stream = new TransformStream({ transform: (chunk, controller) => this.#transform(chunk, controller), @@ -153,7 +153,7 @@ export class CSVReader { return createProxy(this, stream); } - /** @type {import('node:stream/web').TransformerTransformCallback} */ + /** @type {import('node:stream/web').TransformerTransformCallback>} */ #transform(chunk, controller) { for (const char of chunk) { /* Escape mode logic */ @@ -189,7 +189,7 @@ export class CSVReader { else if (char === '\n') { // Terminate the field and row and do not add char to field this.#row.push(this.#field); this.#field = ''; - controller.enqueue(JSON.stringify(this.#row)); + controller.enqueue(this.#row.slice()); this.#row.length = 0; } else { // No special cases match, add the literal char @@ -201,14 +201,14 @@ export class CSVReader { } } - /** @type {import('node:stream/web').TransformerFlushCallback} */ + /** @type {import('node:stream/web').TransformerFlushCallback>} */ #flush(controller) { if (this.#field !== '' || this.#row.length !== 0) { // CSV terminated without a trailing CRLF, leaving a row in the queue this.#row.push(this.#field); // Last field is always un-pushed if CSV terminated with nothing - controller.enqueue(JSON.stringify(this.#row)); + controller.enqueue(this.#row.slice()); } else if (this.#lastChar === null) { // CSV is blank - controller.enqueue(JSON.stringify([''])); + controller.enqueue(['']); } } } @@ -225,13 +225,6 @@ export class CSVReader { * @returns {TransformationOutput|Promise} */ -/** - * A function to process a row or rows of raw CSV data from `CSVReader`. - * @callback TransformationFunctionRaw - * @param {string} row A row or rows of input CSV data represented as a raw `string`. - * @returns {TransformationOutput|Promise} - */ - /** * A function to process multiple rows of CSV data from `CSVReader`. * @callback TransformationFunctionBatch @@ -248,15 +241,6 @@ export class CSVReader { * @returns {TransformationOutput|Promise} */ -/** - * A function to handle errors thrown by a transformation function. - * @callback TransformationErrorFunctionRaw - * @param {string} row A row or rows of raw input CSV data passed to the transformation function which threw the error. - * @param {Error} error The error thrown by the transformation function. - * @param {TransformationFunctionRaw} fn The transformation function itself. This argument can be used to retry a transformation. - * @returns {TransformationOutput|Promise} - */ - /** * A function to handle errors thrown by a transformation function. * @callback TransformationErrorFunctionBatch @@ -275,10 +259,6 @@ export class CSVReader { * to the `TransformationFunction`. The default value is `false`. * * NOTE: If the CSV has no header row, set `handleHeaders` to `true` and process the first row normally in `fn()`. - * @property {false} [rawInput] Set to `true` to send the raw CSV row to `fn()` as a `string`. Otherwise, the CSV row will be deserialized - * using `JSON.parse()` before being sent to `fn()`. The default value is `false`. - * @property {boolean} [rawOutput=false] Set to `true` to send the raw return value of `fn()` to the next stream. Otherwise, the return value of - * `fn()` will be serialized using `JSON.stringify()` before being sent to the next stream. The default value is `false`. * @property {null|TransformationErrorFunction} [onError=null] Set to a function to catch errors thrown by the transformation function. The * input data, the error that was thrown, and the transformation function itself will be passed to the `onError` function. The default value is * `null` (errors will not be caught). @@ -288,30 +268,6 @@ export class CSVReader { * resolved). The default value is `1`. */ -/** - * Options to configure `CSVTransformer`. - * @typedef {Object} CSVTransformerOptionsRaw - * @property {boolean|Array|TransformationFunctionRaw} [handleHeaders=false] A `boolean`, `Array`, or `TransformationFunctionRaw` to - * process the header row (first row) of the CSV. If `false`, the header row will pass through the stream. If `true`, the header row will be passed - * to `fn()`. If `Array`, the header row will be replaced with the `Array`. If `TransformationFunctionRaw`, the header row will be passed - * to the `TransformationFunctionRaw` as a `string`. The default value is `false`. - * - * NOTE: If the CSV has no header row, set `handleHeaders` to `true` and process the first row normally in `fn()`. - * @property {true} rawInput Set to `true` to send the raw CSV row to `fn()` as a `string`. Otherwise, the CSV row will be deserialized - * using `JSON.parse()` before being sent to `fn()`. The default value is `false`. - * @property {boolean} [rawOutput=false] Set to `true` to send the raw return value of `fn()` to the next stream. Otherwise, the return value of - * `fn()` will be serialized using `JSON.stringify()` before being sent to the next stream. The default value is `false`. - * @property {null|TransformationErrorFunctionRaw} [onError=null] Set to a function to catch errors thrown by the transformation function. The - * input data, the error that was thrown, and the transformation function itself will be passed to the `onError` function. The default value is - * `null` (errors will not be caught). - * @property {number} [maxBatchSize=1] Set to the maximum number of rows that will be passed to the transformation function per function call - * (greedy). - * @property {number} [maxConcurrent=1] The maximum concurrent executions of the transformation function (greedy). The transformation function - * will be automatically turned into a promise if it isn't already async. Execution is blocked until all promises in a concurrent group settle - * (i.e., if one transformation in a group is hanging, further rows will NOT be processed even if all other transformations in the group are - * resolved). The default value is `1`. - */ - /** * Options to configure `CSVTransformer`. * @typedef {Object} CSVTransformerOptionsBatch @@ -321,10 +277,6 @@ export class CSVReader { * to the `TransformationFunction`. The default value is `false`. * * NOTE: If the CSV has no header row, set `handleHeaders` to `true` and process the first row normally in `fn()`. - * @property {false} [rawInput] Set to `true` to send the raw CSV row to `fn()` as a `string`. Otherwise, the CSV row will be deserialized - * using `JSON.parse()` before being sent to `fn()`. The default value is `false`. - * @property {boolean} [rawOutput=false] Set to `true` to send the raw return value of `fn()` to the next stream. Otherwise, the return value of - * `fn()` will be serialized using `JSON.stringify()` before being sent to the next stream. The default value is `false`. * @property {null|TransformationErrorFunctionBatch} [onError=null] Set to a function * to catch errors thrown by the transformation function. The input data, the error that was thrown, and the transformation function itself will * be passed to the `onError` function. The default value is `null` (errors will not be caught). @@ -351,8 +303,6 @@ export class CSVTransformer extends TransformStream { #firstChunk = true; #handleHeaders; - #rawInput; - #rawOutput; #onError; #maxBatchSize; #maxConcurrent; @@ -366,11 +316,6 @@ export class CSVTransformer extends TransformStream { * @param {TransformationFunction} fn A function to process a row of CSV data. * @param {CSVTransformerOptions} options Object containing flags to configure the stream logic. */ - /** - * @overload - * @param {TransformationFunctionRaw} fn A function to process a row or rows of CSV data represented as a raw `string`. - * @param {CSVTransformerOptionsRaw} options Object containing flags to configure the stream logic. - */ /** * @overload * @param {TransformationFunctionBatch} fn A function to process multiple rows of CSV data from `CSVReader`. @@ -383,8 +328,6 @@ export class CSVTransformer extends TransformStream { }); this.#handleHeaders = options.handleHeaders ?? false; - this.#rawInput = options.rawInput ?? false; - this.#rawOutput = options.rawOutput ?? false; this.#onError = options.onError ?? null; this.#maxBatchSize = options.maxBatchSize ?? null; this.#maxConcurrent = options.maxConcurrent ?? 1; @@ -414,19 +357,19 @@ export class CSVTransformer extends TransformStream { if (row === null || row === undefined) { // Input row is consumed without emitting any output row return; } - else if (this.#rawOutput || typeof row === 'string') { + else if (typeof row === 'string') { controller.enqueue(row); } else if (Array.isArray(row)) { if (Array.isArray(row[0])) { // Multiple rows returned, enqueue each row separately - row.forEach(r => controller.enqueue(JSON.stringify(r))); + row.forEach(r => controller.enqueue(r.slice())); } else { // One row returned - controller.enqueue(JSON.stringify(row)); + controller.enqueue(row.slice()); } } else { // Assume ArrayLike output - controller.enqueue(JSON.stringify(Array.from(row))); + controller.enqueue(Array.from(row)); } } @@ -448,33 +391,31 @@ export class CSVTransformer extends TransformStream { } } - /** @type {import('node:stream/web').TransformerTransformCallback} */ + /** @type {import('node:stream/web').TransformerTransformCallback,Array>} */ async #transform(chunk, controller) { - const row = this.#rawInput ? chunk : JSON.parse(chunk); - // If row is an array of objects, then the transformation has "normalized" the data and there is no header row. // Instead, the header data has been incorporated into the object keys. The transformation is responsible // for "denormalizing" the object back into an (extra) header row in the output. Ignore `handleHeaders`. - if (typeof row[0] === 'object') { + if (typeof chunk[0] === 'object') { this.#firstChunk = false; } if (this.#firstChunk) { this.#firstChunk = false; - const out = await this.#wrappedHeaderFn(row); + const out = await this.#wrappedHeaderFn(chunk); this.#enqueueRow(out, controller); return; } if (this.#maxBatchSize !== null) { - this.#batch.push(row); + this.#batch.push(chunk); if (this.#batch.length === this.#maxBatchSize) { this.#concurrent.push(this.#wrappedFn(this.#batch)); this.#batch.length = 0; } } else { - this.#concurrent.push(this.#wrappedFn(row)); + this.#concurrent.push(this.#wrappedFn(chunk)); } if (this.#concurrent.length === this.#maxConcurrent) { @@ -482,7 +423,7 @@ export class CSVTransformer extends TransformStream { } } - /** @type {import('node:stream/web').TransformerFlushCallback} */ + /** @type {import('node:stream/web').TransformerFlushCallback>} */ async #flush(controller) { if (this.#batch.length > 0) { this.#concurrent.push(this.#wrappedFn(this.#batch)); @@ -537,12 +478,11 @@ export class CSVWriter extends WritableStream { }, async write(chunk) { let data; - try { - const parsed = JSON.parse(chunk); - data = CSVWriter.arrayToCSVString(parsed); - status.rows += parsed.length; + if (Array.isArray(chunk)) { + data = CSVWriter.arrayToCSVString(chunk); + status.rows += chunk.length; } - catch { + else { data = chunk; status.rows++; } diff --git a/src/index.js b/src/index.js index 514f46a..c7c2135 100644 --- a/src/index.js +++ b/src/index.js @@ -5,13 +5,10 @@ export { ErrorStrategies } from './errorStrategies.js'; /** @typedef {import('./core.js').CSVReaderOptions} CSVReaderOptions */ /** @typedef {import('./core.js').TransformationOutput} TransformationOutput */ /** @typedef {import('./core.js').TransformationFunction} TransformationFunction */ -/** @typedef {import('./core.js').TransformationFunctionRaw} TransformationFunctionRaw */ /** @typedef {import('./core.js').TransformationFunctionBatch} TransformationFunctionBatch */ /** @typedef {import('./core.js').TransformationErrorFunction} TransformationErrorFunction */ -/** @typedef {import('./core.js').TransformationErrorFunctionRaw} TransformationErrorFunctionRaw */ /** @typedef {import('./core.js').TransformationErrorFunctionBatch} TransformationErrorFunctionBatch */ /** @typedef {import('./core.js').CSVTransformerOptions} CSVTransformerOptions */ -/** @typedef {import('./core.js').CSVTransformerOptionsRaw} CSVTransformerOptionsRaw */ /** @typedef {import('./core.js').CSVTransformerOptionsBatch} CSVTransformerOptionsBatch */ /** @typedef {import('./core.js').CSVWriterStatus} CSVWriterStatus */ /** @typedef {import('./normalization.js').CSVNormalizerHeader} CSVNormalizerHeader */ diff --git a/src/normalization.js b/src/normalization.js index 69d91ba..0432288 100644 --- a/src/normalization.js +++ b/src/normalization.js @@ -143,14 +143,12 @@ export class CSVNormalizer extends TransformStream { } } - /** @type {import('node:stream/web').TransformerTransformCallback} */ + /** @type {import('node:stream/web').TransformerTransformCallback,Array>} */ #transform(chunk, controller) { - const row = JSON.parse(chunk); - // Assume first row is headers and use it to prepare the columns object // Note: the headers row is NOT forwarded downstream if (this.#firstChunk) { - const normalizedRow = this.#useLiteralNames ? row : row.map(f => toCamelCase(f)); + const normalizedRow = this.#useLiteralNames ? chunk : chunk.map(f => toCamelCase(f)); let i = 0; for (const header of normalizedRow) { const col = this.#columns.find(c => c.name === header); @@ -167,7 +165,7 @@ export class CSVNormalizer extends TransformStream { /** @type {Array} */ const out = []; for (const { name, type, displayName, defaultValue, index } of this.#columns) { - let value = row[index] ?? ''; + let value = chunk[index] ?? ''; const emptyField = value === '' ? true : false; switch (type) { case 'string': break; @@ -183,7 +181,7 @@ export class CSVNormalizer extends TransformStream { if (!this.#passthroughEmptyRows && out.every(f => f.emptyField)) { return; } - controller.enqueue(JSON.stringify(out)); + controller.enqueue(out); } /** @@ -308,14 +306,13 @@ export class CSVDenormalizer extends TransformStream { constructor() { let firstRow = true; super({ - /** @type {import('node:stream/web').TransformerTransformCallback} */ + /** @type {import('node:stream/web').TransformerTransformCallback,Array>} */ transform(chunk, controller) { - const row = JSON.parse(chunk); if (firstRow) { firstRow = false; - controller.enqueue(JSON.stringify(row.map(f => f?.displayName ?? f.name))); + controller.enqueue(chunk.map(f => f?.displayName ?? f.name)); } - controller.enqueue(JSON.stringify(row.map(f => f.value))); + controller.enqueue(chunk.map(f => f.value instanceof Date ? f.value.toISOString() : f.value)); } }); } diff --git a/test/core.test.js b/test/core.test.js index 329eb3a..e549962 100644 --- a/test/core.test.js +++ b/test/core.test.js @@ -324,28 +324,6 @@ suite('CSVTransformer', { concurrency: true }, () => { .pipeThrough(new CSVTransformer(r => r)) .pipeTo(csvStreamEqualWritable(csv)); }); - test('passes through raw input', { concurrency: true }, async () => { - await ReadableStream.from((function* () { - yield '["headerA","headerB"]'; - yield '["111","222"]'; - })()) - .pipeThrough(new CSVTransformer(r => r, { handleHeaders: true, rawInput: true })) - .pipeTo(csvStreamEqualWritable([ - ['headerA', 'headerB'], - ['111', '222'] - ])); - }); - test('passes through raw output', { concurrency: true }, async () => { - await createCSVMockStream([ - ['columnA', 'columnB'], - ['a', 'b'] - ]) - .pipeThrough(new CSVTransformer(() => '["abc", "def"]', { handleHeaders: true, rawOutput: true })) - .pipeTo(csvStreamEqualWritable([ - ['abc', 'def'], - ['abc', 'def'] - ])); - }); test('handles ArrayLike outputs', { concurrency: true }, async () => { await createCSVMockStream([ ['columnA', 'columnB'], @@ -682,7 +660,7 @@ suite('CSVWriter', { concurrency: true }, () => { const temp = await createTempFile(); t.after(async () => await unlink(temp)); await createCSVMockStream([['']]) - .pipeThrough(new CSVTransformer(() => 'row 1,a\r\nrow 2,b\r\nrow 3,c', { handleHeaders: true, rawOutput: true })) + .pipeThrough(new CSVTransformer(() => 'row 1,a\r\nrow 2,b\r\nrow 3,c', { handleHeaders: true })) .pipeTo(new CSVWriter(temp)); await new CSVReader(temp).pipeTo( csvStreamEqualWritable([ @@ -697,8 +675,8 @@ suite('CSVWriter', { concurrency: true }, () => { const writer = new CSVWriter(temp); await rejects(async () => { await ReadableStream.from((function* () { - yield '["header1","header2"]'; - yield '["1","1"]'; + yield ['header1', 'header2']; + yield ['1', '1']; throw new Error(); })()).pipeTo(writer) }); @@ -803,7 +781,7 @@ suite('CSVWriter status', { concurrency: true }, () => { t.after(async () => await unlink(temp)); const writer = new CSVWriter(temp); await createCSVMockStream([['']]) - .pipeThrough(new CSVTransformer(() => 'row 1,a\r\nrow 2,b\r\nrow 3,c', { handleHeaders: true, rawOutput: true })) + .pipeThrough(new CSVTransformer(() => 'row 1,a\r\nrow 2,b\r\nrow 3,c', { handleHeaders: true })) .pipeTo(writer); deepStrictEqual(writer.status.rows, 1); await new CSVReader(temp) diff --git a/test/large.test.js b/test/large.test.js index 3d2252b..987339a 100644 --- a/test/large.test.js +++ b/test/large.test.js @@ -34,21 +34,20 @@ suite('large: 100000 row x 100 column CSV (~100 MB)', { concurrency: 1 }, async const reader = new CSVReader(temp); let rowCount = 0; for await (const row of reader) { - const parsed = JSON.parse(row); if (rowCount === 0) { - deepStrictEqual(parsed[0], '1717370683'); // A1 - deepStrictEqual(parsed[49], '1159187275'); // AX1 - deepStrictEqual(parsed[99], '398856332'); // CV1 + deepStrictEqual(row[0], '1717370683'); // A1 + deepStrictEqual(row[49], '1159187275'); // AX1 + deepStrictEqual(row[99], '398856332'); // CV1 } else if (rowCount === 49_999) { - deepStrictEqual(parsed[0], '1950971124'); // A50000 - deepStrictEqual(parsed[49], '1212575531'); // AX50000 - deepStrictEqual(parsed[99], '135758415'); // CV50000 + deepStrictEqual(row[0], '1950971124'); // A50000 + deepStrictEqual(row[49], '1212575531'); // AX50000 + deepStrictEqual(row[99], '135758415'); // CV50000 } else if (rowCount === 99_999) { - deepStrictEqual(parsed[0], '737583361'); // A100000 - deepStrictEqual(parsed[49], '793036957'); // AX100000 - deepStrictEqual(parsed[99], '1923608001'); // CV100000 + deepStrictEqual(row[0], '737583361'); // A100000 + deepStrictEqual(row[49], '793036957'); // AX100000 + deepStrictEqual(row[99], '1923608001'); // CV100000 } rowCount++; } @@ -66,21 +65,20 @@ suite('large: 100000 row x 100 column CSV (~100 MB)', { concurrency: 1 }, async .pipeThrough(new CSVTransformer(r => r.map(f => Number(f) + 1), { handleHeaders: true })); let rowCount = 0; for await (const row of reader) { - const parsed = JSON.parse(row); if (rowCount === 0) { - deepStrictEqual(parsed[0], 1717370684); - deepStrictEqual(parsed[49], 1159187276); - deepStrictEqual(parsed[99], 398856333); + deepStrictEqual(row[0], 1717370684); + deepStrictEqual(row[49], 1159187276); + deepStrictEqual(row[99], 398856333); } else if (rowCount === 49_999) { - deepStrictEqual(parsed[0], 1950971125); - deepStrictEqual(parsed[49], 1212575532); - deepStrictEqual(parsed[99], 135758416); + deepStrictEqual(row[0], 1950971125); + deepStrictEqual(row[49], 1212575532); + deepStrictEqual(row[99], 135758416); } else if (rowCount === 99_999) { - deepStrictEqual(parsed[0], 737583362); - deepStrictEqual(parsed[49], 793036958); - deepStrictEqual(parsed[99], 1923608002); + deepStrictEqual(row[0], 737583362); + deepStrictEqual(row[49], 793036958); + deepStrictEqual(row[99], 1923608002); } rowCount++; } @@ -116,21 +114,20 @@ suite('large: 100 row x 100000 column CSV (~100 MB)', { concurrency: 1 }, async const reader = new CSVReader(temp); let rowCount = 0; for await (const row of reader) { - const parsed = JSON.parse(row); if (rowCount === 0) { - deepStrictEqual(parsed[0], '1717370683'); - deepStrictEqual(parsed[49_999], '689610881'); - deepStrictEqual(parsed[99_999], '183082010'); + deepStrictEqual(row[0], '1717370683'); + deepStrictEqual(row[49_999], '689610881'); + deepStrictEqual(row[99_999], '183082010'); } else if (rowCount === 49) { - deepStrictEqual(parsed[0], '1521260985'); - deepStrictEqual(parsed[49_999], '1397216552'); - deepStrictEqual(parsed[99_999], '817708515'); + deepStrictEqual(row[0], '1521260985'); + deepStrictEqual(row[49_999], '1397216552'); + deepStrictEqual(row[99_999], '817708515'); } else if (rowCount === 99) { - deepStrictEqual(parsed[0], '1749947568'); - deepStrictEqual(parsed[49_999], '2119460449'); - deepStrictEqual(parsed[99_999], '127114681'); + deepStrictEqual(row[0], '1749947568'); + deepStrictEqual(row[49_999], '2119460449'); + deepStrictEqual(row[99_999], '127114681'); } rowCount++; } @@ -148,21 +145,20 @@ suite('large: 100 row x 100000 column CSV (~100 MB)', { concurrency: 1 }, async .pipeThrough(new CSVTransformer(r => r.map(f => Number(f) + 1), { handleHeaders: true })); let rowCount = 0; for await (const row of reader) { - const parsed = JSON.parse(row); if (rowCount === 0) { - deepStrictEqual(parsed[0], 1717370684); - deepStrictEqual(parsed[49_999], 689610882); - deepStrictEqual(parsed[99_999], 183082011); + deepStrictEqual(row[0], 1717370684); + deepStrictEqual(row[49_999], 689610882); + deepStrictEqual(row[99_999], 183082011); } else if (rowCount === 49) { - deepStrictEqual(parsed[0], 1521260986); - deepStrictEqual(parsed[49_999], 1397216553); - deepStrictEqual(parsed[99_999], 817708516); + deepStrictEqual(row[0], 1521260986); + deepStrictEqual(row[49_999], 1397216553); + deepStrictEqual(row[99_999], 817708516); } else if (rowCount === 99) { - deepStrictEqual(parsed[0], 1749947569); - deepStrictEqual(parsed[49_999], 2119460450); - deepStrictEqual(parsed[99_999], 127114682); + deepStrictEqual(row[0], 1749947569); + deepStrictEqual(row[49_999], 2119460450); + deepStrictEqual(row[99_999], 127114682); } rowCount++; } diff --git a/test/utils.js b/test/utils.js index b14a1c4..948bfbc 100644 --- a/test/utils.js +++ b/test/utils.js @@ -19,10 +19,9 @@ export function csvStreamEqualWritable(csv) { let i = 0; return new WritableStream({ write(chunk) { - const row = JSON.parse(chunk); - deepStrictEqual(row.length, csv[i].length); + deepStrictEqual(chunk.length, csv[i].length); for (let j = 0; j < csv[i].length; j++) { - deepStrictEqual(row[j], csv[i][j]); + deepStrictEqual(chunk[j], csv[i][j]); } i++; } @@ -40,9 +39,8 @@ export function csvStreamNotEqualWritable(csv) { let i = 0; return new WritableStream({ write(chunk) { - const row = JSON.parse(chunk); for (let j = 0; j < csv[i].length; j++) { - notStrictEqual(row[j], csv[i][j]); + notStrictEqual(chunk[j], csv[i][j]); } i++; } @@ -149,7 +147,7 @@ export function createCSVMockStream(data) { controller.close(); } else { - controller.enqueue(JSON.stringify(value)); + controller.enqueue(value.slice()); } } }); @@ -222,7 +220,7 @@ export function createRandomCSV(rows, columns, seed) { const data = new Uint32Array(columns); getPseudoRandomValues(data, seed); seed = data[columns - 1]; - yield JSON.stringify(Array.from(data)); + yield Array.from(data); } })(); return ReadableStream.from(chunks);