From 33ecd556a04366cb007a0e441575bc6871936f48 Mon Sep 17 00:00:00 2001 From: Ardis Lu Date: Mon, 27 Jan 2025 23:22:04 -0800 Subject: [PATCH] Add initial large CSV tests --- README.md | 8 ++- package.json | 4 +- test/large.test.js | 126 +++++++++++++++++++++++++++++++++++++++++++++ test/utils.js | 64 ++++++++++++++++++++++- 4 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 test/large.test.js diff --git a/README.md b/README.md index 861afbb..c1a2ea2 100644 --- a/README.md +++ b/README.md @@ -150,12 +150,18 @@ Example and test CSVs use `CRLF` (`\r\n`) as the record delimiter, as suggested If your IDE is configured to automatically convert `CRLF` line endings to `LF`, you must undo this conversion for all the CSV files inside `/test/data` and `/examples/data`. -Run all tests: +Run primary tests: ```plaintext npm test ``` +Run `large.test.js` tests (writing, reading, and transforming ~100 MB CSV files). These tests take ~30s to complete, so they are separated from the primary tests: + +```plaintext +npm run test:large +``` + ## Other CSV libraries Here are more comprehensive CSV libraries that may be better for your use case. diff --git a/package.json b/package.json index 5c2e8fa..b268040 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ ], "exports": "./src/index.js", "scripts": { - "test": "node --test", - "test:coverage": "npm run test -- --experimental-test-coverage" + "test": "node --test --test-skip-pattern=\"/^large:*/\"", + "test:large": "node --test --test-name-pattern=\"/^large:*/\"" } } diff --git a/test/large.test.js b/test/large.test.js new file mode 100644 index 0000000..e959939 --- /dev/null +++ b/test/large.test.js @@ -0,0 +1,126 @@ +import { suite, test, after } from 'node:test'; +import { ok, deepStrictEqual } from 'node:assert/strict'; +import { unlink, stat } from 'node:fs/promises'; + +import { createRandomCSV, createTempFile } from './utils.js'; +import { CSVReader, CSVTransformer, CSVWriter } from '../src/core.js'; + +suite('large: 100000 row x 100 column CSV (~100 MB)', { concurrency: 1 }, async () => { + const temp = await createTempFile(); + after(async () => await unlink(temp)); + + test('CSVWriter writes file', async () => { + await createRandomCSV(100000, 100, 1).pipeTo(new CSVWriter(temp)); + const { size } = await stat(temp, { bigint: true }); + ok(size > 100_000_000n); + }); + + test('CSVReader reads file', async () => { + const reader = new CSVReader(temp); + let rowCount = 0; + for await (const row of reader) { + const parsed = JSON.parse(row); + if (rowCount === 0) { + deepStrictEqual(parsed[0], '1717370683'); // A1 + deepStrictEqual(parsed[49], '1159187275'); // AX1 + deepStrictEqual(parsed[99], '398856332'); // CV1 + } + else if (rowCount === 49_999) { + deepStrictEqual(parsed[0], '1950971124'); // A50000 + deepStrictEqual(parsed[49], '1212575531'); // AX50000 + deepStrictEqual(parsed[99], '135758415'); // CV50000 + } + else if (rowCount === 99_999) { + deepStrictEqual(parsed[0], '737583361'); // A100000 + deepStrictEqual(parsed[49], '793036957'); // AX100000 + deepStrictEqual(parsed[99], '1923608001'); // CV100000 + } + rowCount++; + } + }); + + test('CSVTransformer transforms file', async () => { + const reader = new CSVReader(temp) + .pipeThrough(new CSVTransformer(r => r.map(f => Number(f) + 1), { handleHeaders: true })); + let rowCount = 0; + for await (const row of reader) { + const parsed = JSON.parse(row); + if (rowCount === 0) { + deepStrictEqual(parsed[0], 1717370684); + deepStrictEqual(parsed[49], 1159187276); + deepStrictEqual(parsed[99], 398856333); + } + else if (rowCount === 49_999) { + deepStrictEqual(parsed[0], 1950971125); + deepStrictEqual(parsed[49], 1212575532); + deepStrictEqual(parsed[99], 135758416); + } + else if (rowCount === 99_999) { + deepStrictEqual(parsed[0], 737583362); + deepStrictEqual(parsed[49], 793036958); + deepStrictEqual(parsed[99], 1923608002); + } + rowCount++; + } + }); +}); + +suite('large: 100 row x 100000 column CSV (~100 MB)', { concurrency: 1 }, async () => { + const temp = await createTempFile(); + after(async () => await unlink(temp)); + + test('CSVWriter writes file', async () => { + await createRandomCSV(100, 100000, 1).pipeTo(new CSVWriter(temp)); + const { size } = await stat(temp, { bigint: true }); + ok(size > 100_000_000n); + }); + + test('CSVReader reads file', async () => { + const reader = new CSVReader(temp); + let rowCount = 0; + for await (const row of reader) { + const parsed = JSON.parse(row); + if (rowCount === 0) { + deepStrictEqual(parsed[0], '1717370683'); + deepStrictEqual(parsed[49_999], '689610881'); + deepStrictEqual(parsed[99_999], '183082010'); + } + else if (rowCount === 49) { + deepStrictEqual(parsed[0], '1521260985'); + deepStrictEqual(parsed[49_999], '1397216552'); + deepStrictEqual(parsed[99_999], '817708515'); + } + else if (rowCount === 99) { + deepStrictEqual(parsed[0], '1749947568'); + deepStrictEqual(parsed[49_999], '2119460449'); + deepStrictEqual(parsed[99_999], '127114681'); + } + rowCount++; + } + }); + + test('CSVTransformer transforms file', async () => { + const reader = new CSVReader(temp) + .pipeThrough(new CSVTransformer(r => r.map(f => Number(f) + 1), { handleHeaders: true })); + let rowCount = 0; + for await (const row of reader) { + const parsed = JSON.parse(row); + if (rowCount === 0) { + deepStrictEqual(parsed[0], 1717370684); + deepStrictEqual(parsed[49_999], 689610882); + deepStrictEqual(parsed[99_999], 183082011); + } + else if (rowCount === 49) { + deepStrictEqual(parsed[0], 1521260986); + deepStrictEqual(parsed[49_999], 1397216553); + deepStrictEqual(parsed[99_999], 817708516); + } + else if (rowCount === 99) { + deepStrictEqual(parsed[0], 1749947569); + deepStrictEqual(parsed[49_999], 2119460450); + deepStrictEqual(parsed[99_999], 127114682); + } + rowCount++; + } + }); +}); \ No newline at end of file diff --git a/test/utils.js b/test/utils.js index a62a477..69dd8c1 100644 --- a/test/utils.js +++ b/test/utils.js @@ -1,6 +1,6 @@ import { deepStrictEqual, notStrictEqual } from 'node:assert/strict'; import { tmpdir } from 'node:os'; -import { open } from 'node:fs/promises'; +import { mkdtemp, open } from 'node:fs/promises'; import { normalize } from 'node:path'; /** @@ -149,3 +149,65 @@ export async function createTempFile() { await open(path, 'a+'); return path; } + +/** + * Simple, small, and fast pseudorandom number generator to deterministically generate large amounts of mock test data. + * + * API is intended to follow [`crypto.getRandomValues()`](https://developer.mozilla.org/en-US/docs/Web/API/Crypto/getRandomValues). + * + * Hash function source: + * - https://burtleburtle.net/bob/hash/integer.html + * - https://web.archive.org/web/20090408063205/http://www.cris.com/~Ttwang/tech/inthash.htm + * @param {ArrayBufferView} typedArray An integer-based `TypedArray` with a byte length that is a multiple of 4. + * All elements in the array will be overwritten with random numbers. + * @param {Number} seed A number to initiate the pseudorandom hash function. + * @returns {ArrayBufferView} The same array passed as `typedArray` but with its contents replaced with pseudorandom + * numbers. Note that `typedArray` is modified in-place and no copy is made. + */ +function getPseudoRandomValues(typedArray, seed) { + function hash(n) { + n = n ^ (n >> 4); + n = (n ^ 0xdeadbeef) + (n << 5); + n = n ^ (n >> 11); + return n; + } + + const array = new Uint32Array(typedArray.buffer); + let h = hash(seed); + for (let i = 0; i < array.length; i++) { + h = hash(h); + array[i] = h; + } + return typedArray; +} + +/** + * Create a stream of pseudorandom CSV data for test mocking purposes. + * + * @param {Number} rows The number of rows the CSV stream will return. + * @param {Number} columns The number of columns in each CSV row. + * @param {Number} seed The seed used to deterministically generate the pseudorandom values in the CSV stream. + * @returns {ReadableStream} A `ReadableStream` where each row is emitted as a separate chunk. + */ +export function createRandomCSV(rows, columns, seed) { + const chunks = (function* () { + while (rows) { + const data = new Uint32Array(columns); + getPseudoRandomValues(data, seed); + seed = data[columns - 1]; + rows--; + yield Array.from(data); + } + })(); + return new ReadableStream({ + pull(controller) { + const { value, done } = chunks.next(); + if (done) { + controller.close(); + } + else { + controller.enqueue(JSON.stringify(value)); + } + } + }); +} \ No newline at end of file -- 2.47.3