Add initial large CSV tests

diff --git a/README.md b/README.md
index 861afbbeca6694d8eeafc05a939e4c34f3286246..c1a2ea294b489a4fc436c714b4b01eba6a59aa09 100644 (file)
--- a/README.md
+++ b/README.md
@@ -150,12 +150,18 @@ Example and test CSVs use `CRLF` (`\r\n`) as the record delimiter, as suggested
 
 If your IDE is configured to automatically convert `CRLF` line endings to `LF`, you must undo this conversion for all the CSV files inside `/test/data` and `/examples/data`.
 
-Run all tests:
+Run primary tests:
 
 ```plaintext
 npm test
 ```
 
+Run `large.test.js` tests (writing, reading, and transforming ~100 MB CSV files). These tests take ~30s to complete, so they are separated from the primary tests:
+
+```plaintext
+npm run test:large
+```
+
 ## Other CSV libraries
 
 Here are more comprehensive CSV libraries that may be better for your use case.

diff --git a/package.json b/package.json
index 5c2e8fa217d292a8aafcbea3fc6924377a10ced9..b2680408a34e3c5fa1fc6ceb17ae40932a033bcd 100644 (file)
--- a/package.json
+++ b/package.json
@@ -14,7 +14,7 @@
   ],
   "exports": "./src/index.js",
   "scripts": {
-    "test": "node --test",
-    "test:coverage": "npm run test -- --experimental-test-coverage"
+    "test": "node --test --test-skip-pattern=\"/^large:*/\"",
+    "test:large": "node --test --test-name-pattern=\"/^large:*/\""
   }
 }

diff --git a/test/large.test.js b/test/large.test.js
new file mode 100644 (file)
index 0000000..e959939
--- /dev/null
+++ b/test/large.test.js
@@ -0,0 +1,126 @@
+import { suite, test, after } from 'node:test';
+import { ok, deepStrictEqual } from 'node:assert/strict';
+import { unlink, stat } from 'node:fs/promises';
+
+import { createRandomCSV, createTempFile } from './utils.js';
+import { CSVReader, CSVTransformer, CSVWriter } from '../src/core.js';
+
+suite('large: 100000 row x 100 column CSV (~100 MB)', { concurrency: 1 }, async () => {
+  const temp = await createTempFile();
+  after(async () => await unlink(temp));
+
+  test('CSVWriter writes file', async () => {
+    await createRandomCSV(100000, 100, 1).pipeTo(new CSVWriter(temp));
+    const { size } = await stat(temp, { bigint: true });
+    ok(size > 100_000_000n);
+  });
+
+  test('CSVReader reads file', async () => {
+    const reader = new CSVReader(temp);
+    let rowCount = 0;
+    for await (const row of reader) {
+      const parsed = JSON.parse(row);
+      if (rowCount === 0) {
+        deepStrictEqual(parsed[0], '1717370683'); // A1
+        deepStrictEqual(parsed[49], '1159187275'); // AX1
+        deepStrictEqual(parsed[99], '398856332'); // CV1
+      }
+      else if (rowCount === 49_999) {
+        deepStrictEqual(parsed[0], '1950971124'); // A50000
+        deepStrictEqual(parsed[49], '1212575531'); // AX50000
+        deepStrictEqual(parsed[99], '135758415'); // CV50000
+      }
+      else if (rowCount === 99_999) {
+        deepStrictEqual(parsed[0], '737583361'); // A100000
+        deepStrictEqual(parsed[49], '793036957'); // AX100000
+        deepStrictEqual(parsed[99], '1923608001'); // CV100000
+      }
+      rowCount++;
+    }
+  });
+
+  test('CSVTransformer transforms file', async () => {
+    const reader = new CSVReader(temp)
+      .pipeThrough(new CSVTransformer(r => r.map(f => Number(f) + 1), { handleHeaders: true }));
+    let rowCount = 0;
+    for await (const row of reader) {
+      const parsed = JSON.parse(row);
+      if (rowCount === 0) {
+        deepStrictEqual(parsed[0], 1717370684);
+        deepStrictEqual(parsed[49], 1159187276);
+        deepStrictEqual(parsed[99], 398856333);
+      }
+      else if (rowCount === 49_999) {
+        deepStrictEqual(parsed[0], 1950971125);
+        deepStrictEqual(parsed[49], 1212575532);
+        deepStrictEqual(parsed[99], 135758416);
+      }
+      else if (rowCount === 99_999) {
+        deepStrictEqual(parsed[0], 737583362);
+        deepStrictEqual(parsed[49], 793036958);
+        deepStrictEqual(parsed[99], 1923608002);
+      }
+      rowCount++;
+    }
+  });
+});
+
+suite('large: 100 row x 100000 column CSV (~100 MB)', { concurrency: 1 }, async () => {
+  const temp = await createTempFile();
+  after(async () => await unlink(temp));
+
+  test('CSVWriter writes file', async () => {
+    await createRandomCSV(100, 100000, 1).pipeTo(new CSVWriter(temp));
+    const { size } = await stat(temp, { bigint: true });
+    ok(size > 100_000_000n);
+  });
+
+  test('CSVReader reads file', async () => {
+    const reader = new CSVReader(temp);
+    let rowCount = 0;
+    for await (const row of reader) {
+      const parsed = JSON.parse(row);
+      if (rowCount === 0) {
+        deepStrictEqual(parsed[0], '1717370683');
+        deepStrictEqual(parsed[49_999], '689610881');
+        deepStrictEqual(parsed[99_999], '183082010');
+      }
+      else if (rowCount === 49) {
+        deepStrictEqual(parsed[0], '1521260985');
+        deepStrictEqual(parsed[49_999], '1397216552');
+        deepStrictEqual(parsed[99_999], '817708515');
+      }
+      else if (rowCount === 99) {
+        deepStrictEqual(parsed[0], '1749947568');
+        deepStrictEqual(parsed[49_999], '2119460449');
+        deepStrictEqual(parsed[99_999], '127114681');
+      }
+      rowCount++;
+    }
+  });
+
+  test('CSVTransformer transforms file', async () => {
+    const reader = new CSVReader(temp)
+      .pipeThrough(new CSVTransformer(r => r.map(f => Number(f) + 1), { handleHeaders: true }));
+    let rowCount = 0;
+    for await (const row of reader) {
+      const parsed = JSON.parse(row);
+      if (rowCount === 0) {
+        deepStrictEqual(parsed[0], 1717370684);
+        deepStrictEqual(parsed[49_999], 689610882);
+        deepStrictEqual(parsed[99_999], 183082011);
+      }
+      else if (rowCount === 49) {
+        deepStrictEqual(parsed[0], 1521260986);
+        deepStrictEqual(parsed[49_999], 1397216553);
+        deepStrictEqual(parsed[99_999], 817708516);
+      }
+      else if (rowCount === 99) {
+        deepStrictEqual(parsed[0], 1749947569);
+        deepStrictEqual(parsed[49_999], 2119460450);
+        deepStrictEqual(parsed[99_999], 127114682);
+      }
+      rowCount++;
+    }
+  });
+});
\ No newline at end of file

diff --git a/test/utils.js b/test/utils.js
index a62a4772123733eee3ac00f9f2feea05a2a2b169..69dd8c16e6800585f5583aef8ffa200c13b72e65 100644 (file)
--- a/test/utils.js
+++ b/test/utils.js
@@ -1,6 +1,6 @@
 import { deepStrictEqual, notStrictEqual } from 'node:assert/strict';
 import { tmpdir } from 'node:os';
-import { open } from 'node:fs/promises';
+import { mkdtemp, open } from 'node:fs/promises';
 import { normalize } from 'node:path';
 
 /**
@@ -149,3 +149,65 @@ export async function createTempFile() {
   await open(path, 'a+');
   return path;
 }
+
+/**
+ * Simple, small, and fast pseudorandom number generator to deterministically generate large amounts of mock test data.
+ * 
+ * API is intended to follow [`crypto.getRandomValues()`](https://developer.mozilla.org/en-US/docs/Web/API/Crypto/getRandomValues).
+ * 
+ * Hash function source:
+ * - https://burtleburtle.net/bob/hash/integer.html
+ * - https://web.archive.org/web/20090408063205/http://www.cris.com/~Ttwang/tech/inthash.htm
+ * @param {ArrayBufferView<ArrayBufferLike>} typedArray An integer-based `TypedArray` with a byte length that is a multiple of 4.
+ * All elements in the array will be overwritten with random numbers.
+ * @param {Number} seed A number to initiate the pseudorandom hash function.
+ * @returns {ArrayBufferView<ArrayBufferLike>} The same array passed as `typedArray` but with its contents replaced with pseudorandom
+ * numbers. Note that `typedArray` is modified in-place and no copy is made.
+ */
+function getPseudoRandomValues(typedArray, seed) {
+  function hash(n) {
+    n = n ^ (n >> 4);
+    n = (n ^ 0xdeadbeef) + (n << 5);
+    n = n ^ (n >> 11);
+    return n;
+  }
+
+  const array = new Uint32Array(typedArray.buffer);
+  let h = hash(seed);
+  for (let i = 0; i < array.length; i++) {
+    h = hash(h);
+    array[i] = h;
+  }
+  return typedArray;
+}
+
+/**
+ * Create a stream of pseudorandom CSV data for test mocking purposes.
+ * 
+ * @param {Number} rows The number of rows the CSV stream will return.
+ * @param {Number} columns The number of columns in each CSV row.
+ * @param {Number} seed The seed used to deterministically generate the pseudorandom values in the CSV stream.
+ * @returns {ReadableStream} A `ReadableStream` where each row is emitted as a separate chunk.
+ */
+export function createRandomCSV(rows, columns, seed) {
+  const chunks = (function* () {
+    while (rows) {
+      const data = new Uint32Array(columns);
+      getPseudoRandomValues(data, seed);
+      seed = data[columns - 1];
+      rows--;
+      yield Array.from(data);
+    }
+  })();
+  return new ReadableStream({
+    pull(controller) {
+      const { value, done } = chunks.next();
+      if (done) {
+        controller.close();
+      }
+      else {
+        controller.enqueue(JSON.stringify(value));
+      }
+    }
+  });
+}
\ No newline at end of file