From: Ardis Lu Date: Thu, 2 Jan 2025 02:43:28 +0000 (-0800) Subject: Initial commit X-Git-Tag: v0.1.0~144 X-Git-Url: https://git.ardis.lu/?a=commitdiff_plain;h=4e8c2e699e28c22ec85f3e4173e6bed5778a21db;p=csvio Initial commit --- 4e8c2e699e28c22ec85f3e4173e6bed5778a21db diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..88b892a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Do not do any EOL conversions for CSV files +*.csv -text \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..930ad91 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Ardis Lu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..d5f67ca --- /dev/null +++ b/README.md @@ -0,0 +1,129 @@ +# Streamed CSV + +Streamed CSV is a Node.js library to read, transform, and write CSV files using the [Web Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API). + +## Example + +Assume `./data/example-in.csv` looks like: + +```plaintext +columnA,columnB +1,1 +100,100 +223423,455947 +348553,692708 +536368,676147 +``` + +Run: + +```javascript +import { createCSVReadableStream, createCSVTransformStream, createCSVWritableStream } from './src/index.js'; + +function timesTwo(row) { + return [Number(row[0]) * 2, Number(row[1]) * 2]; +} + +createCSVReadableStream('./data/example-in.csv') + .pipeThrough(createCSVTransformStream(timesTwo)) + .pipeTo(createCSVWritableStream('./data/example-out.csv')); +``` + +A new `./data/example-out.csv` file will be created: + +```plaintext +columnA,columnB +2,2 +200,200 +446846,911894 +697106,1385416 +1072736,1352294 +``` + +See the [`examples`](./examples) folder for more end-to-end examples. + +## Why? + +A common workflow is: + +1. Read an existing CSV file row by row. +2. Perform some transformation using the data in each row. +3. Write the transformed data to another CSV file. + +Most people are interested in coding the transformation logic for step (2), and much less interested in coding the error-prone boilerplate code for steps (1) and (3). + +Streamed CSV provides the bare minimum functions to accomplish **this specific workflow**. This library is *not* intended for any other workflow (e.g., parsing CSV files with custom delimiters, converting between CSV and other formats, performing data analytics on CSV data). For those workflows, see ["Other CSV libraries"](#other-csv-libraries). + +## Features + +- No dependencies other than the Node.js standard library +- Reads and writes [RFC 4180](https://www.ietf.org/rfc/rfc4180.txt) compliant CSV files +- Single `core.js` file contains *all* core API code +- Uses web standard APIs (i.e., the [Web Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) over [Node.js Streams](https://nodejs.org/api/stream.html)) + +## Core API + +The core API is contained in `src/core.js` and exposes three functions to read, transform, and write CSV files. + +### `createCSVReadableStream` + +Create a `ReadableStream` from a local CSV file where each CSV row is one chunk. + +```javascript +import { createCSVReadableStream } from './src/index.js'; + +const readableStream = createCSVReadableStream('./data/example-in.csv'); +``` + +### `createCSVTransformStream` + +Create a `TransformStream` to apply a given function to each row in a streamed CSV. + +```javascript +import { createCSVTransformStream } from './src/index.js'; + +function timesTwo(row) { + return [Number(row[0]) * 2, Number(row[1]) * 2]; +} + +const transformStream = createCSVTransformStream(timesTwo); +``` + +### `createCSVWritableStream` + +Create a `WritableStream` to save CSV row data to disk. + +```javascript +import { createCSVWritableStream } from './src/index.js'; + +const writableStream = createCSVWritableStream('./data/example-out.csv'); +``` + +## Extended API + +In addition to the core API, optional utilities are provided to address common CSV transformation tasks. + +### `createCSVNormalizationStream` and `createCSVDenormalizationStream` + +`TransformStream`s to parse CSVs that have been mangled by spreadsheet programs such as Excel. + +## Tests + +Example and test CSVs use `CRLF` (`\r\n`) as the record delimiter, as required by [RFC 4180](https://www.ietf.org/rfc/rfc4180.txt). + +If your IDE is configured to automatically convert `CRLF` line endings to `LF`, you must undo this conversion for all the CSV files inside `/test/data` and `/examples/data`. + +Run all tests: + +```plaintext +npm test +``` + +## Other CSV libraries + +Here are more comprehensive CSV libraries that may be better for your use case. + +- [Papa Parse](https://github.com/mholt/PapaParse) +- [node-csv](https://github.com/adaltas/node-csv) +- [fast-csv](https://github.com/C2FO/fast-csv) +- [csv-parser](https://github.com/mafintosh/csv-parser) diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..97cfdb7 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,10 @@ +# Examples + +This folder contains end-to-end examples of using this library. + +All examples assume your current working directory is **this folder**. All examples can be run directly without additional setup using `node`. E.g.: + +``` +cd examples +node ./ex1.js +``` diff --git a/examples/data/ex1-in.csv b/examples/data/ex1-in.csv new file mode 100644 index 0000000..4bd7c6a --- /dev/null +++ b/examples/data/ex1-in.csv @@ -0,0 +1,6 @@ +columnA,columnB +1,1 +100,100 +223423,455947 +348553,692708 +536368,676147 \ No newline at end of file diff --git a/examples/data/ex1-out.csv b/examples/data/ex1-out.csv new file mode 100644 index 0000000..a3373a3 --- /dev/null +++ b/examples/data/ex1-out.csv @@ -0,0 +1,6 @@ +columnA,columnB +2,2 +200,200 +446846,911894 +697106,1385416 +1072736,1352294 diff --git a/examples/data/ex2-in.csv b/examples/data/ex2-in.csv new file mode 100644 index 0000000..4bd7c6a --- /dev/null +++ b/examples/data/ex2-in.csv @@ -0,0 +1,6 @@ +columnA,columnB +1,1 +100,100 +223423,455947 +348553,692708 +536368,676147 \ No newline at end of file diff --git a/examples/data/ex2-out.csv b/examples/data/ex2-out.csv new file mode 100644 index 0000000..69e4d65 --- /dev/null +++ b/examples/data/ex2-out.csv @@ -0,0 +1,6 @@ +columnA,columnB,sum +1,1,2 +100,100,200 +223423,455947,679370 +348553,692708,1041261 +536368,676147,1212515 diff --git a/examples/data/ex3-in.csv b/examples/data/ex3-in.csv new file mode 100644 index 0000000..3b298c9 --- /dev/null +++ b/examples/data/ex3-in.csv @@ -0,0 +1,3 @@ +Alice.Example1_123 +Bob.Example2_456456 +Carol.Example3_789 \ No newline at end of file diff --git a/examples/data/ex3-out.csv b/examples/data/ex3-out.csv new file mode 100644 index 0000000..74d9c1a --- /dev/null +++ b/examples/data/ex3-out.csv @@ -0,0 +1,3 @@ +"Example1, Alice",000123 +"Example2, Bob",456456 +"Example3, Carol",000789 diff --git a/examples/data/ex4-in.csv b/examples/data/ex4-in.csv new file mode 100644 index 0000000..5d24191 --- /dev/null +++ b/examples/data/ex4-in.csv @@ -0,0 +1,31 @@ +Index,Phone Number,Join Timestamp +$1.00 ,1.4415550172E+10,44067.63 +$2.00 ,1.8125550104E+10,44309.63502 +$3.00 ,1.8105550166E+10,43915.31372 +$4.00 ,1.4075550162E+10,43984.96909 +$5.00 ,1.2205550156E+10,44303.77208 +$6.00 ,1.8785550149E+10,43886.58531 +$7.00 ,1.6645550128E+10,44432.76038 +$8.00 ,1.3175550117E+10,44298.14111 +$9.00 ,1.9185550198E+10,43843.6739 +$10.00 ,1.2465550106E+10,44508.37237 +$11.00 ,1.5645550137E+10,44489.69565 +$12.00 ,1.4585550182E+10,44164.71351 +$13.00 ,1.9285550166E+10,44532.98664 +$14.00 ,1.4085550102E+10,44235.21617 +$15.00 ,1.9255550174E+10,44587.31713 +$16.00 ,1.7435550169E+10,44663.93397 +$17.00 ,1.3075550120E+10,44265.49723 +$18.00 ,1.9095550108E+10,44646.72493 +$19.00 ,1.8685550128E+10,44098.93199 +$20.00 ,1.2515550148E+10,44463.86527 +$21.00 ,1.2035550168E+10,44573.49945 +$22.00 ,1.9095550164E+10,44089.25389 +$23.00 ,1.6095550101E+10,44062.44509 +$24.00 ,1.7165550187E+10,44082.87805 +$25.00 ,1.4785550187E+10,44292.12938 +,, +,, +,, +,, +,, diff --git a/examples/data/ex4-out.csv b/examples/data/ex4-out.csv new file mode 100644 index 0000000..c84b004 --- /dev/null +++ b/examples/data/ex4-out.csv @@ -0,0 +1,26 @@ +Index,Phone Number,Join Timestamp +1,14415550172,2020-08-24T15:07:11.999Z +2,18125550104,2021-04-23T15:14:25.728Z +3,18105550166,2020-03-25T07:31:45.407Z +4,14075550162,2020-06-02T23:15:29.375Z +5,12205550156,2021-04-17T18:31:47.712Z +6,18785550149,2020-02-25T14:02:50.784Z +7,16645550128,2021-08-24T18:14:56.831Z +8,13175550117,2021-04-12T03:23:11.903Z +9,19185550198,2020-01-13T16:10:24.960Z +10,12465550106,2021-11-08T08:56:12.767Z +11,15645550137,2021-10-20T16:41:44.160Z +12,14585550182,2020-11-29T17:07:27.264Z +13,19285550166,2021-12-02T23:40:45.696Z +14,14085550102,2021-02-08T05:11:17.087Z +15,19255550174,2022-01-26T07:36:40.032Z +16,17435550169,2022-04-12T22:24:55.007Z +17,13075550120,2021-03-10T11:56:00.672Z +18,19095550108,2022-03-26T17:23:53.951Z +19,18685550128,2020-09-24T22:22:03.935Z +20,12515550148,2021-09-24T20:45:59.328Z +21,12035550168,2022-01-12T11:59:12.480Z +22,19095550164,2020-09-15T06:05:36.095Z +23,16095550101,2020-08-19T10:40:55.776Z +24,17165550187,2020-09-08T21:04:23.519Z +25,14785550187,2021-04-06T03:06:18.431Z diff --git a/examples/ex1.js b/examples/ex1.js new file mode 100644 index 0000000..426671d --- /dev/null +++ b/examples/ex1.js @@ -0,0 +1,19 @@ +// ## Example 1: Basic usage +// +// Minimal example of a user-defined function to transform a CSV. +// +// Notes: +// - The header row (first row) of the input CSV is not passed to the logic function, it skips through to the output CSV. +// - The input to the transformation function is already deserialized to an array, however **each field in the array is a string**. +// - Each call to the transformation function passes 1 row of the input CSV. +// - The expected output of the transformation function is another array representing 1 row of the output CSV. + +import { createCSVReadableStream, createCSVTransformStream, createCSVWritableStream } from '../src/index.js'; + +function timesTwo(row) { + return [Number(row[0]) * 2, Number(row[1]) * 2]; +} + +createCSVReadableStream('./data/ex1-in.csv') + .pipeThrough(createCSVTransformStream(timesTwo)) + .pipeTo(createCSVWritableStream('./data/ex1-out.csv')); \ No newline at end of file diff --git a/examples/ex2.js b/examples/ex2.js new file mode 100644 index 0000000..2471092 --- /dev/null +++ b/examples/ex2.js @@ -0,0 +1,23 @@ +// ## Example 2: Updating headers in the output +// +// Set `includeHeaders: true` to manually handle the header row (first row). +// +// Notes: +// - The header row is assumed to be the first row in the input CSV, and by default it goes directly to the output (skipping the user-provided logic function). +// - If your input CSV does not have a header row, set `includeHeaders: true` and process the header row (first row) normally. +// - Otherwise, you will need distinct logic to handle the header row vs. data rows. A simple boolean variable is a good way to switch between this logic. + +import { createCSVReadableStream, createCSVTransformStream, createCSVWritableStream } from '../src/index.js'; + +let firstRow = true; +function sum(row) { + if (firstRow) { + firstRow = false; + return [...row, 'sum']; + } + return [...row, Number(row[0]) + Number(row[1])]; +} + +createCSVReadableStream('./data/ex2-in.csv') + .pipeThrough(createCSVTransformStream(sum, { includeHeaders: true })) + .pipeTo(createCSVWritableStream('./data/ex2-out.csv')); \ No newline at end of file diff --git a/examples/ex3.js b/examples/ex3.js new file mode 100644 index 0000000..f841e6e --- /dev/null +++ b/examples/ex3.js @@ -0,0 +1,28 @@ +// ## Example 3: Chaining multiple transformations +// +// Set `rawInput: true` and/or `rawOutput: true` to skip JSON deserialization (`rawInput`) and/or JSON serialization +// (`rawOutput`) when handling the transformation function input/output. These options are intended for advanced +// transformations (e.g., transformations spread across multiple functions or lower-level transformations directly +// on byte arrays). + +import { createCSVReadableStream, createCSVTransformStream, createCSVWritableStream } from '../src/index.js'; + +// Parse the raw CSV data and transform the data into an object for convenience +function parse(row) { + const [name, id] = row[0].split('_'); + const [first, last] = name.split('.') + return JSON.stringify({ first, last, id }); // Manually handling serialization here, so use `rawOutput` to turn off automatic serialization +} + +// Contains the logic for the transformation +function calculate(obj) { + const { first, last, id } = JSON.parse(obj); // Manually handling deserialization here, so use `rawInput` to turn off automatic deserialization + const name = `${last}, ${first}`; + const paddedId = id.padStart(6, '0'); + return [name, paddedId]; +} + +createCSVReadableStream('./data/ex3-in.csv') + .pipeThrough(createCSVTransformStream(parse, { includeHeaders: true, rawOutput: true })) + .pipeThrough(createCSVTransformStream(calculate, { includeHeaders: true, rawInput: true })) + .pipeTo(createCSVWritableStream('./data/ex3-out.csv')); \ No newline at end of file diff --git a/examples/ex4.js b/examples/ex4.js new file mode 100644 index 0000000..18a05f4 --- /dev/null +++ b/examples/ex4.js @@ -0,0 +1,36 @@ +// ## Example 4: Using `createCSVNormalizationStream` and `createCSVDenormalizationStream` +// +// These functions fix common CSV data mangling caused by Excel. In `ex4-in.csv`: +// - The "Index" column has been mangled into Excel's "Accounting" format +// - The "Phone Number" column has been mangled into scientific notation +// - The "Join Timestamp" column has been mangled into a number +// - There are trailing empty rows after deleting excess data from the CSV +// `createCSVNormalizationStream` performs transformations to un-mangle the above using only a declaration of the header names and data types. +// `createCSVDenormalizationStream` converts the data stream back into a CSV row. + +import { createCSVReadableStream, createCSVWritableStream, createCSVNormalizationStream, createCSVDenormalizationStream } from '../src/index.js'; + +// The order of items in `headers` is significant, it determines the order of the output CSV columns. +// If an input column is not provided in `headers`, the column is removed from the output CSV. +const headers = [ + { + name: 'index', // MUST match the input CSV column name in camelCase + type: 'number', + displayName: 'Index' // `displayName` can also be used to rename the column in the output CSV + }, + { + name: 'phoneNumber', + type: 'bigint', + displayName: 'Phone Number' + }, + { + name: 'joinTimestamp', + type: 'date', + displayName: 'Join Timestamp' + } +]; + +createCSVReadableStream('./data/ex4-in.csv') + .pipeThrough(createCSVNormalizationStream(headers)) + .pipeThrough(createCSVDenormalizationStream()) + .pipeTo(createCSVWritableStream('./data/ex4-out.csv')); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..277e2fc --- /dev/null +++ b/package-lock.json @@ -0,0 +1,15 @@ +{ + "name": "streamed-csv", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "streamed-csv", + "version": "0.1.0", + "engines": { + "node": ">=20" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..563a367 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "streamed-csv", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=20" + }, + "scripts": { + "test": "node --test", + "test:coverage": "npm run test -- --experimental-test-coverage" + } +} diff --git a/src/core.js b/src/core.js new file mode 100644 index 0000000..803ebb1 --- /dev/null +++ b/src/core.js @@ -0,0 +1,191 @@ +import { createReadStream } from 'node:fs'; +import { writeFile, mkdir, open } from 'node:fs/promises'; +import { dirname, normalize } from 'node:path'; +import { Readable } from 'node:stream'; +/** @import { PathLike } from 'node:fs' */ + +/** + * Convert a JavaScript array into a CSV string. + * + * @param {Array} arr An array of strings representing one row of a CSV file. + * @returns {string} A string representing one row of a CSV file, with character escaping and line endings compliant with + * [RFC 4180](https://www.ietf.org/rfc/rfc4180.txt). + */ +export function arrayToCSVString(arr) { + let out = ''; + for (const field of arr) { + if (/[,"\r\n]/g.test(field)) { + out += `"${field.replaceAll('"', '""')}",`; + } + else { + out += `${field},`; + } + } + return `${out.slice(0, -1)}\r\n`; +} + +/** + * A simple streaming parser for CSV files. + * + * Uses Node.js's [`fs.createReadStream()`](https://nodejs.org/api/fs.html#fscreatereadstreampath-options) to read a local + * CSV file then parses the file contents and returns the data through a [Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API) + * `ReadableStream`. + * @param {PathLike} path A `string`, `Buffer`, or `URL` representing a path to a local CSV file. + * @returns {ReadableStream} A `ReadableStream` where each chunk is one row of the CSV file, in the form of a string + * that may be deserialized using `JSON.parse()`. + */ +export function createCSVReadableStream(path) { + const reader = createReadStream(path, { encoding: 'utf-8' }); + const stream = Readable.toWeb(reader); + + const row = []; + let field = ''; + let lastChar = null; + let escapeMode = false; + let justExitedEscapeMode = false; + return stream.pipeThrough(new TransformStream({ + transform(chunk, controller) { + for (const char of chunk) { + /* Escape mode logic */ + if (escapeMode) { + if (char === '"') { // Exit escape mode and do not add char to field + escapeMode = false; + justExitedEscapeMode = true; + } + else { // Add the literal char because in escape mode + justExitedEscapeMode = false; + field += char; + } + } + /* Normal mode logic */ + else { + if (char === '"') { + if (justExitedEscapeMode) { // Re-enter escape mode because the exit char was escaped, and add char to field + escapeMode = true; + field += char; + } + else if (lastChar === ',' || lastChar === '\n' || lastChar === null) { // Enter escape mode and do not add char to field + escapeMode = true; + } + else { // No special cases match, add the literal char + field += char; + } + } + else if (char === '\r' || char === '\uFEFF') { } // Ignore CR (newline logic is handled by LF) and byte order mark + else if (char === ',') { // Terminate the field and do not add char to field + row.push(field); + field = ''; + } + else if (char === '\n') { // Terminate the field and row and do not add char to field + row.push(field); + field = ''; + controller.enqueue(JSON.stringify(row)); + row.length = 0; + } + else { // No special cases match, add the literal char + field += char; + } + justExitedEscapeMode = false; + } + lastChar = char; + } + }, + flush(controller) { + if (field !== '' || row.length !== 0) { // CSV terminated without a trailing CRLF, leaving a row in the queue + row.push(field); // Last field is always un-pushed if CSV terminated with nothing + controller.enqueue(JSON.stringify(row)); + } + } + })); +} + +/** + * Options to configure `createCSVTransformStream`. + * @typedef {Object} createCSVTransformStreamOptions + * @property {boolean} [includeHeaders=false] Set to `true` to pass the header row (assumed to be the first row of the CSV) to `fn()`. Otherwise, + * the header row will flow through to the next stream without going through `fn()`. The default value is `false`. + * + * NOTE: If the CSV has no header row, set `includeHeaders` to `true` and process the first row normally in `fn()`. + * @property {boolean} [rawInput=false] Set to `true` to send the raw CSV row to `fn()` as a `string`. Otherwise, the CSV row will be deserialized + * using `JSON.parse()` before being sent to `fn()`. The default value is `false`. + * @property {boolean} [rawOutput=false] Set to `true` to send the raw return value of `fn()` to the next stream. Otherwise, the return value of + * `fn()` will be serialized using `JSON.stringify()` before being sent to the next stream. The default value is `false`. + */ + +/** + * Create a new [TransformStream](https://developer.mozilla.org/en-US/docs/Web/API/TransformStream) to process CSV data. + * + * @param {function(Array|string):Array|string} fn A function to process a row of CSV data from `createCSVReadableStream`. + * + * If `options.rawInput` is `false` (default), the input will be a `Array` representing the CSV row. If `options.rawInput` is `true`, + * the input will be a JSON `string` representing the CSV row. + * + * If `options.rawOutput` is `false` (default), the expected output is a `Array` which will be serialized with `JSON.stringify()`. If + * `options.rawOutput` is `true`, the raw output will be sent the next stream unmodified. + * @param {createCSVTransformStreamOptions} options Object containing flags to configure the stream logic. + * @returns {TransformStream} A `TransformStream` where each chunk is one row of the CSV file, after transformations applied by `fn()`. + */ +export function createCSVTransformStream(fn, options = {}) { + options.includeHeaders ??= false; + options.rawInput ??= false; + options.rawOutput ??= false; + + let firstChunk = true; + return new TransformStream({ + async transform(chunk, controller) { + const row = options.rawInput ? chunk : JSON.parse(chunk); + let out; + if (firstChunk) { + firstChunk = false; + out = options.includeHeaders ? await fn(row) : row; + } + else { + out = await fn(row); + } + if (options.rawOutput || typeof out === 'string') { + controller.enqueue(out); + } + else { + controller.enqueue(JSON.stringify(out)); + } + } + }); +} + +/** + * Write a streamed CSV file to disk. + * + * A simple wrapper around Node.js's [`fs.writeFile`](https://nodejs.org/api/fs.html#fspromiseswritefilefile-data-options) to write + * streamed data to a file. If the input data is JSON, the data is converted to a string representing a RFC 4180 CSV record. If the + * data is not JSON (e.g., if it is already converted to a CSV string), the data is written to the CSV file directly. + * @param {string} path A `string` representing a path to a local CSV file destination. If the file already exists, its **data + * will be overwritten**. + * @returns {WritableStream} A `WritableStream` to write data to a file on disk. + */ +export function createCSVWritableStream(path) { + const fullPath = normalize(path); + const dir = dirname(fullPath); + let handle; + + return new WritableStream({ + async start() { + await mkdir(dir, { recursive: true }); + await writeFile(fullPath, '', { flush: true }); + handle = await open(fullPath, 'a'); + }, + async write(chunk) { + let data; + try { + data = arrayToCSVString(JSON.parse(chunk)); + } + catch { + data = chunk; + } + await handle.write(data); + await handle.sync(); + }, + async close() { + await handle.close(); + } + }); +} diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..a092391 --- /dev/null +++ b/src/index.js @@ -0,0 +1,2 @@ +export { createCSVReadableStream, createCSVTransformStream, createCSVWritableStream } from './core.js'; +export { createCSVNormalizationStream, createCSVDenormalizationStream } from './normalization.js'; diff --git a/src/normalization.js b/src/normalization.js new file mode 100644 index 0000000..73828ee --- /dev/null +++ b/src/normalization.js @@ -0,0 +1,296 @@ +// User locale's decimal separator +const DECIMAL_SEPARATOR = Intl.NumberFormat().formatToParts(.1).find(p => p.type === 'decimal')?.value ?? '.'; + +/** + * Convert a string to camelCase. + * + * @param {string} str A string with words separated using whitespace, hyphens, or underscores. Words can be in lowercase, + * uppercase, or a mix. + * @returns {string} The same string in camelCase (i.e., all whitespace, hyphens, and underscores removed and the first + * letter of each word after the first word capitalized). + */ +export function toCamelCase(str) { + str = str.trim(); + if (!/[\s\-_]+(.)/.test(str)) { // No separator found, fix the first letter but must assume the rest is already in camelCase + return `${str[0].toLowerCase()}${str.substring(1)}`; + } + return str + .toLowerCase() + .replace(/[\s\-_]+(.)/g, m => m.substring(m.length - 1).toUpperCase()); +} + +/** + * Convert scientific notation ("E" notation) to normal form. + * + * @param {string} str A string representing a number expressed using scientific notation (e.g., `"1E+18"`). + * @param {boolean} truncate If `true`, decimals will be truncated in the expanded string. Defaults to `false`. + * @returns {string|null} A string representing the normal form of the number (e.g., `"1000000000000000000"`), or `null` + * if the input could not be parsed. + */ +export function expandScientificNotation(str, truncate = false) { + let expanded = null; + const [mantissa, exponent] = str.split?.('E') ?? [undefined, undefined]; + if (mantissa !== undefined && exponent !== undefined) { + const e = Number(exponent); + if (mantissa.indexOf(DECIMAL_SEPARATOR) === -1) { + expanded = mantissa + '0'.repeat(e); + } + else { + const [integer, decimal] = mantissa.split('.'); + expanded = integer + decimal.padEnd(e, '0'); + if (truncate) { + expanded = expanded.substring(0, e + 1); + } + else if (mantissa.length - 2 > e) { // Must re-insert decimal + expanded = expanded.substring(0, e + 1) + '.' + expanded.substring(e + 1); + } + } + } + return expanded; +} + +/** + * Fix number mangling by Excel. + * + * For example, converting numbers to dates or adding extra characters (e.g., whitespace, thousands separators, or currency symbols). + * @param {string} str A string that might be a number. + * @returns {number|string} If a fix was possible, a number. Otherwise, the original string is returned. + */ +export function fixExcelNumber(str) { + if (str === '') { return str; } + const original = str; + + // Can parse normally without any fixes + const num1 = Number(original); + if (!Number.isNaN(num1)) { + return num1; + } + + // Excel converted the number to a date + const date = new Date(original); + if (date.toString() !== 'Invalid Date') { + // Excel incorrectly thinks 1900 was a leap year, need to offset 1 day before 3/1/1900 + const leapYearOffset = (original === '2/29/1900' || date.valueOf() < -2203862400000) ? -1 : 0; + // Convert JavaScript date to Excel serialized date: + // Days since Unix epoch - Timezone offset in days + Days between Jan 1, 1900 and Unix epoch start + return (date.valueOf() / 1000 / 60 / 60 / 24) - (date.getTimezoneOffset() / 60 / 24) + 25569 + leapYearOffset; + } + + // Excel added extra characters from setting the format to "Accounting" or other custom formats + // Delete everything that's not a number or the decimal separator + const replaced = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), ''); + const num2 = Number(replaced); + if (!Number.isNaN(num2) && replaced !== '') { + return num2; + } + + // Otherwise, pass through as string + console.warn(`Could not fix number: "${original}". Passing through as a string.`); + return original; +} + +/** + * Fix BigInt mangling by Excel. + * + * For example, converting BigInt to scientific notation. + * @param {string} str A string that might be a BigInt. + * @returns {string} If a fix was possible, a string that may be passed to the `BigInt()` constructor without error. + * Otherwise, the original string is returned. + * + * Note that **a string is always returned** to simplify downstream use because BigInt serialization is not possible with `JSON.stringify()`. + */ +export function fixExcelBigInt(str) { + if (str === '') { return str; } + const original = str; + + // Can parse normally without any fixes + try { + return BigInt(original).toString(); + } catch { } + + // Excel converted to scientific notation + try { + const expanded = expandScientificNotation(original, true); + if (expanded === null) { throw new Error(); } + return BigInt(expanded).toString(); + } catch { } + + // Excel added extra characters from setting the format to "Accounting" or other custom formats + // Delete everything that's not a number or the decimal separator + try { + const str = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), ''); + if (str === '') { throw new Error(); } + return BigInt(str.split(DECIMAL_SEPARATOR)[0]).toString(); + } catch { } + + // Otherwise, pass through as string + console.warn(`Could not fix bigint: "${original}". Passing through as a string.`); + return original; +} + +/** + * Fix date mangling by Excel. + * + * For example, converting date to number. + * @param {string} str A string that might be a date. + * @returns {Date|string} If a fix was possible, a date. Otherwise, the original string is returned. + */ +export function fixExcelDate(str) { + if (str === '') { return str; } + const original = str; + + // Excel converted date to a number + // This check MUST come before the new Date() check because numbers can be parsed as dates. + const num = Number(original); + if (!Number.isNaN(num)) { + const day = Math.trunc(num); // Days elapsed since January 1, 1900, including non-existent February 29, 1900 + const time = (num % 1) * 86400000; // Time of day as decimal, so 0 = 12 AM and 0.999... = 11:59:59.999... PM + if (num < 61) { + return new Date(Date.UTC(0, 0, day, 0, 0, 0, time)); + } + return new Date(Date.UTC(0, 0, day - 1, 0, 0, 0, time)); // Offset for non-existent February 29, 1900 + } + + // Can parse normally without any fixes + const date = new Date(original); + if (date.toString() !== 'Invalid Date') { + return date; + } + + // Otherwise, pass through as string + console.warn(`Could not fix date: "${original}". Passing through as a string.`); + return original; +} + +/** + * Metadata about CSV columns used to normalize raw CSV data. + * @typedef {Object} createCSVNormalizationStreamHeader + * @property {string} name The name of the CSV column, in camelCase. Values in the header row of the CSV will be + * transformed to camelCase for comparison purposes and for downstream usage (e.g., a header with the value `Example Column` + * in the input CSV will match with a `name` value of `exampleColumn`). + * @property {'string'|'number'|'bigint'|'date'} type The JavaScript data type to attempt to cast this column to. If + * the data cannot be unmangled, the data passes through as a `string`. + * @property {string} [displayName=name] Optional `string` value to indicate the desired header name in the output CSV. + * @property {string} [defaultValue] Optional `string` value to use to fill empty CSV fields. If no value is provided, the + * field value will be `''` (empty string) in the output CSV. + */ + +/** + * Options to configure `createCSVNormalizationStream`. + * @typedef {Object} createCSVNormalizationStreamOptions + * @property {boolean} [passthroughEmptyRows=false] Set to `true` to send empty rows (i.e., rows where all field values are + * `''`) downstream. Otherwise, empty rows will be removed. The default value is `false`. + * @property {boolean} [passthroughNumber=false] Set to `true` to NOT apply data casting to `type='number'` columns. Otherwise, + * there will be an attempt to cast columns with `type='number'` to a `number`. The default value is `false`. + * @property {boolean} [passthroughBigInt=false] Set to `true` to NOT apply data casting to `type='bigint'` columns. Otherwise, + * there will be an attempt to cast columns with `type='bigint'` to a `BigInt`. The default value is `false`. + * @property {boolean} [passthroughDate=false] Set to `true` to NOT apply data casting to `type='date'` columns. Otherwise, + * there will be an attempt to cast columns with `type='date'` to a `Date`. The default value is `false`. + */ + +/** + * The output of `createCSVNormalizationStream`. + * @typedef {Object} createCSVNormalizationStreamRow + * @property {string} name The column name after normalization to camelCase. + * @property {string} displayName The desired column name in the output CSV. + * @property {string|number} value The value of the field after attempted data casting. If the original field was empty, this value + * is set to the provided `defaultValue` or `''` (empty string) if no `defaultValue` was provided. + * @property {boolean} emptyField Boolean to indicate whether the original field was empty. + */ + +/** + * Create a `TransformStream` to transform raw CSV data into a JavaScript object using provided metadata. + * + * Use this function to undo common CSV data mangling caused by spreadsheet programs such as Excel and perform other common + * cleanup such as renaming headers and removing empty rows. + * @param {Array} headers An array of metadata objects to configure the data casting and + * cleanup transformations. + * @param {createCSVNormalizationStreamOptions} options Object containing flags to configure the stream logic. + * @returns {TransformStream} A `TransformStream` where each chunk is one row of the CSV file after normalization, represented + * as a JSON string of a `createCSVNormalizationStreamRow` object. + */ +export function createCSVNormalizationStream(headers, options = {}) { + options.passthroughEmptyRows ??= false; + options.passthroughNumber ??= false; + options.passthroughBigInt ??= false; + options.passthroughDate ??= false; + + const columns = []; + for (const { name, type, displayName = name, defaultValue = null } of headers) { + let normalizedType = type.toLowerCase(); + if (!['string', 'number', 'bigint', 'date'].includes(normalizedType)) { + console.warn(`Type "${normalizedType}" is not supported, defaulting to string.`); + normalizedType = 'string'; + } + columns.push({ + name: toCamelCase(name), + type: normalizedType, + displayName, + defaultValue, + index: null + }); + } + + let firstChunk = true; + return new TransformStream({ + transform(chunk, controller) { + const row = JSON.parse(chunk); + + // Assume first row is headers and use it to prepare the columns object + // Note: the headers row is NOT forwarded downstream + if (firstChunk) { + const normalizedRow = row.map(f => toCamelCase(f)); + let i = 0; + for (const header of normalizedRow) { + const col = columns.find(c => c.name === header); + if (col !== undefined) { // Drop columns provided in the CSV but not the headers input + col.index = i; + } + i++; + } + columns.filter(c => c.index !== null); // Drop columns provided in the headers input but not in the CSV + firstChunk = false; + return; + } + + const out = []; + for (const { name, type, displayName, defaultValue, index } of columns) { + let value = row[index]; + const emptyField = value === '' ? true : false; + switch (type) { + case 'string': break; + case 'number': value = options.passthroughNumber ? value : fixExcelNumber(value); break; + case 'bigint': value = options.passthroughBigInt ? value : fixExcelBigInt(value); break; + case 'date': value = options.passthroughDate ? value : fixExcelDate(value); break; + } + if (emptyField && defaultValue !== null) { + value = defaultValue; + } + out.push({ name, displayName, value, emptyField }); + } + + if (!options.passthroughEmptyRows && out.every(f => f.emptyField)) { return; } + + controller.enqueue(JSON.stringify(out)); + } + }); +} + +/** + * Create a `TransformStream` to transform a `createCSVNormalizationStreamRow` into an array that can be converted to CSV data. + * + * @returns {TransformStream} A `TransformStream` where each chunk is one row of the CSV file. + */ +export function createCSVDenormalizationStream() { + let firstRow = true; + return new TransformStream({ + transform(chunk, controller) { + const row = JSON.parse(chunk); + if (firstRow) { + firstRow = false; + controller.enqueue(JSON.stringify(row.map(f => f.displayName))); + } + controller.enqueue(JSON.stringify(row.map(f => f.value))); + } + }); +} diff --git a/test/core.test.js b/test/core.test.js new file mode 100644 index 0000000..22f51b2 --- /dev/null +++ b/test/core.test.js @@ -0,0 +1,301 @@ +import { suite, test } from 'node:test'; +import { deepStrictEqual } from 'node:assert/strict'; +import { unlink } from 'node:fs/promises'; + +import { csvStreamEqual, csvStreamEqualWritable, createCSVMockStream, createTempFile } from './utils.js'; +import { arrayToCSVString, createCSVReadableStream, createCSVTransformStream, createCSVWritableStream } from '../src/core.js'; + +suite('arrayToCSVString', { concurrency: true }, () => { + const vectors = [ + { name: 'converts simple row', input: ['abc', '123'], output: 'abc,123\r\n' }, + { name: 'converts row with commas', input: ['a,bc', '12,3'], output: '"a,bc","12,3"\r\n' }, + { name: 'converts row with double quotes', input: ['a"bc', '12"3'], output: '"a""bc","12""3"\r\n' }, + { name: 'converts row with newlines', input: ['a\nbc', '12\r\n3'], output: '"a\nbc","12\r\n3"\r\n' }, + { name: 'converts row with mixed escaped characters', input: ['a,"\nbc', '12,"\r\n3'], output: '"a,""\nbc","12,""\r\n3"\r\n' }, + { name: 'converts single value (no commas) row', input: ['abc'], output: 'abc\r\n' }, + { name: 'converts empty row', input: [''], output: '\r\n' }, + { name: 'converts empty row with multiple fields', input: ['', '', '', ''], output: ',,,\r\n' } + ]; + for (const { name, input, output } of vectors) { + test(name, { concurrency: true }, () => { + deepStrictEqual(arrayToCSVString(input), output); + }); + } +}); + +suite('createCSVReadableStream', { concurrency: true }, () => { + test('parses simple CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/simple.csv'); + await csvStreamEqual(stream, [ + ['column1', 'column2', 'column3'], + ['abc', 'def', 'ghi'], + ['123', '456', '789'], + ['aaa', 'bbb', 'ccc'] + ]); + }); + test('parses escaping CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/escaping.csv'); + await csvStreamEqual(stream, [ + ['name', 'value'], + ['Three spaces', ' '], + ['Three commas', ',,,'], + ['Three newlines', '\r\n\r\n\r\n'], + ['Three unescaped double quotes', 'a"""'], + ['Three escaped double quotes', '"""'], + ['Unescaped double quotes around delimiter 1 "', 'a'], + ['Unescaped double quotes around delimiter 2 ""', 'a'], + ['Unescaped double quotes around delimiter 3 """', 'a'], + ['Unescaped double quotes around delimiter 4', ' "'], + ['Unescaped double quotes around delimiter 5', ' ""'], + ['Unescaped double quotes around delimiter 6', ' """'], + ['Spaces before and after a value', ' abc '], + ['Spaces before and after three unescaped double quotes', ' """ '], + ['Spaces trailing escaped double quotes', 'abc '], + ['Characters trailing escaped double quotes', 'abc def'], + ['Unescaped double quotes trailing escaped double quotes', 'abc " def 123 """ 456'], + ['Unescaped "double quotes" trailing escaped double quotes', 'abc """ def 123 """ 456'], + ['Unicode test 1', '你好'], + ['Unicode test 2', '😂👌👍'], + ['Unicode test 3', '🏴‍☠️👨‍👩‍👧‍👦'], + ['Mixed', ',\r\n",\r\n",\r\n"'] + ]); + }); + test('parses escaping edges CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/escaping-edges.csv'); + await csvStreamEqual(stream, [ + [',,,', '"""'], + ['"""', ',,,'], + ['a"b""', 'a"b""'], + ['a"b""', 'a"b""'] + ]); + }); + test('parses sparse CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/sparse.csv'); + await csvStreamEqual(stream, [ + ['column1', 'column2', 'column3'], + [''], + ['', ''], + ['', '', ''], + ['', '', '', ''], + ['', '', '', '', ''], + ['aaa', 'bbb', 'ccc'], + ['111', '222', '333', '444', '555', '666', '777', '888'], + ['', 'hhh', 'iii'], + ['ggg', '', 'iii'], + ['ggg', 'hhh', ''], + ['', 'hhh', ''], + ['ggg', '', ''], + ['', '', 'iii'], + ['', '', ''], + ['', '', ''], + ['', '', ''], + [''], + [''] + ]); + }); + test('parses line feed end-of-line CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/lf-eol.csv'); + await csvStreamEqual(stream, [ + ['column1', 'column2'], + ['a', 'b'], + ['c', 'd'], + ['1', '2'], + ['3', '4'] + ]); + }); + test('parses byte order mark CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/bom.csv'); + await csvStreamEqual(stream, [ + ['column1', 'column2'], + ['ab', 'cd'], + ['12', '34'] + ]); + }); + test('parses single value (no commas or newlines) CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/single-value.csv'); + await csvStreamEqual(stream, [ + ['abc'] + ]); + }); + test('parses single column (no commas) CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/single-column.csv'); + await csvStreamEqual(stream, [ + ['abc'], + ['def'], + ['ghi'] + ]); + }); + test('parses single row (no newlines) CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/single-row.csv'); + await csvStreamEqual(stream, [ + ['abc', 'def', 'ghi'] + ]); + }); + test('parses blank CSV', { concurrency: true }, async () => { + const stream = createCSVReadableStream('./test/data/blank.csv'); + await csvStreamEqual(stream, [ + [''] + ]); + }); +}); + +suite('createCSVTransformStream', { concurrency: true }, () => { + test('transforms data in place', { concurrency: true }, async () => { + function timesTwo(row) { + return [Number(row[0]) * 2, Number(row[1]) * 2]; + } + const stream = createCSVMockStream([ + ['columnA', 'columnB'], + ['1', '1'], + ['100', '100'], + ['223423', '455947'], + ['348553', '692708'], + ['536368', '676147'] + ]).pipeThrough(createCSVTransformStream(timesTwo)); + await csvStreamEqual(stream, [ + ['columnA', 'columnB'], + [2, 2], + [200, 200], + [446846, 911894], + [697106, 1385416], + [1072736, 1352294] + ]); + }); + test('can add new column', { concurrency: true }, async () => { + let firstRow = true; + function sum(row) { + if (firstRow) { + firstRow = false; + return [...row, 'sum']; + } + return [...row, Number(row[0]) + Number(row[1])]; + } + const stream = createCSVMockStream([ + ['columnA', 'columnB'], + ['1', '1'], + ['100', '100'], + ['223423', '455947'], + ['348553', '692708'], + ['536368', '676147'] + ]).pipeThrough(createCSVTransformStream(sum, { includeHeaders: true })); + await csvStreamEqual(stream, [ + ['columnA', 'columnB', 'sum'], + ['1', '1', 2], + ['100', '100', 200], + ['223423', '455947', 679370], + ['348553', '692708', 1041261], + ['536368', '676147', 1212515] + ]); + }); +}); + +suite('createCSVWritableStream', { concurrency: true }, () => { + test('writes simple CSV', { concurrency: true }, async (t) => { + const temp = await createTempFile(); + t.after(async () => await unlink(temp)); + const csv = [ + ['column1', 'column2', 'column3'], + ['abc', 'def', 'ghi'], + ['123', '456', '789'], + ['aaa', 'bbb', 'ccc'] + ]; + await createCSVMockStream(csv).pipeTo(createCSVWritableStream(temp)); + await createCSVReadableStream(temp).pipeTo(csvStreamEqualWritable(csv)); + }); + test('writes escaping CSV', { concurrency: true }, async (t) => { + const temp = await createTempFile(); + t.after(async () => await unlink(temp)); + const csv = [ + ['name', 'value'], + ['Three spaces', ' '], + ['Three commas', ',,,'], + ['Three newlines', '\r\n\r\n\r\n'], + ['Three unescaped double quotes', 'a"""'], + ['Three escaped double quotes', '"""'], + ['Unescaped double quotes around delimiter 1 "', 'a'], + ['Unescaped double quotes around delimiter 2 ""', 'a'], + ['Unescaped double quotes around delimiter 3 """', 'a'], + ['Unescaped double quotes around delimiter 4', ' "'], + ['Unescaped double quotes around delimiter 5', ' ""'], + ['Unescaped double quotes around delimiter 6', ' """'], + ['Spaces before and after a value', ' abc '], + ['Spaces before and after three unescaped double quotes', ' """ '], + ['Spaces trailing escaped double quotes', 'abc '], + ['Characters trailing escaped double quotes', 'abc def'], + ['Unescaped double quotes trailing escaped double quotes', 'abc " def 123 """ 456'], + ['Unescaped "double quotes" trailing escaped double quotes', 'abc """ def 123 """ 456'], + ['Unicode test 1', '你好'], + ['Unicode test 2', '😂👌👍'], + ['Unicode test 3', '🏴‍☠️👨‍👩‍👧‍👦'], + ['Mixed', ',\r\n",\r\n",\r\n"'] + ]; + await createCSVMockStream(csv).pipeTo(createCSVWritableStream(temp)); + await createCSVReadableStream(temp).pipeTo(csvStreamEqualWritable(csv)); + }); + test('writes escaping edges CSV', { concurrency: true }, async (t) => { + const temp = await createTempFile(); + t.after(async () => await unlink(temp)); + const csv = [ + [',,,', '"""'], + ['"""', ',,,'], + ['a"b""', 'a"b""'], + ['a"b""', 'a"b""'] + ]; + await createCSVMockStream(csv).pipeTo(createCSVWritableStream(temp)); + await createCSVReadableStream(temp).pipeTo(csvStreamEqualWritable(csv)); + }); + test('writes sparse CSV', { concurrency: true }, async (t) => { + const temp = await createTempFile(); + t.after(async () => await unlink(temp)); + const csv = [ + ['column1', 'column2', 'column3'], + [''], + ['', ''], + ['', '', ''], + ['', '', '', ''], + ['', '', '', '', ''], + ['aaa', 'bbb', 'ccc'], + ['111', '222', '333', '444', '555', '666', '777', '888'], + ['', 'hhh', 'iii'], + ['ggg', '', 'iii'], + ['ggg', 'hhh', ''], + ['', 'hhh', ''], + ['ggg', '', ''], + ['', '', 'iii'], + ['', '', ''], + ['', '', ''], + ['', '', ''], + [''], + [''] + ]; + await createCSVMockStream(csv).pipeTo(createCSVWritableStream(temp)); + await createCSVReadableStream(temp).pipeTo(csvStreamEqualWritable(csv)); + }); + test('writes single value (no commas or newlines)', { concurrency: true }, async (t) => { + const temp = await createTempFile(); + t.after(async () => await unlink(temp)); + const csv = [ + ['abc'] + ]; + await createCSVMockStream(csv).pipeTo(createCSVWritableStream(temp)); + await createCSVReadableStream(temp).pipeTo(csvStreamEqualWritable(csv)); + }); + test('writes single row (no newlines) CSV', { concurrency: true }, async (t) => { + const temp = await createTempFile(); + t.after(async () => await unlink(temp)); + const csv = [ + ['abc', 'def', 'ghi'] + ]; + await createCSVMockStream(csv).pipeTo(createCSVWritableStream(temp)); + await createCSVReadableStream(temp).pipeTo(csvStreamEqualWritable(csv)); + }); + test('writes blank CSV', { concurrency: true }, async (t) => { + const temp = await createTempFile(); + t.after(async () => await unlink(temp)); + const csv = [ + [''] + ]; + await createCSVMockStream(csv).pipeTo(createCSVWritableStream(temp)); + await createCSVReadableStream(temp).pipeTo(csvStreamEqualWritable(csv)); + }); +}); \ No newline at end of file diff --git a/test/data/blank.csv b/test/data/blank.csv new file mode 100644 index 0000000..e69de29 diff --git a/test/data/bom.csv b/test/data/bom.csv new file mode 100644 index 0000000..458624c --- /dev/null +++ b/test/data/bom.csv @@ -0,0 +1,3 @@ +column1,column2 +ab,cd +12,34 \ No newline at end of file diff --git a/test/data/escaping-edges.csv b/test/data/escaping-edges.csv new file mode 100644 index 0000000..741f397 --- /dev/null +++ b/test/data/escaping-edges.csv @@ -0,0 +1,4 @@ +",,,","""""""" +"""""""",",,," +"a"""b"","a"""b"" +"a""b""""","a""b""""" \ No newline at end of file diff --git a/test/data/escaping.csv b/test/data/escaping.csv new file mode 100644 index 0000000..12be60f --- /dev/null +++ b/test/data/escaping.csv @@ -0,0 +1,28 @@ +name,value +Three spaces, +Three commas,",,," +Three newlines," + + +" +Three unescaped double quotes,a""" +Three escaped double quotes,"""""""" +Unescaped double quotes around delimiter 1 ",a +Unescaped double quotes around delimiter 2 "",a +Unescaped double quotes around delimiter 3 """,a +Unescaped double quotes around delimiter 4, " +Unescaped double quotes around delimiter 5, "" +Unescaped double quotes around delimiter 6, """ +Spaces before and after a value, abc +Spaces before and after three unescaped double quotes, """ +Spaces trailing escaped double quotes,"abc" +Characters trailing escaped double quotes,"abc" def +Unescaped double quotes trailing escaped double quotes,"abc "" def" 123 """ 456 +"Unescaped ""double quotes"" trailing escaped double" quotes,"abc """""" def" 123 """ 456 +Unicode test 1,你好 +Unicode test 2,😂👌👍 +Unicode test 3,🏴‍☠️👨‍👩‍👧‍👦 +Mixed,", +"", +"", +""" \ No newline at end of file diff --git a/test/data/lf-eol.csv b/test/data/lf-eol.csv new file mode 100644 index 0000000..b926aec --- /dev/null +++ b/test/data/lf-eol.csv @@ -0,0 +1,5 @@ +column1,column2 +a,b +c,d +1,2 +3,4 \ No newline at end of file diff --git a/test/data/normalization.csv b/test/data/normalization.csv new file mode 100644 index 0000000..db5bf7b --- /dev/null +++ b/test/data/normalization.csv @@ -0,0 +1,8 @@ +extraColumn1,string col,numberCol,BIGINT_COL,date-col,extraColumn2 +extra1,abc 👨‍👩‍👧‍👦," 123,456,789.123456 ",1E+18,45292,extra2 +extra1,", "" 🏴‍☠️"," 1,000.00 ",-1.23456789E+25,45444,extra2 +extra1,,,,,extra2 +extra1,,"123,100",1000000000000000000,12/31/24,extra2 +,,,,, +,,,,, +,,,,, \ No newline at end of file diff --git a/test/data/simple.csv b/test/data/simple.csv new file mode 100644 index 0000000..43a84f5 --- /dev/null +++ b/test/data/simple.csv @@ -0,0 +1,4 @@ +column1,column2,column3 +abc,def,ghi +123,456,789 +aaa,bbb,ccc \ No newline at end of file diff --git a/test/data/single-column.csv b/test/data/single-column.csv new file mode 100644 index 0000000..e830b82 --- /dev/null +++ b/test/data/single-column.csv @@ -0,0 +1,3 @@ +abc +def +ghi \ No newline at end of file diff --git a/test/data/single-row.csv b/test/data/single-row.csv new file mode 100644 index 0000000..2d08fd7 --- /dev/null +++ b/test/data/single-row.csv @@ -0,0 +1 @@ +abc,def,ghi \ No newline at end of file diff --git a/test/data/single-value.csv b/test/data/single-value.csv new file mode 100644 index 0000000..f2ba8f8 --- /dev/null +++ b/test/data/single-value.csv @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/test/data/sparse.csv b/test/data/sparse.csv new file mode 100644 index 0000000..6b98f12 --- /dev/null +++ b/test/data/sparse.csv @@ -0,0 +1,19 @@ +column1,column2,column3 + +, +,, +,,, +,,,, +aaa,bbb,ccc +111,222,333,444,555,666,777,888 +,hhh,iii +ggg,,iii +ggg,hhh, +,hhh, +ggg,, +,,iii +,, +,, +,, + + diff --git a/test/normalization.test.js b/test/normalization.test.js new file mode 100644 index 0000000..7f3e611 --- /dev/null +++ b/test/normalization.test.js @@ -0,0 +1,273 @@ +import { suite, test } from 'node:test'; +import { deepStrictEqual } from 'node:assert/strict'; + +import { csvStreamEqual, csvStreamEqualWritable, createCSVMockStream } from './utils.js'; +import { createCSVReadableStream } from '../src/core.js'; +import { toCamelCase, expandScientificNotation, fixExcelNumber, fixExcelBigInt, fixExcelDate, createCSVNormalizationStream, createCSVDenormalizationStream } from '../src/normalization.js'; + +suite('toCamelCase', { concurrency: true }, () => { + test('does not modify strings already in camelCase', { concurrency: true }, () => { + const test1 = 'example'; + const test2 = 'alreadyInCamelCase'; + const test3 = 'withAcronymsABCNotDEFCapitalizedGHI'; + deepStrictEqual(toCamelCase(test1), test1); + deepStrictEqual(toCamelCase(test2), test2); + deepStrictEqual(toCamelCase(test3), test3); + }); + test('converts single word with capital first letter', { concurrency: true }, () => { + deepStrictEqual(toCamelCase('Example'), 'example'); + deepStrictEqual(toCamelCase('ExampleOne'), 'exampleOne'); + }); + test('converts strings with spaces', { concurrency: true }, () => { + deepStrictEqual(toCamelCase('example one'), 'exampleOne'); + deepStrictEqual(toCamelCase('example two with more words'), 'exampleTwoWithMoreWords'); + }); + test('converts strings with hyphens', { concurrency: true }, () => { + deepStrictEqual(toCamelCase('example-one'), 'exampleOne'); + deepStrictEqual(toCamelCase('example-two-with-more-words'), 'exampleTwoWithMoreWords'); + }); + test('converts strings with underscores', { concurrency: true }, () => { + deepStrictEqual(toCamelCase('example_one'), 'exampleOne'); + deepStrictEqual(toCamelCase('example_two_with_more_words'), 'exampleTwoWithMoreWords'); + }); + test('converts strings with mixed casing', { concurrency: true }, () => { + deepStrictEqual(toCamelCase('ExAmPlE oNe'), 'exampleOne'); + deepStrictEqual(toCamelCase('eXaMpLe TwO-wItH_mOrE wOrDs'), 'exampleTwoWithMoreWords'); + }); + test('converts strings with excess whitespace', { concurrency: true }, () => { + deepStrictEqual(toCamelCase(' exampleOne \r\n'), 'exampleOne'); + deepStrictEqual(toCamelCase(' \r\n\r\n example Two\rWith More \r\nWords \r\n'), 'exampleTwoWithMoreWords'); + }); +}); + +suite('expandScientificNotation', { concurrency: true }, () => { + test('returns null for numbers not in scientific notation', { concurrency: true }, () => { + deepStrictEqual(expandScientificNotation('123'), null); + deepStrictEqual(expandScientificNotation('123n'), null); + deepStrictEqual(expandScientificNotation('123.456'), null); + }); + test('returns null for values that are not strings of numbers', { concurrency: true }, () => { + deepStrictEqual(expandScientificNotation('abc'), null); + deepStrictEqual(expandScientificNotation(123), null); + deepStrictEqual(expandScientificNotation(123n), null); + deepStrictEqual(expandScientificNotation([]), null); + deepStrictEqual(expandScientificNotation({}), null); + }); + test('returns expanded form for positive exponents', { concurrency: true }, () => { + deepStrictEqual(expandScientificNotation('1E2'), '100'); + deepStrictEqual(expandScientificNotation('1E10'), '10000000000'); + deepStrictEqual(expandScientificNotation('1E18'), '1000000000000000000'); + deepStrictEqual(expandScientificNotation('1E+18'), '1000000000000000000'); + }); + test('returns expanded form for negative mantissas', { concurrency: true }, () => { + deepStrictEqual(expandScientificNotation('-1E2'), '-100'); + deepStrictEqual(expandScientificNotation('-1E10'), '-10000000000'); + deepStrictEqual(expandScientificNotation('-1E18'), '-1000000000000000000'); + deepStrictEqual(expandScientificNotation('-1E+18'), '-1000000000000000000'); + }); + test('returns expanded form for mantissas with decimals', { concurrency: true }, () => { + deepStrictEqual(expandScientificNotation('1.1E2'), '110'); + deepStrictEqual(expandScientificNotation('1.11E2'), '111'); + deepStrictEqual(expandScientificNotation('1.111E2'), '111.1'); + deepStrictEqual(expandScientificNotation('1.12345E10'), '11234500000'); + deepStrictEqual(expandScientificNotation('1.123456789123456789E18'), '1123456789123456789'); + deepStrictEqual(expandScientificNotation('1.1234567891234567891234E18'), '1123456789123456789.1234'); + }); + test('truncates decimals correctly', { concurrency: true }, () => { + deepStrictEqual(expandScientificNotation('1.1E2', true), '110'); + deepStrictEqual(expandScientificNotation('1.11E2', true), '111'); + deepStrictEqual(expandScientificNotation('1.111E2', true), '111'); + deepStrictEqual(expandScientificNotation('1.1111E2', true), '111'); + deepStrictEqual(expandScientificNotation('1.11111E2', true), '111'); + deepStrictEqual(expandScientificNotation('1.111111111E10', true), '11111111110'); + deepStrictEqual(expandScientificNotation('1.1111111111E10', true), '11111111111'); + deepStrictEqual(expandScientificNotation('1.11111111111E10', true), '11111111111'); + deepStrictEqual(expandScientificNotation('1.111111111111E10', true), '11111111111'); + }); +}); + +suite('fixExcelNumber', { concurrency: true }, () => { + test('parses numbers', { concurrency: true }, () => { + deepStrictEqual(fixExcelNumber('123'), 123); + deepStrictEqual(fixExcelNumber('123.456'), 123.456); + }); + test('fixes numbers mangled to dates', { concurrency: true }, () => { + deepStrictEqual(fixExcelNumber('1/1/1900'), 1); + deepStrictEqual(fixExcelNumber('1/2/1900'), 2); + deepStrictEqual(fixExcelNumber('2/28/1900'), 59); + deepStrictEqual(fixExcelNumber('2/29/1900'), 60); + deepStrictEqual(fixExcelNumber('3/1/1900'), 61); + deepStrictEqual(fixExcelNumber('12/31/1969'), 25568); + deepStrictEqual(fixExcelNumber('1/1/1970'), 25569); + deepStrictEqual(fixExcelNumber('1/2/1970'), 25570); + deepStrictEqual(fixExcelNumber('12/31/2023'), 45291); + deepStrictEqual(fixExcelNumber('1/1/2024'), 45292); + }); + test('fixes numbers mangled to accounting format', { concurrency: true }, () => { + deepStrictEqual(fixExcelNumber(' $123.00 '), 123); + deepStrictEqual(fixExcelNumber(' $123.45 '), 123.45); + deepStrictEqual(fixExcelNumber(' $10,000,000.00 '), 10000000); + }); + test('passes through non-number values', { concurrency: true }, () => { + deepStrictEqual(fixExcelNumber('abc'), 'abc'); + }); + test('passes through blank value', { concurrency: true }, () => { + deepStrictEqual(fixExcelNumber(''), ''); + }); +}); + +suite('fixExcelBigInt', { concurrency: true }, () => { + test('parses BigInt', { concurrency: true }, () => { + deepStrictEqual(fixExcelBigInt('123'), '123'); + deepStrictEqual(fixExcelBigInt('123456'), '123456'); + deepStrictEqual(fixExcelBigInt('123456789123456789123456789'), '123456789123456789123456789'); + }); + test('fixes BigInt mangled to scientific notation', { concurrency: true }, () => { + deepStrictEqual(fixExcelBigInt('1E+18'), '1000000000000000000'); + deepStrictEqual(fixExcelBigInt('1.123456789123456789E18'), '1123456789123456789'); + deepStrictEqual(fixExcelBigInt('1.1234567891234567891234E18'), '1123456789123456789'); + }); + test('fixes BigInt mangled to accounting format', { concurrency: true }, () => { + deepStrictEqual(fixExcelBigInt(' $123.00 '), '123'); + deepStrictEqual(fixExcelBigInt(' $123.45 '), '123'); + deepStrictEqual(fixExcelBigInt(' $1,123,123,123,123,123,123.12 '), '1123123123123123123'); + }); + test('passes through non-BigInt values', { concurrency: true }, () => { + deepStrictEqual(fixExcelBigInt('abc'), 'abc'); + }); + test('passes through blank value', { concurrency: true }, () => { + deepStrictEqual(fixExcelBigInt(''), ''); + }); +}); + +suite('fixExcelDate', { concurrency: true }, () => { + test('parses dates', { concurrency: true }, () => { + deepStrictEqual(fixExcelDate('12/31/24').valueOf(), new Date('12/31/24').valueOf()); + }); + test('fixes dates mangled to numbers', { concurrency: true }, () => { + deepStrictEqual(fixExcelDate('1').valueOf(), new Date('1900-01-01T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('2').valueOf(), new Date('1900-01-02T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('59').valueOf(), new Date('1900-02-28T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('60').valueOf(), new Date('1900-02-29T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('61').valueOf(), new Date('1900-03-01T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('25568').valueOf(), new Date('1969-12-31T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('25569').valueOf(), new Date('1970-01-01T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('25570').valueOf(), new Date('1970-01-02T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('45291').valueOf(), new Date('2023-12-31T00:00:00.000Z').valueOf()); + deepStrictEqual(fixExcelDate('45292').valueOf(), new Date('2024-01-01T00:00:00.000Z').valueOf()); + }); + test('fixes date times mangled to numbers', { concurrency: true }, () => { + deepStrictEqual(fixExcelDate('43915.31372').valueOf(), new Date('2020-03-25T07:31:45.407Z').valueOf()); + deepStrictEqual(fixExcelDate('44067.63').valueOf(), new Date('2020-08-24T15:07:11.999Z').valueOf()); + deepStrictEqual(fixExcelDate('44309.63502').valueOf(), new Date('2021-04-23T15:14:25.728Z').valueOf()); + }); + test('passes through non-date values', { concurrency: true }, () => { + deepStrictEqual(fixExcelDate('abc'), 'abc'); + }); + test('passes through blank value', { concurrency: true }, () => { + deepStrictEqual(fixExcelDate(''), ''); + }); +}); + +suite('createCSVNormalizationStream', { concurrency: true }, () => { + test('can remove extra columns', { concurrency: true }, async () => { + await createCSVMockStream([ + ['extra1', 'columnA', 'extra2', 'extra3', 'columnB'], + ['123', 'a', '123', '123', 'b'] + ]) + .pipeThrough(createCSVNormalizationStream([ + { name: 'columnA', type: 'string' }, + { name: 'columnB', type: 'string' } + ])) + .pipeThrough(createCSVDenormalizationStream()) + .pipeTo(csvStreamEqualWritable([ + ['columnA', 'columnB'], + ['a', 'b'] + ])); + }); + test('can reorder columns', { concurrency: true }, async () => { + await createCSVMockStream([ + ['columnC', 'columnA', 'columnB'], + ['c', 'a', 'b'] + ]) + .pipeThrough(createCSVNormalizationStream([ + { name: 'columnA', type: 'string' }, + { name: 'columnB', type: 'string' }, + { name: 'columnC', type: 'string' } + ])) + .pipeThrough(createCSVDenormalizationStream()) + .pipeTo(csvStreamEqualWritable([ + ['columnA', 'columnB', 'columnC'], + ['a', 'b', 'c'] + ])); + }); + test('can rename columns', { concurrency: true }, async () => { + await createCSVMockStream([ + ['columnA', 'columnB'], + ['a', 'b'] + ]) + .pipeThrough(createCSVNormalizationStream([ + { name: 'columnA', type: 'string', displayName: 'Column A' }, + { name: 'columnB', type: 'string', displayName: 'Column B' } + ])) + .pipeThrough(createCSVDenormalizationStream()) + .pipeTo(csvStreamEqualWritable([ + ['Column A', 'Column B'], + ['a', 'b'] + ])); + }); + test('can remove empty rows', { concurrency: true }, async () => { + await createCSVMockStream([ + ['columnA', 'columnB'], + ['',''], + ['a', 'b'], + ['',''], + ['',''], + ['',''] + ]) + .pipeThrough(createCSVNormalizationStream([ + { name: 'columnA', type: 'string' }, + { name: 'columnB', type: 'string' } + ])) + .pipeThrough(createCSVDenormalizationStream()) + .pipeTo(csvStreamEqualWritable([ + ['columnA', 'columnB'], + ['a', 'b'] + ])); + }); + test('can ignore incorrect data types', { concurrency: true }, async () => { + await createCSVMockStream([ + ['columnA', 'columnB'], + ['a', 'b'] + ]) + .pipeThrough(createCSVNormalizationStream([ + { name: 'columnA', type: 'abc' }, + { name: 'columnB', type: 'def' } + ])) + .pipeThrough(createCSVDenormalizationStream()) + .pipeTo(csvStreamEqualWritable([ + ['columnA', 'columnB'], + ['a', 'b'] + ])); + }); +}); + +suite('createCSVNormalizationStream and createCSVDenormalizationStream end-to-end', { concurrency: true }, () => { + test('can normalize', { concurrency: true }, async () => { + const headers = [ + { name: 'stringCol', displayName: 'String Column', type: 'string', defaultValue: 'N/A' }, + { name: 'numberCol', displayName: 'Number Column', type: 'number' }, + { name: 'bigintCol', displayName: 'BigInt Column', type: 'bigint' }, + { name: 'dateCol', displayName: 'Date Column', type: 'date' } + ] + const stream = createCSVReadableStream('./test/data/normalization.csv') + .pipeThrough(createCSVNormalizationStream(headers)) + .pipeThrough(createCSVDenormalizationStream()); + await csvStreamEqual(stream, [ + ['String Column', 'Number Column', 'BigInt Column', 'Date Column'], + ['abc 👨‍👩‍👧‍👦', 123456789.123456, '1000000000000000000', '2024-01-01T00:00:00.000Z'], + [', " 🏴‍☠️', 1000, '-1234567890000000000000000', '2024-06-01T00:00:00.000Z'], + ['N/A', 123100, '1000000000000000000', '2024-12-31T08:00:00.000Z'] + ]); + }); +}); \ No newline at end of file diff --git a/test/utils.js b/test/utils.js new file mode 100644 index 0000000..d95aa4c --- /dev/null +++ b/test/utils.js @@ -0,0 +1,104 @@ +import { deepStrictEqual } from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { open } from 'node:fs/promises'; +import { normalize } from 'node:path'; + +/** + * Log an entire stream along with the field count for each row and the total row count. + * + * @param {ReadableStream} stream A stream of CSV data. + * @param {string} [id] Optional label to include in the log. + */ +export async function csvStreamLog(stream, id) { + let out = '============================================================\n'; + out += `\x1b[45m${id ?? ''}\x1b[0m`; + out += '\n------------------------------------------------------------\n'; + let totalRows = 0; + for await (const value of stream) { + const row = JSON.parse(value); + const rowNoNewlines = row.map(f => f.replaceAll('\n', '\\n').replaceAll('\r', '\\r')); + out += `${row.length} || ${rowNoNewlines.join(' | ')}\n`; + totalRows++; + } + out += '------------------------------------------------------------\n'; + out += `Total row count: ${totalRows}\n`; + out += '============================================================'; + console.log(out); +} + +/** + * Compare each field in a CSV stream with a given array, assuming each chunk of the stream is a + * serialized `Array` representing a row of a CSV. + * + * @param {ReadableStream} stream A stream of CSV data. + * @param {Array>} csv A 2-D array representing the CSV file, where each inner array + * is a row of the CSV. + */ +export async function csvStreamEqual(stream, csv) { + let i = 0; + for await (const value of stream) { + const row = JSON.parse(value); + for (let j = 0; j < csv[i].length; j++) { + deepStrictEqual(row[j], csv[i][j]); + } + i++; + } +} + +/** + * A `WritableStream` version of `csvStreamEqual`. Compares each field in a CSV stream with a given + * array, assuming each chunk of the stream is a serialized `Array` representing a row of a CSV. + * + * @param {Array>} csv A 2-D array representing the CSV file, where each inner array + * is a row of the CSV. + * @returns {WritableStream} A `WritableStream` sink of a CSV stream that will be compared against the given array. + */ +export function csvStreamEqualWritable(csv) { + let i = 0; + return new WritableStream({ + write(chunk) { + const row = JSON.parse(chunk); + for (let j = 0; j < csv[i].length; j++) { + deepStrictEqual(row[j], csv[i][j]); + } + i++; + } + }); +} + +/** + * Simulate a stream of CSV data for test mocking purposes. + * + * @param {Array>} data A 2-D array representing the CSV file to mock, where each inner array + * is a row of the CSV. + * @returns {ReadableStream} A `ReadableStream` where each row is emitted as a separate chunk. + */ +export function createCSVMockStream(data) { + const chunks = (function* () { + for (const chunk of data) { + yield chunk; + } + })(); + return new ReadableStream({ + pull(controller) { + const { value, done } = chunks.next(); + if (done) { + controller.close(); + } + else { + controller.enqueue(JSON.stringify(value)); + } + } + }); +} + +/** + * Create a new empty temporary file for testing purposes. + * + * @returns {Promise} A `Promise` for the full path to a newly-created temporary file. + */ +export async function createTempFile() { + const path = normalize(`${tmpdir()}/scsv_test_${crypto.randomUUID()}.tmp`); + await open(path, 'a+'); + return path; +}