From: Ardis Lu Date: Sun, 19 Jan 2025 07:47:34 +0000 (-0800) Subject: Refactor createCSVNormalizationStream to CSVNormalizer X-Git-Tag: v0.1.0~112 X-Git-Url: https://git.ardis.lu/?a=commitdiff_plain;h=b8d50b36032bd934e3a9dca3eae7a284e32cb32a;p=csvio Refactor createCSVNormalizationStream to CSVNormalizer --- diff --git a/README.md b/README.md index 83d5abe..e11b450 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ const writableStream = new CSVWriter('./data/example-out.csv'); In addition to the core API, optional utilities are provided to address common CSV transformation tasks. -### `createCSVNormalizationStream` and `createCSVDenormalizationStream` +### `CSVNormalizer` and `createCSVDenormalizationStream` `TransformStream`s to parse CSVs that have been mangled by spreadsheet programs such as Excel. diff --git a/examples/ex4.js b/examples/ex4.js index 2d46f86..717b2fe 100644 --- a/examples/ex4.js +++ b/examples/ex4.js @@ -1,14 +1,14 @@ -// ## Example 4: Using `createCSVNormalizationStream` and `createCSVDenormalizationStream` +// ## Example 4: Using `CSVNormalizer` and `createCSVDenormalizationStream` // // These functions fix common CSV data mangling caused by Excel. In `ex4-in.csv`: // - The "Index" column has been mangled into Excel's "Accounting" format // - The "Phone Number" column has been mangled into scientific notation // - The "Join Timestamp" column has been mangled into a number // - There are trailing empty rows after deleting excess data from the CSV -// `createCSVNormalizationStream` performs transformations to un-mangle the above using only a declaration of the header names and data types. +// `CSVNormalizer` performs transformations to un-mangle the above using only a declaration of the header names and data types. // `createCSVDenormalizationStream` converts the data stream back into a CSV row. -import { CSVReader, CSVWriter, createCSVNormalizationStream, createCSVDenormalizationStream } from '../src/index.js'; +import { CSVReader, CSVWriter, CSVNormalizer, createCSVDenormalizationStream } from '../src/index.js'; // The order of items in `headers` is significant, it determines the order of the output CSV columns. // If an input column is not provided in `headers`, the column is removed from the output CSV. @@ -31,6 +31,6 @@ const headers = [ ]; await new CSVReader(new URL('./data/ex4-in.csv', import.meta.url)) - .pipeThrough(createCSVNormalizationStream(headers)) + .pipeThrough(new CSVNormalizer(headers)) .pipeThrough(createCSVDenormalizationStream()) .pipeTo(new CSVWriter(new URL('./data/ex4-out.csv', import.meta.url))); \ No newline at end of file diff --git a/src/index.js b/src/index.js index c78b4f1..b5c4138 100644 --- a/src/index.js +++ b/src/index.js @@ -1,2 +1,2 @@ export { CSVReader, CSVTransformer, CSVWriter } from './core.js'; -export { createCSVNormalizationStream, createCSVDenormalizationStream } from './normalization.js'; +export { CSVNormalizer, createCSVDenormalizationStream } from './normalization.js'; diff --git a/src/normalization.js b/src/normalization.js index a64f313..6b78111 100644 --- a/src/normalization.js +++ b/src/normalization.js @@ -164,7 +164,7 @@ export function fixExcelDate(str) { /** * Metadata about CSV columns used to normalize raw CSV data. - * @typedef {Object} CreateCSVNormalizationStreamHeader + * @typedef {Object} CSVNormalizerHeader * @property {string} name The name of the CSV column, in camelCase. Values in the header row of the CSV will be * transformed to camelCase for comparison purposes and for downstream usage (e.g., a header with the value `Example Column` * in the input CSV will match with a `name` value of `exampleColumn`). @@ -176,8 +176,8 @@ export function fixExcelDate(str) { */ /** - * Options to configure `createCSVNormalizationStream`. - * @typedef {Object} CreateCSVNormalizationStreamOptions + * Options to configure `CSVNormalizer`. + * @typedef {Object} CSVNormalizerOptions * @property {boolean} [passthroughEmptyRows=false] Set to `true` to send empty rows (i.e., rows where all field values are * `''`) downstream. Otherwise, empty rows will be removed. The default value is `false`. * @property {boolean} [passthroughNumber=false] Set to `true` to NOT apply data casting to `type='number'` columns. Otherwise, @@ -189,8 +189,8 @@ export function fixExcelDate(str) { */ /** - * The output of `createCSVNormalizationStream`. - * @typedef {Object} CreateCSVNormalizationStreamRow + * The output of `CSVNormalizer`. + * @typedef {Object} CSVNormalizerRow * @property {string} name The column name after normalization to camelCase. * @property {string} displayName The desired column name in the output CSV. * @property {string|number} value The value of the field after attempted data casting. If the original field was empty, this value @@ -199,85 +199,94 @@ export function fixExcelDate(str) { */ /** - * Create a `TransformStream` to transform raw CSV data into a JavaScript object using provided metadata. + * A `TransformStream` to transform raw CSV data into a JavaScript object using provided metadata. * - * Use this function to undo common CSV data mangling caused by spreadsheet programs such as Excel and perform other common + * Use this class to undo common CSV data mangling caused by spreadsheet programs such as Excel and perform other common * cleanup such as renaming headers and removing empty rows. - * @param {Array} headers An array of metadata objects to configure the data casting and - * cleanup transformations. - * @param {CreateCSVNormalizationStreamOptions} options Object containing flags to configure the stream logic. - * @returns {TransformStream} A `TransformStream` where each chunk is one row of the CSV file after normalization, represented - * as a JSON string of a `createCSVNormalizationStreamRow` object. + * @extends TransformStream */ -export function createCSVNormalizationStream(headers, options = {}) { - options.passthroughEmptyRows ??= false; - options.passthroughNumber ??= false; - options.passthroughBigInt ??= false; - options.passthroughDate ??= false; +export class CSVNormalizer extends TransformStream { + #columns = []; + #firstChunk = true; + #options; - const columns = []; - for (const { name, type, displayName = name, defaultValue = null } of headers) { - let normalizedType = type.toLowerCase(); - if (!['string', 'number', 'bigint', 'date'].includes(normalizedType)) { - console.warn(`Type "${normalizedType}" is not supported, defaulting to string.`); - normalizedType = 'string'; - } - columns.push({ - name: toCamelCase(name), - type: normalizedType, - displayName, - defaultValue, - index: null + /** + * @param {Array} headers An array of metadata objects to configure the data casting and + * cleanup transformations. + * @param {CSVNormalizerOptions} options Object containing flags to configure the stream logic. + */ + constructor(headers, options = {}) { + super({ + transform: (chunk, controller) => this.#transform(chunk, controller) }); + + options.passthroughEmptyRows ??= false; + options.passthroughNumber ??= false; + options.passthroughBigInt ??= false; + options.passthroughDate ??= false; + this.#options = options; + + for (const { name, type, displayName = name, defaultValue = null } of headers) { + let normalizedType = type.toLowerCase(); + if (!['string', 'number', 'bigint', 'date'].includes(normalizedType)) { + console.warn(`Type "${normalizedType}" is not supported, defaulting to string.`); + normalizedType = 'string'; + } + this.#columns.push({ + name: toCamelCase(name), + type: normalizedType, + displayName, + defaultValue, + index: null + }); + } } - let firstChunk = true; - return new TransformStream({ - transform(chunk, controller) { - const row = JSON.parse(chunk); + #transform(chunk, controller) { + const { passthroughNumber, passthroughBigInt, passthroughDate, passthroughEmptyRows } = this.#options; + const row = JSON.parse(chunk); - // Assume first row is headers and use it to prepare the columns object - // Note: the headers row is NOT forwarded downstream - if (firstChunk) { - const normalizedRow = row.map(f => toCamelCase(f)); - let i = 0; - for (const header of normalizedRow) { - const col = columns.find(c => c.name === header); - if (col !== undefined) { // Drop columns provided in the CSV but not the headers input - col.index = i; - } - i++; + // Assume first row is headers and use it to prepare the columns object + // Note: the headers row is NOT forwarded downstream + if (this.#firstChunk) { + const normalizedRow = row.map(f => toCamelCase(f)); + let i = 0; + for (const header of normalizedRow) { + const col = this.#columns.find(c => c.name === header); + if (col !== undefined) { // Drop columns provided in the CSV but not the headers input + col.index = i; } - columns.filter(c => c.index !== null); // Drop columns provided in the headers input but not in the CSV - firstChunk = false; - return; + i++; } + this.#columns.filter(c => c.index !== null); // Drop columns provided in the headers input but not in the CSV + this.#firstChunk = false; + return; + } - const out = []; - for (const { name, type, displayName, defaultValue, index } of columns) { - let value = row[index]; - const emptyField = value === '' ? true : false; - switch (type) { - case 'string': break; - case 'number': value = options.passthroughNumber ? value : fixExcelNumber(value); break; - case 'bigint': value = options.passthroughBigInt ? value : fixExcelBigInt(value); break; - case 'date': value = options.passthroughDate ? value : fixExcelDate(value); break; - } - if (emptyField && defaultValue !== null) { - value = defaultValue; - } - out.push({ name, displayName, value, emptyField }); + const out = []; + for (const { name, type, displayName, defaultValue, index } of this.#columns) { + let value = row[index]; + const emptyField = value === '' ? true : false; + switch (type) { + case 'string': break; + case 'number': value = passthroughNumber ? value : fixExcelNumber(value); break; + case 'bigint': value = passthroughBigInt ? value : fixExcelBigInt(value); break; + case 'date': value = passthroughDate ? value : fixExcelDate(value); break; + } + if (emptyField && defaultValue !== null) { + value = defaultValue; } + out.push({ name, displayName, value, emptyField }); + } - if (!options.passthroughEmptyRows && out.every(f => f.emptyField)) { return; } + if (!passthroughEmptyRows && out.every(f => f.emptyField)) { return; } - controller.enqueue(JSON.stringify(out)); - } - }); + controller.enqueue(JSON.stringify(out)); + } } /** - * Create a `TransformStream` to transform a `createCSVNormalizationStreamRow` into an array that can be converted to CSV data. + * Create a `TransformStream` to transform a `CSVNormalizerRow` into an array that can be converted to CSV data. * * @returns {TransformStream} A `TransformStream` where each chunk is one row of the CSV file. */ diff --git a/test/normalization.test.js b/test/normalization.test.js index cf9c95a..34f734d 100644 --- a/test/normalization.test.js +++ b/test/normalization.test.js @@ -3,7 +3,7 @@ import { deepStrictEqual } from 'node:assert/strict'; import { csvStreamEqual, csvStreamEqualWritable, createCSVMockStream } from './utils.js'; import { CSVReader } from '../src/core.js'; -import { toCamelCase, expandScientificNotation, fixExcelNumber, fixExcelBigInt, fixExcelDate, createCSVNormalizationStream, createCSVDenormalizationStream } from '../src/normalization.js'; +import { toCamelCase, expandScientificNotation, fixExcelNumber, fixExcelBigInt, fixExcelDate, CSVNormalizer, createCSVDenormalizationStream } from '../src/normalization.js'; suite('toCamelCase', { concurrency: true }, () => { test('does not modify strings already in camelCase', { concurrency: true }, () => { @@ -169,13 +169,13 @@ suite('fixExcelDate', { concurrency: true }, () => { }); }); -suite('createCSVNormalizationStream', { concurrency: true }, () => { +suite('CSVNormalizer', { concurrency: true }, () => { test('can remove extra columns', { concurrency: true }, async () => { await createCSVMockStream([ ['extra1', 'columnA', 'extra2', 'extra3', 'columnB'], ['123', 'a', '123', '123', 'b'] ]) - .pipeThrough(createCSVNormalizationStream([ + .pipeThrough(new CSVNormalizer([ { name: 'columnA', type: 'string' }, { name: 'columnB', type: 'string' } ])) @@ -190,7 +190,7 @@ suite('createCSVNormalizationStream', { concurrency: true }, () => { ['columnC', 'columnA', 'columnB'], ['c', 'a', 'b'] ]) - .pipeThrough(createCSVNormalizationStream([ + .pipeThrough(new CSVNormalizer([ { name: 'columnA', type: 'string' }, { name: 'columnB', type: 'string' }, { name: 'columnC', type: 'string' } @@ -206,7 +206,7 @@ suite('createCSVNormalizationStream', { concurrency: true }, () => { ['columnA', 'columnB'], ['a', 'b'] ]) - .pipeThrough(createCSVNormalizationStream([ + .pipeThrough(new CSVNormalizer([ { name: 'columnA', type: 'string', displayName: 'Column A' }, { name: 'columnB', type: 'string', displayName: 'Column B' } ])) @@ -225,7 +225,7 @@ suite('createCSVNormalizationStream', { concurrency: true }, () => { ['',''], ['',''] ]) - .pipeThrough(createCSVNormalizationStream([ + .pipeThrough(new CSVNormalizer([ { name: 'columnA', type: 'string' }, { name: 'columnB', type: 'string' } ])) @@ -240,7 +240,7 @@ suite('createCSVNormalizationStream', { concurrency: true }, () => { ['columnA', 'columnB'], ['a', 'b'] ]) - .pipeThrough(createCSVNormalizationStream([ + .pipeThrough(new CSVNormalizer([ { name: 'columnA', type: 'abc' }, { name: 'columnB', type: 'def' } ])) @@ -252,7 +252,7 @@ suite('createCSVNormalizationStream', { concurrency: true }, () => { }); }); -suite('createCSVNormalizationStream and createCSVDenormalizationStream end-to-end', { concurrency: true }, () => { +suite('CSVNormalizer and createCSVDenormalizationStream end-to-end', { concurrency: true }, () => { test('can normalize', { concurrency: true }, async () => { const headers = [ { name: 'stringCol', displayName: 'String Column', type: 'string', defaultValue: 'N/A' }, @@ -261,7 +261,7 @@ suite('createCSVNormalizationStream and createCSVDenormalizationStream end-to-en { name: 'dateCol', displayName: 'Date Column', type: 'date' } ] const stream = new CSVReader('./test/data/normalization.csv') - .pipeThrough(createCSVNormalizationStream(headers)) + .pipeThrough(new CSVNormalizer(headers)) .pipeThrough(createCSVDenormalizationStream()); await csvStreamEqual(stream, [ ['String Column', 'Number Column', 'BigInt Column', 'Date Column'],