From: Ardis Lu Date: Fri, 26 Sep 2025 06:45:57 +0000 (-0700) Subject: Remove empty rows before the header row X-Git-Tag: v0.1.7~11 X-Git-Url: https://git.ardis.lu/?a=commitdiff_plain;h=e0f3b849c1bc5b936dbf707bd288dc61019c6ba1;p=csvio Remove empty rows before the header row Empty rows before the header are sometimes used for formatting purposes, and they should be removed from the data. It is not possible to normalize an empty row before the headers are known, so empty rows that come before the header row are *always* removed, even if `passthroughEmptyRows` is `true`. --- diff --git a/README.md b/README.md index e9321e1..994e8b0 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ In addition to the core API, optional utilities are provided to address common C ### `CSVNormalizer` and `CSVDenormalizer` `CSVNormalizer` is a `TransformStream` that applies opinionated transformations to correct data mangling caused by spreadsheet programs and manual user manipulation: -- **Remove empty or extra data**: Spreadsheet programs may append empty rows or columns at the ends of a CSV file, or users may insert empty rows or columns for formatting purposes. These empty rows are removed. Additionally, columns which are not explicitly listed in the `CSVNormalizerHeader` array are also removed. +- **Remove empty or extra data**: Spreadsheet programs may append empty rows or columns at the ends of a CSV file, or users may insert empty rows or columns for formatting purposes. These empty rows are removed. Additionally, columns which are not explicitly listed in the `CSVNormalizerHeader` array are also removed. Note that empty rows that come *before* the header row are *always* removed, even if you have set `passthroughEmptyRows: true`. - **Fix date and number mangling**: Spreadsheet programs may [automatically cast data types in unexpected ways](https://www.theverge.com/2020/8/6/21355674/human-genes-rename-microsoft-excel-misreading-dates). `CSVNormalizer` will perform various transformations to reverse data type casting that was unlikely to be intentional. - **Flexible column naming**: Column names are normalized to camelCase to prevent issues with subtly inconsistent names (e.g., `Column A`, `column A`, and `ColumnA` are all considered to be the same name). - **Column ordering**: Columns may appear in any order on the input CSV. The columns in the output are re-ordered to match the order in the `CSVNormalizerHeader` array. @@ -176,11 +176,12 @@ const headers = [ ] const rows = ReadableStream.from((function* () { - yield ['numberCol', 'dateCol', '', '']; // Empty columns inserted - yield ['1E+18', '45292', '', '']; // Number mangled to scientific notation; date mangled to number - yield ['', '', '', '']; // Empty rows inserted - yield ['', '', '', '']; - yield ['', '', '', '']; + yield ['', '', '', '', '']; // Empty row inserted before the header row + yield ['', 'numberCol', 'dateCol', '', '']; // Empty columns inserted, including before the data + yield ['', '1E+18', '45292', '', '']; // Number mangled to scientific notation; date mangled to number + yield ['', '', '', '', '']; // Empty rows inserted + yield ['', '', '', '', '']; + yield ['', '', '', '', '']; })()) .pipeThrough(new CSVNormalizer(headers)) .pipeThrough(new CSVDenormalizer()); diff --git a/src/normalization.js b/src/normalization.js index eeb0c82..dc7462b 100644 --- a/src/normalization.js +++ b/src/normalization.js @@ -108,7 +108,7 @@ export function expandScientificNotation(str, truncate = false) { */ export class CSVNormalizer extends TransformStream { #columns = []; - #firstChunk = true; + #headersCaptured = false; #useLiteralNames; #passthroughEmptyRows; #typeCastOnly; @@ -145,9 +145,10 @@ export class CSVNormalizer extends TransformStream { /** @type {import('node:stream/web').TransformerTransformCallback,Array>>} */ #transform(chunk, controller) { - // Assume first row is headers and use it to prepare the columns object + // Assume first non-empty row is headers and use it to prepare the columns object // Note: the headers row is NOT forwarded downstream - if (this.#firstChunk) { + if (!this.#headersCaptured) { + if (chunk.every(v => v === '')) { return; } // All empty rows before the header row are removed, even if passthroughEmptyRows is true const normalizedRow = this.#useLiteralNames ? chunk : chunk.map(f => toCamelCase(f)); let i = 0; for (const header of normalizedRow) { @@ -160,7 +161,7 @@ export class CSVNormalizer extends TransformStream { i++; } this.#columns = this.#columns.filter(c => c.index !== null); // Drop columns provided in the headers input but not in the CSV - this.#firstChunk = false; + this.#headersCaptured = true; return; } diff --git a/test/normalization.test.js b/test/normalization.test.js index d4e15b4..c229655 100644 --- a/test/normalization.test.js +++ b/test/normalization.test.js @@ -361,6 +361,46 @@ suite('CSVNormalizer', { concurrency: true }, () => { ['', ''] ])); }); + test('can remove empty rows before the header row', { concurrency: true }, async () => { + await createCSVMockStream([ + ['', ''], + ['', ''], + ['columnA', 'columnB'], + ['', ''], + ['a', 'b'], + ['', ''], + ]) + .pipeThrough(new CSVNormalizer([ + { name: 'columnA', type: 'string' }, + { name: 'columnB', type: 'string' } + ])) + .pipeThrough(new CSVDenormalizer()) + .pipeTo(csvStreamEqualWritable([ + ['columnA', 'columnB'], + ['a', 'b'] + ])); + }); + test('passes through empty rows EXCEPT rows before the header when passthroughEmptyRows is true', { concurrency: true }, async () => { + await createCSVMockStream([ + ['', ''], + ['', ''], + ['columnA', 'columnB'], + ['', ''], + ['a', 'b'], + ['', ''] + ]) + .pipeThrough(new CSVNormalizer([ + { name: 'columnA', type: 'string' }, + { name: 'columnB', type: 'string' } + ], { passthroughEmptyRows: true })) + .pipeThrough(new CSVDenormalizer()) + .pipeTo(csvStreamEqualWritable([ + ['columnA', 'columnB'], + ['', ''], + ['a', 'b'], + ['', ''] + ])); + }); test('can remove empty columns', { concurrency: true }, async () => { await createCSVMockStream([ ['columnA', '', 'columnB', '', ''],