Remove empty rows before the header row

Empty rows before the header are sometimes used for formatting purposes, and they should be removed from the data.

It is not possible to normalize an empty row before the headers are known, so empty rows that come before the header row are *always* removed, even if `passthroughEmptyRows` is `true`.

diff --git a/README.md b/README.md
index e9321e108337922d0d035bfbe25eef9dc89193b8..994e8b0e30bf30cb2e0581e718e6bc27cd66d151 100644 (file)
--- a/README.md
+++ b/README.md
@@ -159,7 +159,7 @@ In addition to the core API, optional utilities are provided to address common C
 ### `CSVNormalizer` and `CSVDenormalizer`
 
 `CSVNormalizer` is a `TransformStream` that applies opinionated transformations to correct data mangling caused by spreadsheet programs and manual user manipulation:
-- **Remove empty or extra data**: Spreadsheet programs may append empty rows or columns at the ends of a CSV file, or users may insert empty rows or columns for formatting purposes. These empty rows are removed. Additionally, columns which are not explicitly listed in the `CSVNormalizerHeader` array are also removed.
+- **Remove empty or extra data**: Spreadsheet programs may append empty rows or columns at the ends of a CSV file, or users may insert empty rows or columns for formatting purposes. These empty rows are removed. Additionally, columns which are not explicitly listed in the `CSVNormalizerHeader` array are also removed. Note that empty rows that come *before* the header row are *always* removed, even if you have set `passthroughEmptyRows: true`.
 - **Fix date and number mangling**: Spreadsheet programs may [automatically cast data types in unexpected ways](https://www.theverge.com/2020/8/6/21355674/human-genes-rename-microsoft-excel-misreading-dates). `CSVNormalizer` will perform various transformations to reverse data type casting that was unlikely to be intentional.
 - **Flexible column naming**: Column names are normalized to camelCase to prevent issues with subtly inconsistent names (e.g., `Column A`, `column A`, and `ColumnA` are all considered to be the same name).
 - **Column ordering**: Columns may appear in any order on the input CSV. The columns in the output are re-ordered to match the order in the `CSVNormalizerHeader` array.
@@ -176,11 +176,12 @@ const headers = [
 ]
 
 const rows = ReadableStream.from((function* () {
-  yield ['numberCol', 'dateCol', '', '']; // Empty columns inserted
-  yield ['1E+18', '45292', '', '']; // Number mangled to scientific notation; date mangled to number
-  yield ['', '', '', '']; // Empty rows inserted
-  yield ['', '', '', ''];
-  yield ['', '', '', ''];
+  yield ['', '', '', '', '']; // Empty row inserted before the header row
+  yield ['', 'numberCol', 'dateCol', '', '']; // Empty columns inserted, including before the data
+  yield ['', '1E+18', '45292', '', '']; // Number mangled to scientific notation; date mangled to number
+  yield ['', '', '', '', '']; // Empty rows inserted
+  yield ['', '', '', '', ''];
+  yield ['', '', '', '', ''];
 })())
   .pipeThrough(new CSVNormalizer(headers))
   .pipeThrough(new CSVDenormalizer());

diff --git a/src/normalization.js b/src/normalization.js
index eeb0c823591978be2dda56437389215ce03fc6f2..dc7462b375a58ae275c54875dad48147b62d2e24 100644 (file)
--- a/src/normalization.js
+++ b/src/normalization.js
@@ -108,7 +108,7 @@ export function expandScientificNotation(str, truncate = false) {
  */
 export class CSVNormalizer extends TransformStream {
   #columns = [];
-  #firstChunk = true;
+  #headersCaptured = false;
   #useLiteralNames;
   #passthroughEmptyRows;
   #typeCastOnly;
@@ -145,9 +145,10 @@ export class CSVNormalizer extends TransformStream {
 
   /** @type {import('node:stream/web').TransformerTransformCallback<Array<string>,Array<Required<CSVNormalizerField>>>} */
   #transform(chunk, controller) {
-    // Assume first row is headers and use it to prepare the columns object
+    // Assume first non-empty row is headers and use it to prepare the columns object
     // Note: the headers row is NOT forwarded downstream
-    if (this.#firstChunk) {
+    if (!this.#headersCaptured) {
+      if (chunk.every(v => v === '')) { return; } // All empty rows before the header row are removed, even if passthroughEmptyRows is true
       const normalizedRow = this.#useLiteralNames ? chunk : chunk.map(f => toCamelCase(f));
       let i = 0;
       for (const header of normalizedRow) {
@@ -160,7 +161,7 @@ export class CSVNormalizer extends TransformStream {
         i++;
       }
       this.#columns = this.#columns.filter(c => c.index !== null); // Drop columns provided in the headers input but not in the CSV
-      this.#firstChunk = false;
+      this.#headersCaptured = true;
       return;
     }
 

diff --git a/test/normalization.test.js b/test/normalization.test.js
index d4e15b44bf0f68d4f0b90cfe5d3fee61adf91795..c229655e1ba3627022aebe1ad774206f9e9c86ce 100644 (file)
--- a/test/normalization.test.js
+++ b/test/normalization.test.js
@@ -361,6 +361,46 @@ suite('CSVNormalizer', { concurrency: true }, () => {
         ['', '']
       ]));
   });
+  test('can remove empty rows before the header row', { concurrency: true }, async () => {
+    await createCSVMockStream([
+      ['', ''],
+      ['', ''],
+      ['columnA', 'columnB'],
+      ['', ''],
+      ['a', 'b'],
+      ['', ''],
+    ])
+      .pipeThrough(new CSVNormalizer([
+        { name: 'columnA', type: 'string' },
+        { name: 'columnB', type: 'string' }
+      ]))
+      .pipeThrough(new CSVDenormalizer())
+      .pipeTo(csvStreamEqualWritable([
+        ['columnA', 'columnB'],
+        ['a', 'b']
+      ]));
+  });
+  test('passes through empty rows EXCEPT rows before the header when passthroughEmptyRows is true', { concurrency: true }, async () => {
+    await createCSVMockStream([
+      ['', ''],
+      ['', ''],
+      ['columnA', 'columnB'],
+      ['', ''],
+      ['a', 'b'],
+      ['', '']
+    ])
+      .pipeThrough(new CSVNormalizer([
+        { name: 'columnA', type: 'string' },
+        { name: 'columnB', type: 'string' }
+      ], { passthroughEmptyRows: true }))
+      .pipeThrough(new CSVDenormalizer())
+      .pipeTo(csvStreamEqualWritable([
+        ['columnA', 'columnB'],
+        ['', ''],
+        ['a', 'b'],
+        ['', '']
+      ]));
+  });
   test('can remove empty columns', { concurrency: true }, async () => {
     await createCSVMockStream([
       ['columnA', '', 'columnB', '', ''],