Add `useLiteralNames` option to disable column name normalization

This behavior is useful to preserve data when an input CSV has multiple columns with the same normalized name. By default, `CSVNormalizer` will remove these duplicate columns (the data from the last column is used).

Note that columns with the same *literal* name are still removed even when `useLiteralNames` is `true`. If you need to preserve columns with the same literal name, use a lower-level transformation (e.g., `CSVTransformer`).

diff --git a/src/normalization.js b/src/normalization.js
index bc43d0b9930a8b406d26db0f788cb2dd92f9c8cb..69d91baa4633b2382b0bb1ca93c32cafd8e54e7c 100644 (file)
--- a/src/normalization.js
+++ b/src/normalization.js
@@ -80,6 +80,9 @@ export function expandScientificNotation(str, truncate = false) {
 /**
  * Options to configure `CSVNormalizer`.
  * @typedef {Object} CSVNormalizerOptions
+ * @property {boolean} [useLiteralNames=false] Set to `true` to use the literal `name` value to match with literal input CSV header
+ * names. Otherwise, both `name` and the input CSV header name will be normalized (i.e., trimmed and converted to camelCase).
+ * The default value is `false`.
  * @property {boolean} [passthroughEmptyRows=false] Set to `true` to send empty rows (i.e., rows where all field values are
  * `''`) downstream. Otherwise, empty rows will be removed. The default value is `false`.
  * @property {boolean} [typeCastOnly=false] Set to `true` to NOT apply any error correction to 'number', `bigint`, or `date`
@@ -106,6 +109,7 @@ export function expandScientificNotation(str, truncate = false) {
 export class CSVNormalizer extends TransformStream {
   #columns = [];
   #firstChunk = true;
+  #useLiteralNames;
   #passthroughEmptyRows;
   #typeCastOnly;
 
@@ -119,6 +123,7 @@ export class CSVNormalizer extends TransformStream {
       transform: (chunk, controller) => this.#transform(chunk, controller)
     });
 
+    this.#useLiteralNames = options.useLiteralNames ?? false;
     this.#passthroughEmptyRows = options.passthroughEmptyRows ?? false;
     this.#typeCastOnly = options.typeCastOnly ?? false;
 
@@ -129,7 +134,7 @@ export class CSVNormalizer extends TransformStream {
         normalizedType = 'string';
       }
       this.#columns.push({
-        name: toCamelCase(name),
+        name: this.#useLiteralNames ? name : toCamelCase(name),
         type: normalizedType,
         displayName,
         defaultValue,
@@ -145,7 +150,7 @@ export class CSVNormalizer extends TransformStream {
     // Assume first row is headers and use it to prepare the columns object
     // Note: the headers row is NOT forwarded downstream
     if (this.#firstChunk) {
-      const normalizedRow = row.map(f => toCamelCase(f));
+      const normalizedRow = this.#useLiteralNames ? row : row.map(f => toCamelCase(f));
       let i = 0;
       for (const header of normalizedRow) {
         const col = this.#columns.find(c => c.name === header);

diff --git a/test/normalization.test.js b/test/normalization.test.js
index 5b02cedfa993eaf5b77a25a647f19646797c47bd..fb4805f1a583f8e7b4256e61a2d4404a0aae491c 100644 (file)
--- a/test/normalization.test.js
+++ b/test/normalization.test.js
@@ -239,6 +239,50 @@ suite('CSVNormalizer', { concurrency: true }, () => {
         ['a', 'b']
       ]));
   });
+  test('removes columns that have the same normalized name (last column prevails)', { concurrency: true }, async () => {
+    await createCSVMockStream([
+      ['columnA', 'column A', 'column-A'],
+      ['a1', 'a2', 'a3']
+    ])
+      .pipeThrough(new CSVNormalizer([
+        { name: 'columnA' }
+      ]))
+      .pipeThrough(new CSVDenormalizer())
+      .pipeTo(csvStreamEqualWritable([
+        ['columnA'],
+        ['a3']
+      ]));
+  });
+  test('can match by literal column names', { concurrency: true }, async () => {
+    await createCSVMockStream([
+      ['columnA', 'column A', 'column-A'],
+      ['a1', 'a2', 'a3']
+    ])
+      .pipeThrough(new CSVNormalizer([
+        { name: 'columnA' },
+        { name: 'column A' },
+        { name: 'column-A' }
+      ], { useLiteralNames: true }))
+      .pipeThrough(new CSVDenormalizer())
+      .pipeTo(csvStreamEqualWritable([
+        ['columnA', 'column A', 'column-A'],
+        ['a1', 'a2', 'a3']
+      ]));
+  });
+  test('removes columns that have the same literal name (last column prevails)', { concurrency: true }, async () => {
+    await createCSVMockStream([
+      ['columnA', 'columnA', 'columnA'],
+      ['a1', 'a2', 'a3']
+    ])
+      .pipeThrough(new CSVNormalizer([
+        { name: 'columnA' }
+      ], { useLiteralNames: true }))
+      .pipeThrough(new CSVDenormalizer())
+      .pipeTo(csvStreamEqualWritable([
+        ['columnA'],
+        ['a3']
+      ]));
+  });
   test('can remove empty rows', { concurrency: true }, async () => {
     await createCSVMockStream([
       ['columnA', 'columnB'],