Pass through native type whenever possible

The stringification was legacy code from the original design of always JSON serializing each row between every stream. Since that serialization is no longer done, `CSVNormalizer` can just pass the native data type through. The downstream `CSVDenormalizer` should stringify all the data if the old behavior is desired.

diff --git a/src/normalization.js b/src/normalization.js
index dc7462b375a58ae275c54875dad48147b62d2e24..dbba4b34649f70c58c3bef4a80b91645efd7828f 100644 (file)
--- a/src/normalization.js
+++ b/src/normalization.js
@@ -173,8 +173,8 @@ export class CSVNormalizer extends TransformStream {
       switch (type) {
         case 'string': break;
         case 'number': value = this.#typeCastOnly ? Number(value) : CSVNormalizer.fixExcelNumber(value); break;
-        case 'bigint': value = this.#typeCastOnly ? BigInt(value).toString() : CSVNormalizer.fixExcelBigInt(value); break;
-        case 'date': value = this.#typeCastOnly ? new Date(value).toISOString() : CSVNormalizer.fixExcelDate(value); break;
+        case 'bigint': value = this.#typeCastOnly ? BigInt(value) : CSVNormalizer.fixExcelBigInt(value); break;
+        case 'date': value = this.#typeCastOnly ? new Date(value) : CSVNormalizer.fixExcelDate(value); break;
       }
       if (emptyField) {
         value = defaultValue;
@@ -232,10 +232,7 @@ export class CSVNormalizer extends TransformStream {
    * 
    * For example, converting BigInt to scientific notation.
    * @param {string} str A string that might be a BigInt.
-   * @returns {string} If a fix was possible, a string that may be passed to the `BigInt()` constructor without error.
-   * Otherwise, the original string is returned.
-   * 
-   * Note that **a string is always returned** to simplify downstream use because BigInt serialization is not possible with `JSON.stringify()`.
+   * @returns {bigint|string} If a fix was possible, a `bigint`. Otherwise, the original string is returned.
    */
   static fixExcelBigInt(str) {
     if (str === '') { return str; }
@@ -243,13 +240,13 @@ export class CSVNormalizer extends TransformStream {
 
     // Can parse normally without any fixes
     try {
-      return BigInt(original).toString();
+      return BigInt(original);
     } catch { }
 
     // Excel converted to scientific notation
     try {
       const expanded = expandScientificNotation(original, true);
-      return BigInt(expanded).toString();
+      return BigInt(expanded);
     } catch { }
 
     // Excel added extra characters from setting the format to "Accounting" or other custom formats
@@ -258,7 +255,7 @@ export class CSVNormalizer extends TransformStream {
       const decimalSeparator = getDecimalSeparator();
       const str = original.replaceAll(new RegExp(`[^0-9${decimalSeparator}]`, 'g'), '');
       if (str === '') { throw new Error(); }
-      return BigInt(str.split(decimalSeparator)[0]).toString();
+      return BigInt(str.split(decimalSeparator)[0]);
     } catch { }
 
     // Otherwise, pass through as string

diff --git a/test/normalization.test.js b/test/normalization.test.js
index c229655e1ba3627022aebe1ad774206f9e9c86ce..4f74bfcc5c95aee934911b19b483ee10d575c3b5 100644 (file)
--- a/test/normalization.test.js
+++ b/test/normalization.test.js
@@ -121,13 +121,13 @@ suite('CSVNormalizer.fixExcelNumber', { concurrency: true }, () => {
 
 suite('CSVNormalizer.fixExcelBigInt', { concurrency: true }, () => {
   const vectors = [
-    { name: 'parses BigInt', input: '123456789123456789123456789', output: '123456789123456789123456789' },
-    { name: 'fixes scientific notation mangling', input: '1E+18', output: '1000000000000000000' },
-    { name: 'fixes scientific notation mangling with decimal', input: '1.123456789123456789E18', output: '1123456789123456789' },
-    { name: 'fixes scientific notation mangling with decimal (truncated)', input: '1.1234567891234567891234E18', output: '1123456789123456789' },
-    { name: 'fixes accounting mangling integer', input: ' $123.00 ', output: '123' },
-    { name: 'fixes accounting mangling number (truncated)', input: ' $123.45 ', output: '123' },
-    { name: 'fixes accounting mangling BigInt with commas', input: '  $1,123,123,123,123,123,123.12  ', output: '1123123123123123123' },
+    { name: 'parses BigInt', input: '123456789123456789123456789', output: 123456789123456789123456789n },
+    { name: 'fixes scientific notation mangling', input: '1E+18', output: 1000000000000000000n },
+    { name: 'fixes scientific notation mangling with decimal', input: '1.123456789123456789E18', output: 1123456789123456789n },
+    { name: 'fixes scientific notation mangling with decimal (truncated)', input: '1.1234567891234567891234E18', output: 1123456789123456789n },
+    { name: 'fixes accounting mangling integer', input: ' $123.00 ', output: 123n },
+    { name: 'fixes accounting mangling number (truncated)', input: ' $123.45 ', output: 123n },
+    { name: 'fixes accounting mangling BigInt with commas', input: '  $1,123,123,123,123,123,123.12  ', output: 1123123123123123123n },
     { name: 'passes through non-BigInt values', input: 'abc', output: 'abc', consoleCounts: { warn: 1 } },
     { name: 'passes through blank value', input: '', output: '' },
   ];
@@ -501,7 +501,7 @@ suite('CSVNormalizer', { concurrency: true }, () => {
       .pipeThrough(new CSVDenormalizer())
       .pipeTo(csvStreamEqualWritable([
         ['columnA', 'columnB'],
-        ['123', '456']
+        [123n, 456n]
       ]));
   });
   test('can pass through date', { concurrency: true }, async () => {
@@ -588,9 +588,9 @@ suite('CSVNormalizer and CSVDenormalizer end-to-end', { concurrency: true }, ()
       .pipeThrough(new CSVDenormalizer())
       .pipeTo(csvStreamEqualWritable([
         ['String Column', 'Number Column', 'BigInt Column', 'Date Column'],
-        ['abc 👨‍👩‍👧‍👦', 123456789.123456, '1000000000000000000', '2024-01-01T00:00:00.000Z'],
-        [', " 🏴‍☠️', 1000, '-1234567890000000000000000', '2024-06-01T00:00:00.000Z'],
-        ['N/A', 123100, '1000000000000000000', '2024-12-31T08:00:00.000Z']
+        ['abc 👨‍👩‍👧‍👦', 123456789.123456, 1000000000000000000n, '2024-01-01T00:00:00.000Z'],
+        [', " 🏴‍☠️', 1000, -1234567890000000000000000n, '2024-06-01T00:00:00.000Z'],
+        ['N/A', 123100, 1000000000000000000n, '2024-12-31T08:00:00.000Z']
       ]));
   });
   test('can normalize with transformation', { concurrency: true }, async () => {