From 0d1e14d078acdf9d5f154738c6d240b705d24000 Mon Sep 17 00:00:00 2001 From: Ardis Lu Date: Fri, 26 Sep 2025 00:00:09 -0700 Subject: [PATCH] Pass through native type whenever possible The stringification was legacy code from the original design of always JSON serializing each row between every stream. Since that serialization is no longer done, `CSVNormalizer` can just pass the native data type through. The downstream `CSVDenormalizer` should stringify all the data if the old behavior is desired. --- src/normalization.js | 15 ++++++--------- test/normalization.test.js | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/normalization.js b/src/normalization.js index dc7462b..dbba4b3 100644 --- a/src/normalization.js +++ b/src/normalization.js @@ -173,8 +173,8 @@ export class CSVNormalizer extends TransformStream { switch (type) { case 'string': break; case 'number': value = this.#typeCastOnly ? Number(value) : CSVNormalizer.fixExcelNumber(value); break; - case 'bigint': value = this.#typeCastOnly ? BigInt(value).toString() : CSVNormalizer.fixExcelBigInt(value); break; - case 'date': value = this.#typeCastOnly ? new Date(value).toISOString() : CSVNormalizer.fixExcelDate(value); break; + case 'bigint': value = this.#typeCastOnly ? BigInt(value) : CSVNormalizer.fixExcelBigInt(value); break; + case 'date': value = this.#typeCastOnly ? new Date(value) : CSVNormalizer.fixExcelDate(value); break; } if (emptyField) { value = defaultValue; @@ -232,10 +232,7 @@ export class CSVNormalizer extends TransformStream { * * For example, converting BigInt to scientific notation. * @param {string} str A string that might be a BigInt. - * @returns {string} If a fix was possible, a string that may be passed to the `BigInt()` constructor without error. - * Otherwise, the original string is returned. - * - * Note that **a string is always returned** to simplify downstream use because BigInt serialization is not possible with `JSON.stringify()`. + * @returns {bigint|string} If a fix was possible, a `bigint`. Otherwise, the original string is returned. */ static fixExcelBigInt(str) { if (str === '') { return str; } @@ -243,13 +240,13 @@ export class CSVNormalizer extends TransformStream { // Can parse normally without any fixes try { - return BigInt(original).toString(); + return BigInt(original); } catch { } // Excel converted to scientific notation try { const expanded = expandScientificNotation(original, true); - return BigInt(expanded).toString(); + return BigInt(expanded); } catch { } // Excel added extra characters from setting the format to "Accounting" or other custom formats @@ -258,7 +255,7 @@ export class CSVNormalizer extends TransformStream { const decimalSeparator = getDecimalSeparator(); const str = original.replaceAll(new RegExp(`[^0-9${decimalSeparator}]`, 'g'), ''); if (str === '') { throw new Error(); } - return BigInt(str.split(decimalSeparator)[0]).toString(); + return BigInt(str.split(decimalSeparator)[0]); } catch { } // Otherwise, pass through as string diff --git a/test/normalization.test.js b/test/normalization.test.js index c229655..4f74bfc 100644 --- a/test/normalization.test.js +++ b/test/normalization.test.js @@ -121,13 +121,13 @@ suite('CSVNormalizer.fixExcelNumber', { concurrency: true }, () => { suite('CSVNormalizer.fixExcelBigInt', { concurrency: true }, () => { const vectors = [ - { name: 'parses BigInt', input: '123456789123456789123456789', output: '123456789123456789123456789' }, - { name: 'fixes scientific notation mangling', input: '1E+18', output: '1000000000000000000' }, - { name: 'fixes scientific notation mangling with decimal', input: '1.123456789123456789E18', output: '1123456789123456789' }, - { name: 'fixes scientific notation mangling with decimal (truncated)', input: '1.1234567891234567891234E18', output: '1123456789123456789' }, - { name: 'fixes accounting mangling integer', input: ' $123.00 ', output: '123' }, - { name: 'fixes accounting mangling number (truncated)', input: ' $123.45 ', output: '123' }, - { name: 'fixes accounting mangling BigInt with commas', input: ' $1,123,123,123,123,123,123.12 ', output: '1123123123123123123' }, + { name: 'parses BigInt', input: '123456789123456789123456789', output: 123456789123456789123456789n }, + { name: 'fixes scientific notation mangling', input: '1E+18', output: 1000000000000000000n }, + { name: 'fixes scientific notation mangling with decimal', input: '1.123456789123456789E18', output: 1123456789123456789n }, + { name: 'fixes scientific notation mangling with decimal (truncated)', input: '1.1234567891234567891234E18', output: 1123456789123456789n }, + { name: 'fixes accounting mangling integer', input: ' $123.00 ', output: 123n }, + { name: 'fixes accounting mangling number (truncated)', input: ' $123.45 ', output: 123n }, + { name: 'fixes accounting mangling BigInt with commas', input: ' $1,123,123,123,123,123,123.12 ', output: 1123123123123123123n }, { name: 'passes through non-BigInt values', input: 'abc', output: 'abc', consoleCounts: { warn: 1 } }, { name: 'passes through blank value', input: '', output: '' }, ]; @@ -501,7 +501,7 @@ suite('CSVNormalizer', { concurrency: true }, () => { .pipeThrough(new CSVDenormalizer()) .pipeTo(csvStreamEqualWritable([ ['columnA', 'columnB'], - ['123', '456'] + [123n, 456n] ])); }); test('can pass through date', { concurrency: true }, async () => { @@ -588,9 +588,9 @@ suite('CSVNormalizer and CSVDenormalizer end-to-end', { concurrency: true }, () .pipeThrough(new CSVDenormalizer()) .pipeTo(csvStreamEqualWritable([ ['String Column', 'Number Column', 'BigInt Column', 'Date Column'], - ['abc 👨‍👩‍👧‍👦', 123456789.123456, '1000000000000000000', '2024-01-01T00:00:00.000Z'], - [', " 🏴‍☠️', 1000, '-1234567890000000000000000', '2024-06-01T00:00:00.000Z'], - ['N/A', 123100, '1000000000000000000', '2024-12-31T08:00:00.000Z'] + ['abc 👨‍👩‍👧‍👦', 123456789.123456, 1000000000000000000n, '2024-01-01T00:00:00.000Z'], + [', " 🏴‍☠️', 1000, -1234567890000000000000000n, '2024-06-01T00:00:00.000Z'], + ['N/A', 123100, 1000000000000000000n, '2024-12-31T08:00:00.000Z'] ])); }); test('can normalize with transformation', { concurrency: true }, async () => { -- 2.47.3