From: Ardis Lu Date: Sun, 19 Jan 2025 08:16:39 +0000 (-0800) Subject: Expose fix functions as static methods of CSVNormalizer X-Git-Tag: v0.1.0~109 X-Git-Url: https://git.ardis.lu/?a=commitdiff_plain;h=8389691be402b3b62187eda2023d96567f3cd8c9;p=csvio Expose fix functions as static methods of CSVNormalizer These functions may be useful to create custom normalizers. --- diff --git a/src/normalization.js b/src/normalization.js index 5824d8d..d05377c 100644 --- a/src/normalization.js +++ b/src/normalization.js @@ -49,119 +49,6 @@ export function expandScientificNotation(str, truncate = false) { return expanded; } -/** - * Fix number mangling by Excel. - * - * For example, converting numbers to dates or adding extra characters (e.g., whitespace, thousands separators, or currency symbols). - * @param {string} str A string that might be a number. - * @returns {number|string} If a fix was possible, a number. Otherwise, the original string is returned. - */ -export function fixExcelNumber(str) { - if (str === '') { return str; } - const original = str; - - // Can parse normally without any fixes - const num1 = Number(original); - if (!Number.isNaN(num1)) { - return num1; - } - - // Excel converted the number to a date - const date = new Date(original); - if (date.toString() !== 'Invalid Date') { - // Excel incorrectly thinks 1900 was a leap year, need to offset 1 day before 3/1/1900 - const leapYearOffset = (original === '2/29/1900' || date.valueOf() < -2203862400000) ? -1 : 0; - // Convert JavaScript date to Excel serialized date: - // Days since Unix epoch - Timezone offset in days + Days between Jan 1, 1900 and Unix epoch start - return (date.valueOf() / 1000 / 60 / 60 / 24) - (date.getTimezoneOffset() / 60 / 24) + 25569 + leapYearOffset; - } - - // Excel added extra characters from setting the format to "Accounting" or other custom formats - // Delete everything that's not a number or the decimal separator - const replaced = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), ''); - const num2 = Number(replaced); - if (!Number.isNaN(num2) && replaced !== '') { - return num2; - } - - // Otherwise, pass through as string - console.warn(`Could not fix number: "${original}". Passing through as a string.`); - return original; -} - -/** - * Fix BigInt mangling by Excel. - * - * For example, converting BigInt to scientific notation. - * @param {string} str A string that might be a BigInt. - * @returns {string} If a fix was possible, a string that may be passed to the `BigInt()` constructor without error. - * Otherwise, the original string is returned. - * - * Note that **a string is always returned** to simplify downstream use because BigInt serialization is not possible with `JSON.stringify()`. - */ -export function fixExcelBigInt(str) { - if (str === '') { return str; } - const original = str; - - // Can parse normally without any fixes - try { - return BigInt(original).toString(); - } catch { } - - // Excel converted to scientific notation - try { - const expanded = expandScientificNotation(original, true); - if (expanded === null) { throw new Error(); } - return BigInt(expanded).toString(); - } catch { } - - // Excel added extra characters from setting the format to "Accounting" or other custom formats - // Delete everything that's not a number or the decimal separator - try { - const str = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), ''); - if (str === '') { throw new Error(); } - return BigInt(str.split(DECIMAL_SEPARATOR)[0]).toString(); - } catch { } - - // Otherwise, pass through as string - console.warn(`Could not fix bigint: "${original}". Passing through as a string.`); - return original; -} - -/** - * Fix date mangling by Excel. - * - * For example, converting date to number. - * @param {string} str A string that might be a date. - * @returns {Date|string} If a fix was possible, a date. Otherwise, the original string is returned. - */ -export function fixExcelDate(str) { - if (str === '') { return str; } - const original = str; - - // Excel converted date to a number - // This check MUST come before the new Date() check because numbers can be parsed as dates. - const num = Number(original); - if (!Number.isNaN(num)) { - const day = Math.trunc(num); // Days elapsed since January 1, 1900, including non-existent February 29, 1900 - const time = (num % 1) * 86400000; // Time of day as decimal, so 0 = 12 AM and 0.999... = 11:59:59.999... PM - if (num < 61) { - return new Date(Date.UTC(0, 0, day, 0, 0, 0, time)); - } - return new Date(Date.UTC(0, 0, day - 1, 0, 0, 0, time)); // Offset for non-existent February 29, 1900 - } - - // Can parse normally without any fixes - const date = new Date(original); - if (date.toString() !== 'Invalid Date') { - return date; - } - - // Otherwise, pass through as string - console.warn(`Could not fix date: "${original}". Passing through as a string.`); - return original; -} - /** * Metadata about CSV columns used to normalize raw CSV data. * @typedef {Object} CSVNormalizerHeader @@ -270,9 +157,9 @@ export class CSVNormalizer extends TransformStream { const emptyField = value === '' ? true : false; switch (type) { case 'string': break; - case 'number': value = passthroughNumber ? value : fixExcelNumber(value); break; - case 'bigint': value = passthroughBigInt ? value : fixExcelBigInt(value); break; - case 'date': value = passthroughDate ? value : fixExcelDate(value); break; + case 'number': value = passthroughNumber ? value : CSVNormalizer.fixExcelNumber(value); break; + case 'bigint': value = passthroughBigInt ? value : CSVNormalizer.fixExcelBigInt(value); break; + case 'date': value = passthroughDate ? value : CSVNormalizer.fixExcelDate(value); break; } if (emptyField && defaultValue !== null) { value = defaultValue; @@ -284,6 +171,119 @@ export class CSVNormalizer extends TransformStream { controller.enqueue(JSON.stringify(out)); } + + /** + * Fix number mangling by Excel. + * + * For example, converting numbers to dates or adding extra characters (e.g., whitespace, thousands separators, or currency symbols). + * @param {string} str A string that might be a number. + * @returns {number|string} If a fix was possible, a number. Otherwise, the original string is returned. + */ + static fixExcelNumber(str) { + if (str === '') { return str; } + const original = str; + + // Can parse normally without any fixes + const num1 = Number(original); + if (!Number.isNaN(num1)) { + return num1; + } + + // Excel converted the number to a date + const date = new Date(original); + if (date.toString() !== 'Invalid Date') { + // Excel incorrectly thinks 1900 was a leap year, need to offset 1 day before 3/1/1900 + const leapYearOffset = (original === '2/29/1900' || date.valueOf() < -2203862400000) ? -1 : 0; + // Convert JavaScript date to Excel serialized date: + // Days since Unix epoch - Timezone offset in days + Days between Jan 1, 1900 and Unix epoch start + return (date.valueOf() / 1000 / 60 / 60 / 24) - (date.getTimezoneOffset() / 60 / 24) + 25569 + leapYearOffset; + } + + // Excel added extra characters from setting the format to "Accounting" or other custom formats + // Delete everything that's not a number or the decimal separator + const replaced = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), ''); + const num2 = Number(replaced); + if (!Number.isNaN(num2) && replaced !== '') { + return num2; + } + + // Otherwise, pass through as string + console.warn(`Could not fix number: "${original}". Passing through as a string.`); + return original; + } + + /** + * Fix BigInt mangling by Excel. + * + * For example, converting BigInt to scientific notation. + * @param {string} str A string that might be a BigInt. + * @returns {string} If a fix was possible, a string that may be passed to the `BigInt()` constructor without error. + * Otherwise, the original string is returned. + * + * Note that **a string is always returned** to simplify downstream use because BigInt serialization is not possible with `JSON.stringify()`. + */ + static fixExcelBigInt(str) { + if (str === '') { return str; } + const original = str; + + // Can parse normally without any fixes + try { + return BigInt(original).toString(); + } catch { } + + // Excel converted to scientific notation + try { + const expanded = expandScientificNotation(original, true); + if (expanded === null) { throw new Error(); } + return BigInt(expanded).toString(); + } catch { } + + // Excel added extra characters from setting the format to "Accounting" or other custom formats + // Delete everything that's not a number or the decimal separator + try { + const str = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), ''); + if (str === '') { throw new Error(); } + return BigInt(str.split(DECIMAL_SEPARATOR)[0]).toString(); + } catch { } + + // Otherwise, pass through as string + console.warn(`Could not fix bigint: "${original}". Passing through as a string.`); + return original; + } + + /** + * Fix date mangling by Excel. + * + * For example, converting date to number. + * @param {string} str A string that might be a date. + * @returns {Date|string} If a fix was possible, a date. Otherwise, the original string is returned. + */ + static fixExcelDate(str) { + if (str === '') { return str; } + const original = str; + + // Excel converted date to a number + // This check MUST come before the new Date() check because numbers can be parsed as dates. + const num = Number(original); + if (!Number.isNaN(num)) { + const day = Math.trunc(num); // Days elapsed since January 1, 1900, including non-existent February 29, 1900 + const time = (num % 1) * 86400000; // Time of day as decimal, so 0 = 12 AM and 0.999... = 11:59:59.999... PM + if (num < 61) { + return new Date(Date.UTC(0, 0, day, 0, 0, 0, time)); + } + return new Date(Date.UTC(0, 0, day - 1, 0, 0, 0, time)); // Offset for non-existent February 29, 1900 + } + + // Can parse normally without any fixes + const date = new Date(original); + if (date.toString() !== 'Invalid Date') { + return date; + } + + // Otherwise, pass through as string + console.warn(`Could not fix date: "${original}". Passing through as a string.`); + return original; + } } /** diff --git a/test/normalization.test.js b/test/normalization.test.js index dba486a..5fff999 100644 --- a/test/normalization.test.js +++ b/test/normalization.test.js @@ -3,7 +3,7 @@ import { deepStrictEqual } from 'node:assert/strict'; import { csvStreamEqual, csvStreamEqualWritable, createCSVMockStream } from './utils.js'; import { CSVReader } from '../src/core.js'; -import { toCamelCase, expandScientificNotation, fixExcelNumber, fixExcelBigInt, fixExcelDate, CSVNormalizer, CSVDenormalizer } from '../src/normalization.js'; +import { toCamelCase, expandScientificNotation, CSVNormalizer, CSVDenormalizer } from '../src/normalization.js'; suite('toCamelCase', { concurrency: true }, () => { test('does not modify strings already in camelCase', { concurrency: true }, () => { @@ -86,86 +86,86 @@ suite('expandScientificNotation', { concurrency: true }, () => { }); }); -suite('fixExcelNumber', { concurrency: true }, () => { +suite('CSVNormalizer.fixExcelNumber', { concurrency: true }, () => { test('parses numbers', { concurrency: true }, () => { - deepStrictEqual(fixExcelNumber('123'), 123); - deepStrictEqual(fixExcelNumber('123.456'), 123.456); + deepStrictEqual(CSVNormalizer.fixExcelNumber('123'), 123); + deepStrictEqual(CSVNormalizer.fixExcelNumber('123.456'), 123.456); }); test('fixes numbers mangled to dates', { concurrency: true }, () => { - deepStrictEqual(fixExcelNumber('1/1/1900'), 1); - deepStrictEqual(fixExcelNumber('1/2/1900'), 2); - deepStrictEqual(fixExcelNumber('2/28/1900'), 59); - deepStrictEqual(fixExcelNumber('2/29/1900'), 60); - deepStrictEqual(fixExcelNumber('3/1/1900'), 61); - deepStrictEqual(fixExcelNumber('12/31/1969'), 25568); - deepStrictEqual(fixExcelNumber('1/1/1970'), 25569); - deepStrictEqual(fixExcelNumber('1/2/1970'), 25570); - deepStrictEqual(fixExcelNumber('12/31/2023'), 45291); - deepStrictEqual(fixExcelNumber('1/1/2024'), 45292); + deepStrictEqual(CSVNormalizer.fixExcelNumber('1/1/1900'), 1); + deepStrictEqual(CSVNormalizer.fixExcelNumber('1/2/1900'), 2); + deepStrictEqual(CSVNormalizer.fixExcelNumber('2/28/1900'), 59); + deepStrictEqual(CSVNormalizer.fixExcelNumber('2/29/1900'), 60); + deepStrictEqual(CSVNormalizer.fixExcelNumber('3/1/1900'), 61); + deepStrictEqual(CSVNormalizer.fixExcelNumber('12/31/1969'), 25568); + deepStrictEqual(CSVNormalizer.fixExcelNumber('1/1/1970'), 25569); + deepStrictEqual(CSVNormalizer.fixExcelNumber('1/2/1970'), 25570); + deepStrictEqual(CSVNormalizer.fixExcelNumber('12/31/2023'), 45291); + deepStrictEqual(CSVNormalizer.fixExcelNumber('1/1/2024'), 45292); }); test('fixes numbers mangled to accounting format', { concurrency: true }, () => { - deepStrictEqual(fixExcelNumber(' $123.00 '), 123); - deepStrictEqual(fixExcelNumber(' $123.45 '), 123.45); - deepStrictEqual(fixExcelNumber(' $10,000,000.00 '), 10000000); + deepStrictEqual(CSVNormalizer.fixExcelNumber(' $123.00 '), 123); + deepStrictEqual(CSVNormalizer.fixExcelNumber(' $123.45 '), 123.45); + deepStrictEqual(CSVNormalizer.fixExcelNumber(' $10,000,000.00 '), 10000000); }); test('passes through non-number values', { concurrency: true }, () => { - deepStrictEqual(fixExcelNumber('abc'), 'abc'); + deepStrictEqual(CSVNormalizer.fixExcelNumber('abc'), 'abc'); }); test('passes through blank value', { concurrency: true }, () => { - deepStrictEqual(fixExcelNumber(''), ''); + deepStrictEqual(CSVNormalizer.fixExcelNumber(''), ''); }); }); -suite('fixExcelBigInt', { concurrency: true }, () => { +suite('CSVNormalizer.fixExcelBigInt', { concurrency: true }, () => { test('parses BigInt', { concurrency: true }, () => { - deepStrictEqual(fixExcelBigInt('123'), '123'); - deepStrictEqual(fixExcelBigInt('123456'), '123456'); - deepStrictEqual(fixExcelBigInt('123456789123456789123456789'), '123456789123456789123456789'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt('123'), '123'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt('123456'), '123456'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt('123456789123456789123456789'), '123456789123456789123456789'); }); test('fixes BigInt mangled to scientific notation', { concurrency: true }, () => { - deepStrictEqual(fixExcelBigInt('1E+18'), '1000000000000000000'); - deepStrictEqual(fixExcelBigInt('1.123456789123456789E18'), '1123456789123456789'); - deepStrictEqual(fixExcelBigInt('1.1234567891234567891234E18'), '1123456789123456789'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt('1E+18'), '1000000000000000000'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt('1.123456789123456789E18'), '1123456789123456789'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt('1.1234567891234567891234E18'), '1123456789123456789'); }); test('fixes BigInt mangled to accounting format', { concurrency: true }, () => { - deepStrictEqual(fixExcelBigInt(' $123.00 '), '123'); - deepStrictEqual(fixExcelBigInt(' $123.45 '), '123'); - deepStrictEqual(fixExcelBigInt(' $1,123,123,123,123,123,123.12 '), '1123123123123123123'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt(' $123.00 '), '123'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt(' $123.45 '), '123'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt(' $1,123,123,123,123,123,123.12 '), '1123123123123123123'); }); test('passes through non-BigInt values', { concurrency: true }, () => { - deepStrictEqual(fixExcelBigInt('abc'), 'abc'); + deepStrictEqual(CSVNormalizer.fixExcelBigInt('abc'), 'abc'); }); test('passes through blank value', { concurrency: true }, () => { - deepStrictEqual(fixExcelBigInt(''), ''); + deepStrictEqual(CSVNormalizer.fixExcelBigInt(''), ''); }); }); -suite('fixExcelDate', { concurrency: true }, () => { +suite('CSVNormalizer.fixExcelDate', { concurrency: true }, () => { test('parses dates', { concurrency: true }, () => { - deepStrictEqual(fixExcelDate('12/31/24').valueOf(), new Date('12/31/24').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('12/31/24').valueOf(), new Date('12/31/24').valueOf()); }); test('fixes dates mangled to numbers', { concurrency: true }, () => { - deepStrictEqual(fixExcelDate('1').valueOf(), new Date('1900-01-01T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('2').valueOf(), new Date('1900-01-02T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('59').valueOf(), new Date('1900-02-28T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('60').valueOf(), new Date('1900-02-29T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('61').valueOf(), new Date('1900-03-01T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('25568').valueOf(), new Date('1969-12-31T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('25569').valueOf(), new Date('1970-01-01T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('25570').valueOf(), new Date('1970-01-02T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('45291').valueOf(), new Date('2023-12-31T00:00:00.000Z').valueOf()); - deepStrictEqual(fixExcelDate('45292').valueOf(), new Date('2024-01-01T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('1').valueOf(), new Date('1900-01-01T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('2').valueOf(), new Date('1900-01-02T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('59').valueOf(), new Date('1900-02-28T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('60').valueOf(), new Date('1900-02-29T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('61').valueOf(), new Date('1900-03-01T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('25568').valueOf(), new Date('1969-12-31T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('25569').valueOf(), new Date('1970-01-01T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('25570').valueOf(), new Date('1970-01-02T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('45291').valueOf(), new Date('2023-12-31T00:00:00.000Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('45292').valueOf(), new Date('2024-01-01T00:00:00.000Z').valueOf()); }); test('fixes date times mangled to numbers', { concurrency: true }, () => { - deepStrictEqual(fixExcelDate('43915.31372').valueOf(), new Date('2020-03-25T07:31:45.407Z').valueOf()); - deepStrictEqual(fixExcelDate('44067.63').valueOf(), new Date('2020-08-24T15:07:11.999Z').valueOf()); - deepStrictEqual(fixExcelDate('44309.63502').valueOf(), new Date('2021-04-23T15:14:25.728Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('43915.31372').valueOf(), new Date('2020-03-25T07:31:45.407Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('44067.63').valueOf(), new Date('2020-08-24T15:07:11.999Z').valueOf()); + deepStrictEqual(CSVNormalizer.fixExcelDate('44309.63502').valueOf(), new Date('2021-04-23T15:14:25.728Z').valueOf()); }); test('passes through non-date values', { concurrency: true }, () => { - deepStrictEqual(fixExcelDate('abc'), 'abc'); + deepStrictEqual(CSVNormalizer.fixExcelDate('abc'), 'abc'); }); test('passes through blank value', { concurrency: true }, () => { - deepStrictEqual(fixExcelDate(''), ''); + deepStrictEqual(CSVNormalizer.fixExcelDate(''), ''); }); });