return expanded;
}
-/**
- * Fix number mangling by Excel.
- *
- * For example, converting numbers to dates or adding extra characters (e.g., whitespace, thousands separators, or currency symbols).
- * @param {string} str A string that might be a number.
- * @returns {number|string} If a fix was possible, a number. Otherwise, the original string is returned.
- */
-export function fixExcelNumber(str) {
- if (str === '') { return str; }
- const original = str;
-
- // Can parse normally without any fixes
- const num1 = Number(original);
- if (!Number.isNaN(num1)) {
- return num1;
- }
-
- // Excel converted the number to a date
- const date = new Date(original);
- if (date.toString() !== 'Invalid Date') {
- // Excel incorrectly thinks 1900 was a leap year, need to offset 1 day before 3/1/1900
- const leapYearOffset = (original === '2/29/1900' || date.valueOf() < -2203862400000) ? -1 : 0;
- // Convert JavaScript date to Excel serialized date:
- // Days since Unix epoch - Timezone offset in days + Days between Jan 1, 1900 and Unix epoch start
- return (date.valueOf() / 1000 / 60 / 60 / 24) - (date.getTimezoneOffset() / 60 / 24) + 25569 + leapYearOffset;
- }
-
- // Excel added extra characters from setting the format to "Accounting" or other custom formats
- // Delete everything that's not a number or the decimal separator
- const replaced = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), '');
- const num2 = Number(replaced);
- if (!Number.isNaN(num2) && replaced !== '') {
- return num2;
- }
-
- // Otherwise, pass through as string
- console.warn(`Could not fix number: "${original}". Passing through as a string.`);
- return original;
-}
-
-/**
- * Fix BigInt mangling by Excel.
- *
- * For example, converting BigInt to scientific notation.
- * @param {string} str A string that might be a BigInt.
- * @returns {string} If a fix was possible, a string that may be passed to the `BigInt()` constructor without error.
- * Otherwise, the original string is returned.
- *
- * Note that **a string is always returned** to simplify downstream use because BigInt serialization is not possible with `JSON.stringify()`.
- */
-export function fixExcelBigInt(str) {
- if (str === '') { return str; }
- const original = str;
-
- // Can parse normally without any fixes
- try {
- return BigInt(original).toString();
- } catch { }
-
- // Excel converted to scientific notation
- try {
- const expanded = expandScientificNotation(original, true);
- if (expanded === null) { throw new Error(); }
- return BigInt(expanded).toString();
- } catch { }
-
- // Excel added extra characters from setting the format to "Accounting" or other custom formats
- // Delete everything that's not a number or the decimal separator
- try {
- const str = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), '');
- if (str === '') { throw new Error(); }
- return BigInt(str.split(DECIMAL_SEPARATOR)[0]).toString();
- } catch { }
-
- // Otherwise, pass through as string
- console.warn(`Could not fix bigint: "${original}". Passing through as a string.`);
- return original;
-}
-
-/**
- * Fix date mangling by Excel.
- *
- * For example, converting date to number.
- * @param {string} str A string that might be a date.
- * @returns {Date|string} If a fix was possible, a date. Otherwise, the original string is returned.
- */
-export function fixExcelDate(str) {
- if (str === '') { return str; }
- const original = str;
-
- // Excel converted date to a number
- // This check MUST come before the new Date() check because numbers can be parsed as dates.
- const num = Number(original);
- if (!Number.isNaN(num)) {
- const day = Math.trunc(num); // Days elapsed since January 1, 1900, including non-existent February 29, 1900
- const time = (num % 1) * 86400000; // Time of day as decimal, so 0 = 12 AM and 0.999... = 11:59:59.999... PM
- if (num < 61) {
- return new Date(Date.UTC(0, 0, day, 0, 0, 0, time));
- }
- return new Date(Date.UTC(0, 0, day - 1, 0, 0, 0, time)); // Offset for non-existent February 29, 1900
- }
-
- // Can parse normally without any fixes
- const date = new Date(original);
- if (date.toString() !== 'Invalid Date') {
- return date;
- }
-
- // Otherwise, pass through as string
- console.warn(`Could not fix date: "${original}". Passing through as a string.`);
- return original;
-}
-
/**
* Metadata about CSV columns used to normalize raw CSV data.
* @typedef {Object} CSVNormalizerHeader
const emptyField = value === '' ? true : false;
switch (type) {
case 'string': break;
- case 'number': value = passthroughNumber ? value : fixExcelNumber(value); break;
- case 'bigint': value = passthroughBigInt ? value : fixExcelBigInt(value); break;
- case 'date': value = passthroughDate ? value : fixExcelDate(value); break;
+ case 'number': value = passthroughNumber ? value : CSVNormalizer.fixExcelNumber(value); break;
+ case 'bigint': value = passthroughBigInt ? value : CSVNormalizer.fixExcelBigInt(value); break;
+ case 'date': value = passthroughDate ? value : CSVNormalizer.fixExcelDate(value); break;
}
if (emptyField && defaultValue !== null) {
value = defaultValue;
controller.enqueue(JSON.stringify(out));
}
+
+ /**
+ * Fix number mangling by Excel.
+ *
+ * For example, converting numbers to dates or adding extra characters (e.g., whitespace, thousands separators, or currency symbols).
+ * @param {string} str A string that might be a number.
+ * @returns {number|string} If a fix was possible, a number. Otherwise, the original string is returned.
+ */
+ static fixExcelNumber(str) {
+ if (str === '') { return str; }
+ const original = str;
+
+ // Can parse normally without any fixes
+ const num1 = Number(original);
+ if (!Number.isNaN(num1)) {
+ return num1;
+ }
+
+ // Excel converted the number to a date
+ const date = new Date(original);
+ if (date.toString() !== 'Invalid Date') {
+ // Excel incorrectly thinks 1900 was a leap year, need to offset 1 day before 3/1/1900
+ const leapYearOffset = (original === '2/29/1900' || date.valueOf() < -2203862400000) ? -1 : 0;
+ // Convert JavaScript date to Excel serialized date:
+ // Days since Unix epoch - Timezone offset in days + Days between Jan 1, 1900 and Unix epoch start
+ return (date.valueOf() / 1000 / 60 / 60 / 24) - (date.getTimezoneOffset() / 60 / 24) + 25569 + leapYearOffset;
+ }
+
+ // Excel added extra characters from setting the format to "Accounting" or other custom formats
+ // Delete everything that's not a number or the decimal separator
+ const replaced = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), '');
+ const num2 = Number(replaced);
+ if (!Number.isNaN(num2) && replaced !== '') {
+ return num2;
+ }
+
+ // Otherwise, pass through as string
+ console.warn(`Could not fix number: "${original}". Passing through as a string.`);
+ return original;
+ }
+
+ /**
+ * Fix BigInt mangling by Excel.
+ *
+ * For example, converting BigInt to scientific notation.
+ * @param {string} str A string that might be a BigInt.
+ * @returns {string} If a fix was possible, a string that may be passed to the `BigInt()` constructor without error.
+ * Otherwise, the original string is returned.
+ *
+ * Note that **a string is always returned** to simplify downstream use because BigInt serialization is not possible with `JSON.stringify()`.
+ */
+ static fixExcelBigInt(str) {
+ if (str === '') { return str; }
+ const original = str;
+
+ // Can parse normally without any fixes
+ try {
+ return BigInt(original).toString();
+ } catch { }
+
+ // Excel converted to scientific notation
+ try {
+ const expanded = expandScientificNotation(original, true);
+ if (expanded === null) { throw new Error(); }
+ return BigInt(expanded).toString();
+ } catch { }
+
+ // Excel added extra characters from setting the format to "Accounting" or other custom formats
+ // Delete everything that's not a number or the decimal separator
+ try {
+ const str = original.replaceAll(new RegExp(`[^0-9${DECIMAL_SEPARATOR}]`, 'g'), '');
+ if (str === '') { throw new Error(); }
+ return BigInt(str.split(DECIMAL_SEPARATOR)[0]).toString();
+ } catch { }
+
+ // Otherwise, pass through as string
+ console.warn(`Could not fix bigint: "${original}". Passing through as a string.`);
+ return original;
+ }
+
+ /**
+ * Fix date mangling by Excel.
+ *
+ * For example, converting date to number.
+ * @param {string} str A string that might be a date.
+ * @returns {Date|string} If a fix was possible, a date. Otherwise, the original string is returned.
+ */
+ static fixExcelDate(str) {
+ if (str === '') { return str; }
+ const original = str;
+
+ // Excel converted date to a number
+ // This check MUST come before the new Date() check because numbers can be parsed as dates.
+ const num = Number(original);
+ if (!Number.isNaN(num)) {
+ const day = Math.trunc(num); // Days elapsed since January 1, 1900, including non-existent February 29, 1900
+ const time = (num % 1) * 86400000; // Time of day as decimal, so 0 = 12 AM and 0.999... = 11:59:59.999... PM
+ if (num < 61) {
+ return new Date(Date.UTC(0, 0, day, 0, 0, 0, time));
+ }
+ return new Date(Date.UTC(0, 0, day - 1, 0, 0, 0, time)); // Offset for non-existent February 29, 1900
+ }
+
+ // Can parse normally without any fixes
+ const date = new Date(original);
+ if (date.toString() !== 'Invalid Date') {
+ return date;
+ }
+
+ // Otherwise, pass through as string
+ console.warn(`Could not fix date: "${original}". Passing through as a string.`);
+ return original;
+ }
}
/**
import { csvStreamEqual, csvStreamEqualWritable, createCSVMockStream } from './utils.js';
import { CSVReader } from '../src/core.js';
-import { toCamelCase, expandScientificNotation, fixExcelNumber, fixExcelBigInt, fixExcelDate, CSVNormalizer, CSVDenormalizer } from '../src/normalization.js';
+import { toCamelCase, expandScientificNotation, CSVNormalizer, CSVDenormalizer } from '../src/normalization.js';
suite('toCamelCase', { concurrency: true }, () => {
test('does not modify strings already in camelCase', { concurrency: true }, () => {
});
});
-suite('fixExcelNumber', { concurrency: true }, () => {
+suite('CSVNormalizer.fixExcelNumber', { concurrency: true }, () => {
test('parses numbers', { concurrency: true }, () => {
- deepStrictEqual(fixExcelNumber('123'), 123);
- deepStrictEqual(fixExcelNumber('123.456'), 123.456);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('123'), 123);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('123.456'), 123.456);
});
test('fixes numbers mangled to dates', { concurrency: true }, () => {
- deepStrictEqual(fixExcelNumber('1/1/1900'), 1);
- deepStrictEqual(fixExcelNumber('1/2/1900'), 2);
- deepStrictEqual(fixExcelNumber('2/28/1900'), 59);
- deepStrictEqual(fixExcelNumber('2/29/1900'), 60);
- deepStrictEqual(fixExcelNumber('3/1/1900'), 61);
- deepStrictEqual(fixExcelNumber('12/31/1969'), 25568);
- deepStrictEqual(fixExcelNumber('1/1/1970'), 25569);
- deepStrictEqual(fixExcelNumber('1/2/1970'), 25570);
- deepStrictEqual(fixExcelNumber('12/31/2023'), 45291);
- deepStrictEqual(fixExcelNumber('1/1/2024'), 45292);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('1/1/1900'), 1);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('1/2/1900'), 2);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('2/28/1900'), 59);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('2/29/1900'), 60);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('3/1/1900'), 61);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('12/31/1969'), 25568);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('1/1/1970'), 25569);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('1/2/1970'), 25570);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('12/31/2023'), 45291);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('1/1/2024'), 45292);
});
test('fixes numbers mangled to accounting format', { concurrency: true }, () => {
- deepStrictEqual(fixExcelNumber(' $123.00 '), 123);
- deepStrictEqual(fixExcelNumber(' $123.45 '), 123.45);
- deepStrictEqual(fixExcelNumber(' $10,000,000.00 '), 10000000);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber(' $123.00 '), 123);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber(' $123.45 '), 123.45);
+ deepStrictEqual(CSVNormalizer.fixExcelNumber(' $10,000,000.00 '), 10000000);
});
test('passes through non-number values', { concurrency: true }, () => {
- deepStrictEqual(fixExcelNumber('abc'), 'abc');
+ deepStrictEqual(CSVNormalizer.fixExcelNumber('abc'), 'abc');
});
test('passes through blank value', { concurrency: true }, () => {
- deepStrictEqual(fixExcelNumber(''), '');
+ deepStrictEqual(CSVNormalizer.fixExcelNumber(''), '');
});
});
-suite('fixExcelBigInt', { concurrency: true }, () => {
+suite('CSVNormalizer.fixExcelBigInt', { concurrency: true }, () => {
test('parses BigInt', { concurrency: true }, () => {
- deepStrictEqual(fixExcelBigInt('123'), '123');
- deepStrictEqual(fixExcelBigInt('123456'), '123456');
- deepStrictEqual(fixExcelBigInt('123456789123456789123456789'), '123456789123456789123456789');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt('123'), '123');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt('123456'), '123456');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt('123456789123456789123456789'), '123456789123456789123456789');
});
test('fixes BigInt mangled to scientific notation', { concurrency: true }, () => {
- deepStrictEqual(fixExcelBigInt('1E+18'), '1000000000000000000');
- deepStrictEqual(fixExcelBigInt('1.123456789123456789E18'), '1123456789123456789');
- deepStrictEqual(fixExcelBigInt('1.1234567891234567891234E18'), '1123456789123456789');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt('1E+18'), '1000000000000000000');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt('1.123456789123456789E18'), '1123456789123456789');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt('1.1234567891234567891234E18'), '1123456789123456789');
});
test('fixes BigInt mangled to accounting format', { concurrency: true }, () => {
- deepStrictEqual(fixExcelBigInt(' $123.00 '), '123');
- deepStrictEqual(fixExcelBigInt(' $123.45 '), '123');
- deepStrictEqual(fixExcelBigInt(' $1,123,123,123,123,123,123.12 '), '1123123123123123123');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt(' $123.00 '), '123');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt(' $123.45 '), '123');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt(' $1,123,123,123,123,123,123.12 '), '1123123123123123123');
});
test('passes through non-BigInt values', { concurrency: true }, () => {
- deepStrictEqual(fixExcelBigInt('abc'), 'abc');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt('abc'), 'abc');
});
test('passes through blank value', { concurrency: true }, () => {
- deepStrictEqual(fixExcelBigInt(''), '');
+ deepStrictEqual(CSVNormalizer.fixExcelBigInt(''), '');
});
});
-suite('fixExcelDate', { concurrency: true }, () => {
+suite('CSVNormalizer.fixExcelDate', { concurrency: true }, () => {
test('parses dates', { concurrency: true }, () => {
- deepStrictEqual(fixExcelDate('12/31/24').valueOf(), new Date('12/31/24').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('12/31/24').valueOf(), new Date('12/31/24').valueOf());
});
test('fixes dates mangled to numbers', { concurrency: true }, () => {
- deepStrictEqual(fixExcelDate('1').valueOf(), new Date('1900-01-01T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('2').valueOf(), new Date('1900-01-02T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('59').valueOf(), new Date('1900-02-28T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('60').valueOf(), new Date('1900-02-29T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('61').valueOf(), new Date('1900-03-01T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('25568').valueOf(), new Date('1969-12-31T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('25569').valueOf(), new Date('1970-01-01T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('25570').valueOf(), new Date('1970-01-02T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('45291').valueOf(), new Date('2023-12-31T00:00:00.000Z').valueOf());
- deepStrictEqual(fixExcelDate('45292').valueOf(), new Date('2024-01-01T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('1').valueOf(), new Date('1900-01-01T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('2').valueOf(), new Date('1900-01-02T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('59').valueOf(), new Date('1900-02-28T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('60').valueOf(), new Date('1900-02-29T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('61').valueOf(), new Date('1900-03-01T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('25568').valueOf(), new Date('1969-12-31T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('25569').valueOf(), new Date('1970-01-01T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('25570').valueOf(), new Date('1970-01-02T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('45291').valueOf(), new Date('2023-12-31T00:00:00.000Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('45292').valueOf(), new Date('2024-01-01T00:00:00.000Z').valueOf());
});
test('fixes date times mangled to numbers', { concurrency: true }, () => {
- deepStrictEqual(fixExcelDate('43915.31372').valueOf(), new Date('2020-03-25T07:31:45.407Z').valueOf());
- deepStrictEqual(fixExcelDate('44067.63').valueOf(), new Date('2020-08-24T15:07:11.999Z').valueOf());
- deepStrictEqual(fixExcelDate('44309.63502').valueOf(), new Date('2021-04-23T15:14:25.728Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('43915.31372').valueOf(), new Date('2020-03-25T07:31:45.407Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('44067.63').valueOf(), new Date('2020-08-24T15:07:11.999Z').valueOf());
+ deepStrictEqual(CSVNormalizer.fixExcelDate('44309.63502').valueOf(), new Date('2021-04-23T15:14:25.728Z').valueOf());
});
test('passes through non-date values', { concurrency: true }, () => {
- deepStrictEqual(fixExcelDate('abc'), 'abc');
+ deepStrictEqual(CSVNormalizer.fixExcelDate('abc'), 'abc');
});
test('passes through blank value', { concurrency: true }, () => {
- deepStrictEqual(fixExcelDate(''), '');
+ deepStrictEqual(CSVNormalizer.fixExcelDate(''), '');
});
});
git.ardis.lu / csvio / commitdiff