-// ## Example 4: Using `CSVNormalizer` and `createCSVDenormalizationStream`
+// ## Example 4: Using `CSVNormalizer` and `CSVDenormalizer`
//
// These functions fix common CSV data mangling caused by Excel. In `ex4-in.csv`:
// - The "Index" column has been mangled into Excel's "Accounting" format
// - The "Join Timestamp" column has been mangled into a number
// - There are trailing empty rows after deleting excess data from the CSV
// `CSVNormalizer` performs transformations to un-mangle the above using only a declaration of the header names and data types.
-// `createCSVDenormalizationStream` converts the data stream back into a CSV row.
+// `CSVDenormalizer` converts the data stream back into a CSV row.
-import { CSVReader, CSVWriter, CSVNormalizer, createCSVDenormalizationStream } from '../src/index.js';
+import { CSVReader, CSVWriter, CSVNormalizer, CSVDenormalizer } from '../src/index.js';
// The order of items in `headers` is significant, it determines the order of the output CSV columns.
// If an input column is not provided in `headers`, the column is removed from the output CSV.
await new CSVReader(new URL('./data/ex4-in.csv', import.meta.url))
.pipeThrough(new CSVNormalizer(headers))
- .pipeThrough(createCSVDenormalizationStream())
+ .pipeThrough(new CSVDenormalizer())
.pipeTo(new CSVWriter(new URL('./data/ex4-out.csv', import.meta.url)));
\ No newline at end of file
}
/**
- * Create a `TransformStream` to transform a `CSVNormalizerRow` into an array that can be converted to CSV data.
- *
- * @returns {TransformStream} A `TransformStream` where each chunk is one row of the CSV file.
+ * A `TransformStream` to transform a `CSVNormalizerRow` into an array that can be converted to CSV data.
+ * @extends TransformStream
*/
-export function createCSVDenormalizationStream() {
- let firstRow = true;
- return new TransformStream({
- transform(chunk, controller) {
- const row = JSON.parse(chunk);
- if (firstRow) {
- firstRow = false;
- controller.enqueue(JSON.stringify(row.map(f => f.displayName)));
+export class CSVDenormalizer extends TransformStream {
+ constructor() {
+ let firstRow = true;
+ super({
+ transform(chunk, controller) {
+ const row = JSON.parse(chunk);
+ if (firstRow) {
+ firstRow = false;
+ controller.enqueue(JSON.stringify(row.map(f => f.displayName)));
+ }
+ controller.enqueue(JSON.stringify(row.map(f => f.value)));
}
- controller.enqueue(JSON.stringify(row.map(f => f.value)));
- }
- });
+ });
+ }
}
import { csvStreamEqual, csvStreamEqualWritable, createCSVMockStream } from './utils.js';
import { CSVReader } from '../src/core.js';
-import { toCamelCase, expandScientificNotation, fixExcelNumber, fixExcelBigInt, fixExcelDate, CSVNormalizer, createCSVDenormalizationStream } from '../src/normalization.js';
+import { toCamelCase, expandScientificNotation, fixExcelNumber, fixExcelBigInt, fixExcelDate, CSVNormalizer, CSVDenormalizer } from '../src/normalization.js';
suite('toCamelCase', { concurrency: true }, () => {
test('does not modify strings already in camelCase', { concurrency: true }, () => {
{ name: 'columnA', type: 'string' },
{ name: 'columnB', type: 'string' }
]))
- .pipeThrough(createCSVDenormalizationStream())
+ .pipeThrough(new CSVDenormalizer())
.pipeTo(csvStreamEqualWritable([
['columnA', 'columnB'],
['a', 'b']
{ name: 'columnB', type: 'string' },
{ name: 'columnC', type: 'string' }
]))
- .pipeThrough(createCSVDenormalizationStream())
+ .pipeThrough(new CSVDenormalizer())
.pipeTo(csvStreamEqualWritable([
['columnA', 'columnB', 'columnC'],
['a', 'b', 'c']
{ name: 'columnA', type: 'string', displayName: 'Column A' },
{ name: 'columnB', type: 'string', displayName: 'Column B' }
]))
- .pipeThrough(createCSVDenormalizationStream())
+ .pipeThrough(new CSVDenormalizer())
.pipeTo(csvStreamEqualWritable([
['Column A', 'Column B'],
['a', 'b']
{ name: 'columnA', type: 'string' },
{ name: 'columnB', type: 'string' }
]))
- .pipeThrough(createCSVDenormalizationStream())
+ .pipeThrough(new CSVDenormalizer())
.pipeTo(csvStreamEqualWritable([
['columnA', 'columnB'],
['a', 'b']
{ name: 'columnA', type: 'abc' },
{ name: 'columnB', type: 'def' }
]))
- .pipeThrough(createCSVDenormalizationStream())
+ .pipeThrough(new CSVDenormalizer())
.pipeTo(csvStreamEqualWritable([
['columnA', 'columnB'],
['a', 'b']
});
});
-suite('CSVNormalizer and createCSVDenormalizationStream end-to-end', { concurrency: true }, () => {
+suite('CSVNormalizer and CSVDenormalizer end-to-end', { concurrency: true }, () => {
test('can normalize', { concurrency: true }, async () => {
const headers = [
{ name: 'stringCol', displayName: 'String Column', type: 'string', defaultValue: 'N/A' },
]
const stream = new CSVReader('./test/data/normalization.csv')
.pipeThrough(new CSVNormalizer(headers))
- .pipeThrough(createCSVDenormalizationStream());
+ .pipeThrough(new CSVDenormalizer());
await csvStreamEqual(stream, [
['String Column', 'Number Column', 'BigInt Column', 'Date Column'],
['abc ๐จโ๐ฉโ๐งโ๐ฆ', 123456789.123456, '1000000000000000000', '2024-01-01T00:00:00.000Z'],