From 7ce9d4a2f039feccded9fcef10db4b0a7ef2ab93 Mon Sep 17 00:00:00 2001 From: Ardis Lu Date: Tue, 21 Jan 2025 21:59:15 -0800 Subject: [PATCH] Renumber and categorize examples Headers make the examples easier to skim. Using minor numberings is also useful to later add more examples without having to renumber existing groupings each time. --- README.md | 10 +++--- examples/README.md | 36 ++++++++++++++------ examples/data/{ex1-in.csv => ex1_1-in.csv} | 0 examples/data/{ex1-out.csv => ex1_1-out.csv} | 0 examples/data/{ex2-in.csv => ex1_2-in.csv} | 0 examples/data/{ex2-out.csv => ex1_2-out.csv} | 0 examples/data/{ex3-in.csv => ex1_3-in.csv} | 0 examples/data/{ex3-out.csv => ex1_3-out.csv} | 0 examples/data/{ex5-in.csv => ex1_4-in.csv} | 0 examples/data/{ex5-out.csv => ex1_4-out.csv} | 0 examples/data/{ex6-in.csv => ex2_1-in.csv} | 0 examples/data/{ex6-out.csv => ex2_1-out.csv} | 0 examples/data/{ex9-in.csv => ex2_2-in.csv} | 0 examples/data/{ex9-out.csv => ex2_2-out.csv} | 0 examples/data/{ex7-in.csv => ex3_1-in.csv} | 0 examples/data/{ex7-out.csv => ex3_1-out.csv} | 0 examples/data/{ex8-in.csv => ex3_2-in.csv} | 0 examples/data/{ex8-out.csv => ex3_2-out.csv} | 0 examples/data/{ex4-in.csv => ex4_1-in.csv} | 0 examples/data/{ex4-out.csv => ex4_1-out.csv} | 0 examples/{ex1.js => ex1_1.js} | 6 ++-- examples/{ex2.js => ex1_2.js} | 6 ++-- examples/{ex3.js => ex1_3.js} | 6 ++-- examples/{ex5.js => ex1_4.js} | 6 ++-- examples/{ex6.js => ex2_1.js} | 6 ++-- examples/{ex9.js => ex2_2.js} | 6 ++-- examples/{ex7.js => ex3_1.js} | 6 ++-- examples/{ex8.js => ex3_2.js} | 6 ++-- examples/{ex4.js => ex4_1.js} | 8 ++--- test/examples.test.js | 2 +- 30 files changed, 60 insertions(+), 44 deletions(-) rename examples/data/{ex1-in.csv => ex1_1-in.csv} (100%) rename examples/data/{ex1-out.csv => ex1_1-out.csv} (100%) rename examples/data/{ex2-in.csv => ex1_2-in.csv} (100%) rename examples/data/{ex2-out.csv => ex1_2-out.csv} (100%) rename examples/data/{ex3-in.csv => ex1_3-in.csv} (100%) rename examples/data/{ex3-out.csv => ex1_3-out.csv} (100%) rename examples/data/{ex5-in.csv => ex1_4-in.csv} (100%) rename examples/data/{ex5-out.csv => ex1_4-out.csv} (100%) rename examples/data/{ex6-in.csv => ex2_1-in.csv} (100%) rename examples/data/{ex6-out.csv => ex2_1-out.csv} (100%) rename examples/data/{ex9-in.csv => ex2_2-in.csv} (100%) rename examples/data/{ex9-out.csv => ex2_2-out.csv} (100%) rename examples/data/{ex7-in.csv => ex3_1-in.csv} (100%) rename examples/data/{ex7-out.csv => ex3_1-out.csv} (100%) rename examples/data/{ex8-in.csv => ex3_2-in.csv} (100%) rename examples/data/{ex8-out.csv => ex3_2-out.csv} (100%) rename examples/data/{ex4-in.csv => ex4_1-in.csv} (100%) rename examples/data/{ex4-out.csv => ex4_1-out.csv} (100%) rename examples/{ex1.js => ex1_1.js} (80%) rename examples/{ex2.js => ex1_2.js} (81%) rename examples/{ex3.js => ex1_3.js} (86%) rename examples/{ex5.js => ex1_4.js} (82%) rename examples/{ex6.js => ex2_1.js} (80%) rename examples/{ex9.js => ex2_2.js} (66%) rename examples/{ex7.js => ex3_1.js} (83%) rename examples/{ex8.js => ex3_2.js} (83%) rename examples/{ex4.js => ex4_1.js} (85%) diff --git a/README.md b/README.md index c9540f3..861afbb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ CSV I/O (`csvio`) is a Node.js library for processing CSV files. ## Example -Assume `./examples/data/ex1-in.csv` looks like: +Assume `./examples/data/ex1_1-in.csv` looks like: ```plaintext columnA,columnB @@ -24,12 +24,12 @@ function timesTwo(row) { return [Number(row[0]) * 2, Number(row[1]) * 2]; } -await new CSVReader('./examples/data/ex1-in.csv') +await new CSVReader('./examples/data/ex1_1-in.csv') .pipeThrough(new CSVTransformer(timesTwo)) - .pipeTo(new CSVWriter('./examples/data/ex1-out.csv')); + .pipeTo(new CSVWriter('./examples/data/ex1_1-out.csv')); ``` -A new `./examples/data/ex1-out.csv` file will be created: +A new `./examples/data/ex1_1-out.csv` file will be created: ```plaintext columnA,columnB @@ -76,7 +76,7 @@ A `ReadableStream` where each chunk is one row from a local CSV file. ```javascript import { CSVReader } from './src/index.js'; -const readableStream = new CSVReader('./examples/data/ex1-in.csv'); +const readableStream = new CSVReader('./examples/data/ex1_1-in.csv'); for await (const row of readableStream) { console.log(row); diff --git a/examples/README.md b/examples/README.md index f2000cd..118c703 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,25 +5,41 @@ This folder contains end-to-end examples of using this library. All examples can be run directly without additional setup using `node`. E.g.: ``` -node ./examples/ex1.js +node ./examples/ex1_1.js ``` ## Table of contents -[Example 1: Basic usage](./ex1.js) +### Section 1: Common usage -[Example 2: Updating headers in the output](./ex2.js) +These examples demonstrate the most common, expected usage. -[Example 3: Chaining multiple transformations](./ex3.js) +[Example 1.1: Basic usage](./ex1_1.js) -[Example 4: Using `CSVNormalizer` and `CSVDenormalizer`](./ex4.js) +[Example 1.2: Updating headers in the output](./ex1_2.js) -[Example 5: Handling an irregularly-shaped CSV file](./ex5.js) +[Example 1.3: Chaining multiple transformations](./ex1_3.js) -[Example 6: Handling a flaky transformation (basic retry strategy)](./ex6.js) +[Example 1.4: Handling an irregularly-shaped CSV file](./ex1_4.js) -[Example 7: Processing rows in batches (multi-row input)](./ex7.js) +### Section 2: Error strategies -[Example 8: Processing rows concurrently (async)](./ex8.js) +These examples demonstrate using the `onError` option to handle transformations that may throw errors. -[Example 9: Handling a flaky transformation (graceful failure)](./ex9.js) \ No newline at end of file +[Example 2.1: Basic retry strategy](./ex2_1.js) + +[Example 2.2: Graceful failure (placeholder value)](./ex2_2.js) + +### Section 3: Batch and concurrent processing + +These examples demonstrate using the `maxBatchSize` and `maxConcurrent` options to process more than one CSV row at a time. + +[Example 3.1: Batch processing (multi-row input) using `maxBatchSize`](./ex3_1.js) + +[Example 3.2: Concurrent processing (async) using `maxConcurrent`](./ex3_2.js) + +### Section 4: Extended API + +These examples demonstrate the extended API. + +[Example 4.1: Using `CSVNormalizer` and `CSVDenormalizer`](./ex4_1.js) diff --git a/examples/data/ex1-in.csv b/examples/data/ex1_1-in.csv similarity index 100% rename from examples/data/ex1-in.csv rename to examples/data/ex1_1-in.csv diff --git a/examples/data/ex1-out.csv b/examples/data/ex1_1-out.csv similarity index 100% rename from examples/data/ex1-out.csv rename to examples/data/ex1_1-out.csv diff --git a/examples/data/ex2-in.csv b/examples/data/ex1_2-in.csv similarity index 100% rename from examples/data/ex2-in.csv rename to examples/data/ex1_2-in.csv diff --git a/examples/data/ex2-out.csv b/examples/data/ex1_2-out.csv similarity index 100% rename from examples/data/ex2-out.csv rename to examples/data/ex1_2-out.csv diff --git a/examples/data/ex3-in.csv b/examples/data/ex1_3-in.csv similarity index 100% rename from examples/data/ex3-in.csv rename to examples/data/ex1_3-in.csv diff --git a/examples/data/ex3-out.csv b/examples/data/ex1_3-out.csv similarity index 100% rename from examples/data/ex3-out.csv rename to examples/data/ex1_3-out.csv diff --git a/examples/data/ex5-in.csv b/examples/data/ex1_4-in.csv similarity index 100% rename from examples/data/ex5-in.csv rename to examples/data/ex1_4-in.csv diff --git a/examples/data/ex5-out.csv b/examples/data/ex1_4-out.csv similarity index 100% rename from examples/data/ex5-out.csv rename to examples/data/ex1_4-out.csv diff --git a/examples/data/ex6-in.csv b/examples/data/ex2_1-in.csv similarity index 100% rename from examples/data/ex6-in.csv rename to examples/data/ex2_1-in.csv diff --git a/examples/data/ex6-out.csv b/examples/data/ex2_1-out.csv similarity index 100% rename from examples/data/ex6-out.csv rename to examples/data/ex2_1-out.csv diff --git a/examples/data/ex9-in.csv b/examples/data/ex2_2-in.csv similarity index 100% rename from examples/data/ex9-in.csv rename to examples/data/ex2_2-in.csv diff --git a/examples/data/ex9-out.csv b/examples/data/ex2_2-out.csv similarity index 100% rename from examples/data/ex9-out.csv rename to examples/data/ex2_2-out.csv diff --git a/examples/data/ex7-in.csv b/examples/data/ex3_1-in.csv similarity index 100% rename from examples/data/ex7-in.csv rename to examples/data/ex3_1-in.csv diff --git a/examples/data/ex7-out.csv b/examples/data/ex3_1-out.csv similarity index 100% rename from examples/data/ex7-out.csv rename to examples/data/ex3_1-out.csv diff --git a/examples/data/ex8-in.csv b/examples/data/ex3_2-in.csv similarity index 100% rename from examples/data/ex8-in.csv rename to examples/data/ex3_2-in.csv diff --git a/examples/data/ex8-out.csv b/examples/data/ex3_2-out.csv similarity index 100% rename from examples/data/ex8-out.csv rename to examples/data/ex3_2-out.csv diff --git a/examples/data/ex4-in.csv b/examples/data/ex4_1-in.csv similarity index 100% rename from examples/data/ex4-in.csv rename to examples/data/ex4_1-in.csv diff --git a/examples/data/ex4-out.csv b/examples/data/ex4_1-out.csv similarity index 100% rename from examples/data/ex4-out.csv rename to examples/data/ex4_1-out.csv diff --git a/examples/ex1.js b/examples/ex1_1.js similarity index 80% rename from examples/ex1.js rename to examples/ex1_1.js index a81f060..11389cd 100644 --- a/examples/ex1.js +++ b/examples/ex1_1.js @@ -1,4 +1,4 @@ -// ## Example 1: Basic usage +// ## Example 1.1: Basic usage // // Minimal example of a user-defined function to transform a CSV. // @@ -14,6 +14,6 @@ function timesTwo(row) { return [Number(row[0]) * 2, Number(row[1]) * 2]; } -await new CSVReader(new URL('./data/ex1-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex1_1-in.csv', import.meta.url)) .pipeThrough(new CSVTransformer(timesTwo)) - .pipeTo(new CSVWriter(new URL('./data/ex1-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex1_1-out.csv', import.meta.url))); \ No newline at end of file diff --git a/examples/ex2.js b/examples/ex1_2.js similarity index 81% rename from examples/ex2.js rename to examples/ex1_2.js index de5cdf9..c62a718 100644 --- a/examples/ex2.js +++ b/examples/ex1_2.js @@ -1,4 +1,4 @@ -// ## Example 2: Updating headers in the output +// ## Example 1.2: Updating headers in the output // // Set `includeHeaders: true` to manually handle the header row (first row). // @@ -18,6 +18,6 @@ function sum(row) { return [...row, Number(row[0]) + Number(row[1])]; } -await new CSVReader(new URL('./data/ex2-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex1_2-in.csv', import.meta.url)) .pipeThrough(new CSVTransformer(sum, { includeHeaders: true })) - .pipeTo(new CSVWriter(new URL('./data/ex2-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex1_2-out.csv', import.meta.url))); \ No newline at end of file diff --git a/examples/ex3.js b/examples/ex1_3.js similarity index 86% rename from examples/ex3.js rename to examples/ex1_3.js index 42223af..0e9e2fb 100644 --- a/examples/ex3.js +++ b/examples/ex1_3.js @@ -1,4 +1,4 @@ -// ## Example 3: Chaining multiple transformations +// ## Example 1.3: Chaining multiple transformations // // Set `rawInput: true` and/or `rawOutput: true` to skip JSON deserialization (`rawInput`) and/or JSON serialization // (`rawOutput`) when handling the transformation function input/output. These options are intended for advanced @@ -22,7 +22,7 @@ function calculate(obj) { return [name, paddedId]; } -await new CSVReader(new URL('./data/ex3-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex1_3-in.csv', import.meta.url)) .pipeThrough(new CSVTransformer(parse, { includeHeaders: true, rawOutput: true })) .pipeThrough(new CSVTransformer(calculate, { includeHeaders: true, rawInput: true })) - .pipeTo(new CSVWriter(new URL('./data/ex3-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex1_3-out.csv', import.meta.url))); \ No newline at end of file diff --git a/examples/ex5.js b/examples/ex1_4.js similarity index 82% rename from examples/ex5.js rename to examples/ex1_4.js index d949fcf..2f32437 100644 --- a/examples/ex5.js +++ b/examples/ex1_4.js @@ -1,4 +1,4 @@ -// ## Example 5: Handling an irregularly-shaped CSV file +// ## Example 1.4: Handling an irregularly-shaped CSV file // // `CSVReader` can parse CSVs that have rows with a mismatching number of columns, for example CSVs // with informational file headers. This information can be handled as required in the transformation function. @@ -24,6 +24,6 @@ function parse(row) { return [report, id, date, ...row]; } -await new CSVReader(new URL('./data/ex5-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex1_4-in.csv', import.meta.url)) .pipeThrough(new CSVTransformer(parse, { includeHeaders: true })) - .pipeTo(new CSVWriter(new URL('./data/ex5-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex1_4-out.csv', import.meta.url))); \ No newline at end of file diff --git a/examples/ex6.js b/examples/ex2_1.js similarity index 80% rename from examples/ex6.js rename to examples/ex2_1.js index 25b6dfa..d6e6885 100644 --- a/examples/ex6.js +++ b/examples/ex2_1.js @@ -1,4 +1,4 @@ -// ## Example 6: Handling a flaky transformation (basic retry strategy) +// ## Example 2.1: Basic retry strategy // // Pass a function to the `onError` option of `CSVTransformer` to catch errors thrown by the transformation // function. @@ -25,6 +25,6 @@ function flaky(row) { return [`${v}: pass`]; } -await new CSVReader(new URL('./data/ex6-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex2_1-in.csv', import.meta.url)) .pipeThrough(new CSVTransformer(flaky, { onError: (row, e, fn) => retry(row, fn, 1000) })) - .pipeTo(new CSVWriter(new URL('./data/ex6-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex2_1-out.csv', import.meta.url))); \ No newline at end of file diff --git a/examples/ex9.js b/examples/ex2_2.js similarity index 66% rename from examples/ex9.js rename to examples/ex2_2.js index 1cbfba3..76b90de 100644 --- a/examples/ex9.js +++ b/examples/ex2_2.js @@ -1,4 +1,4 @@ -// ## Example 9: Handling a flaky transformation (graceful failure) +// ## Example 2.2: Graceful failure (placeholder value) import { CSVReader, CSVTransformer, CSVWriter } from '../src/index.js'; @@ -17,6 +17,6 @@ function flaky(row) { return row; } -await new CSVReader(new URL('./data/ex9-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex2_2-in.csv', import.meta.url)) .pipeThrough(new CSVTransformer(flaky, { onError: row => graceful(row) })) - .pipeTo(new CSVWriter(new URL('./data/ex9-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex2_2-out.csv', import.meta.url))); \ No newline at end of file diff --git a/examples/ex7.js b/examples/ex3_1.js similarity index 83% rename from examples/ex7.js rename to examples/ex3_1.js index 21e55a7..0f1610e 100644 --- a/examples/ex7.js +++ b/examples/ex3_1.js @@ -1,4 +1,4 @@ -// ## Example 7: Processing rows in batches (multi-row input) +// ## Example 3.1: Batch processing (multi-row input) using `maxBatchSize` // // Set the `maxBatchSize` option to a number greater than `1` to collect rows into batches before passing the whole // batch to the transformation function. @@ -28,6 +28,6 @@ function process(batch) { return batch.map(row => [batchNumber, sum, ...row]); } -await new CSVReader(new URL('./data/ex7-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex3_1-in.csv', import.meta.url)) .pipeThrough(new CSVTransformer(process, { includeHeaders: true, maxBatchSize: 5 })) - .pipeTo(new CSVWriter(new URL('./data/ex7-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex3_1-out.csv', import.meta.url))); \ No newline at end of file diff --git a/examples/ex8.js b/examples/ex3_2.js similarity index 83% rename from examples/ex8.js rename to examples/ex3_2.js index f6b0e69..57e0fd4 100644 --- a/examples/ex8.js +++ b/examples/ex3_2.js @@ -1,4 +1,4 @@ -// ## Example 8: Processing rows concurrently (async) +// ## Example 3.2: Concurrent processing (async) using `maxConcurrent` // // Set the `maxConcurrent` option to a number greater than `1` to process rows concurrently. The transformation function // will be automatically turned into a promise if it isn't already async. @@ -27,6 +27,6 @@ async function concurrent(row) { return [tick(), ...row]; } -await new CSVReader(new URL('./data/ex8-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex3_2-in.csv', import.meta.url)) .pipeThrough(new CSVTransformer(concurrent, { includeHeaders: true, maxConcurrent: 5 })) - .pipeTo(new CSVWriter(new URL('./data/ex8-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex3_2-out.csv', import.meta.url))); \ No newline at end of file diff --git a/examples/ex4.js b/examples/ex4_1.js similarity index 85% rename from examples/ex4.js rename to examples/ex4_1.js index 072edc5..0a55f12 100644 --- a/examples/ex4.js +++ b/examples/ex4_1.js @@ -1,6 +1,6 @@ -// ## Example 4: Using `CSVNormalizer` and `CSVDenormalizer` +// ## Example 4.1: Using `CSVNormalizer` and `CSVDenormalizer` // -// These functions fix common CSV data mangling caused by Excel. In `ex4-in.csv`: +// These functions fix common CSV data mangling caused by Excel. In `ex4_1-in.csv`: // - The "Index" column has been mangled into Excel's "Accounting" format // - The "Phone Number" column has been mangled into scientific notation // - The "Join Timestamp" column has been mangled into a number @@ -30,7 +30,7 @@ const headers = [ } ]; -await new CSVReader(new URL('./data/ex4-in.csv', import.meta.url)) +await new CSVReader(new URL('./data/ex4_1-in.csv', import.meta.url)) .pipeThrough(new CSVNormalizer(headers)) .pipeThrough(new CSVDenormalizer()) - .pipeTo(new CSVWriter(new URL('./data/ex4-out.csv', import.meta.url))); \ No newline at end of file + .pipeTo(new CSVWriter(new URL('./data/ex4_1-out.csv', import.meta.url))); \ No newline at end of file diff --git a/test/examples.test.js b/test/examples.test.js index cdb4728..d8e563b 100644 --- a/test/examples.test.js +++ b/test/examples.test.js @@ -22,7 +22,7 @@ await Promise.all([ const tests = []; const files = await opendir(tempExamples, { recursive: true }); for await (const file of files) { - const { name } = /(?ex\d+).js/.exec(file.name)?.groups ?? { name: null }; // Assuming naming convention "ex*.js" + const { name } = /(?ex\d+_\d+).js/.exec(file.name)?.groups ?? { name: null }; // Assuming naming convention "ex*_*.js" if (name === null) { continue; } tests.push({ name, -- 2.47.3