Avoid flushing to disk

Performance cost is too great (for a 100k row/100MB CSV, increases write time from 6s to 60s) for benefit (memory usage, partial recoverability if interrupted). Need to use a more intelligent approach than flushing on every row.

diff --git a/src/core.js b/src/core.js
index 803ebb175e69979e881643a3edf71306d53215ee..209198798539cbee6b395f60b8cdbffdba0c6dd4 100644 (file)
--- a/src/core.js
+++ b/src/core.js
@@ -1,5 +1,5 @@
 import { createReadStream } from 'node:fs';
-import { writeFile, mkdir, open } from 'node:fs/promises';
+import { mkdir, open } from 'node:fs/promises';
 import { dirname, normalize } from 'node:path';
 import { Readable } from 'node:stream';
 /** @import { PathLike } from 'node:fs' */
@@ -170,8 +170,7 @@ export function createCSVWritableStream(path) {
   return new WritableStream({
     async start() {
       await mkdir(dir, { recursive: true });
-      await writeFile(fullPath, '', { flush: true });
-      handle = await open(fullPath, 'a');
+      handle = await open(fullPath, 'w');
     },
     async write(chunk) {
       let data;
@@ -182,7 +181,6 @@ export function createCSVWritableStream(path) {
         data = chunk;
       }
       await handle.write(data);
-      await handle.sync();
     },
     async close() {
       await handle.close();