feat(Spreadsheet File Node): Improve CSV parsing (#7448)

This adds support for
1. custom delimiters
2. reading offsets to avoid having to read a large CSV all at once
3. excluding byte-order-mark

NODE-861
#7443
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
2023-10-18 16:57:37 +02:00
committed by GitHub
parent d8531a53b9
commit 79f23fb939
6 changed files with 273 additions and 15 deletions

View File

@@ -1,5 +1,4 @@
/* eslint-disable n8n-nodes-base/node-filename-against-convention */
import { pipeline } from 'stream/promises';
import type {
IDataObject,
IExecuteFunctions,
@@ -85,7 +84,12 @@ export class SpreadsheetFileV2 implements INodeType {
}
if (fileFormat === 'csv') {
const maxRowCount = options.maxRowCount as number;
const parser = createCSVParser({
delimiter: options.delimiter as string,
fromLine: options.fromLine as number,
bom: options.enableBOM as boolean,
to: maxRowCount > -1 ? maxRowCount : undefined,
columns: options.headerRow !== false,
onRecord: (record) => {
rows.push(record);
@@ -93,9 +97,18 @@ export class SpreadsheetFileV2 implements INodeType {
});
if (binaryData.id) {
const stream = await this.helpers.getBinaryStream(binaryData.id);
await pipeline(stream, parser);
await new Promise<void>(async (resolve, reject) => {
parser.on('error', reject);
parser.on('readable', () => {
stream.unpipe(parser);
stream.destroy();
resolve();
});
stream.pipe(parser);
});
} else {
parser.write(binaryData.data, BINARY_ENCODING);
parser.end();
}
} else {
let workbook: WorkBook;