feat(core): Improvements/overhaul for nodes working with binary data (#7651)

Github issue / Community forum post (link here to close automatically):

---------

Co-authored-by: Giulio Andreini <andreini@netseven.it>
Co-authored-by: Marcus <marcus@n8n.io>
This commit is contained in:
Michael Kret
2024-01-03 13:08:16 +02:00
committed by GitHub
parent 259323b97e
commit 5e16dd4ab4
119 changed files with 4477 additions and 1201 deletions

View File

@@ -0,0 +1,17 @@
{
"node": "n8n-nodes-base.readWriteFile",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.filesreadwrite/"
}
]
},
"alias": ["Binary", "File", "Text", "Open", "Import", "Save", "Export", "Disk", "Transfer"],
"subcategories": {
"Core Nodes": ["Files"]
}
}

View File

@@ -0,0 +1,73 @@
import type {
IExecuteFunctions,
INodeExecutionData,
INodeType,
INodeTypeDescription,
} from 'n8n-workflow';
import * as read from './actions/read.operation';
import * as write from './actions/write.operation';
export class ReadWriteFile implements INodeType {
description: INodeTypeDescription = {
displayName: 'Read/Write Files from Disk',
name: 'readWriteFile',
icon: 'file:readWriteFile.svg',
group: ['input'],
version: 1,
description: 'Read or write files from the computer that runs n8n',
defaults: {
name: 'Read/Write Files from Disk',
},
inputs: ['main'],
outputs: ['main'],
properties: [
{
displayName:
'Use this node to read and write files on the same computer running n8n. To handle files between different computers please use other nodes (e.g. FTP, HTTP Request, AWS).',
name: 'info',
type: 'notice',
default: '',
},
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Read File(s) From Disk',
value: 'read',
description: 'Retrieve one or more files from the computer that runs n8n',
action: 'Read File(s) From Disk',
},
{
name: 'Write File to Disk',
value: 'write',
description: 'Create a binary file on the computer that runs n8n',
action: 'Write File to Disk',
},
],
default: 'read',
},
...read.description,
...write.description,
],
};
async execute(this: IExecuteFunctions) {
const operation = this.getNodeParameter('operation', 0, 'read');
const items = this.getInputData();
let returnData: INodeExecutionData[] = [];
if (operation === 'read') {
returnData = await read.execute.call(this, items);
}
if (operation === 'write') {
returnData = await write.execute.call(this, items);
}
return [returnData];
}
}

View File

@@ -0,0 +1,144 @@
import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
import glob from 'fast-glob';
import { updateDisplayOptions } from '@utils/utilities';
import { errorMapper } from '../helpers/utils';
export const properties: INodeProperties[] = [
{
displayName: 'File(s) Selector',
name: 'fileSelector',
type: 'string',
default: '',
required: true,
placeholder: 'e.g. /home/user/Pictures/**/*.png',
hint: 'Supports patterns, learn more <a href="https://github.com/micromatch/picomatch#basic-globbing" target="_blank">here</a>',
description: "Specify a file's path or path pattern to read multiple files",
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'File Extension',
name: 'fileExtension',
type: 'string',
default: '',
placeholder: 'e.g. zip',
description: 'Extension of the file in the output binary',
},
{
displayName: 'File Name',
name: 'fileName',
type: 'string',
default: '',
placeholder: 'e.g. data.zip',
description: 'Name of the file in the output binary',
},
{
displayName: 'Mime Type',
name: 'mimeType',
type: 'string',
default: '',
placeholder: 'e.g. application/zip',
description: 'Mime type of the file in the output binary',
},
{
displayName: 'Put Output File in Field',
name: 'dataPropertyName',
type: 'string',
default: 'data',
placeholder: 'e.g. data',
description: "By default 'data' is used",
hint: 'The name of the output binary field to put the file in',
},
],
},
];
const displayOptions = {
show: {
operation: ['read'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions, items: INodeExecutionData[]) {
const returnData: INodeExecutionData[] = [];
let fileSelector;
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
try {
fileSelector = this.getNodeParameter('fileSelector', itemIndex) as string;
const options = this.getNodeParameter('options', itemIndex, {});
let dataPropertyName = 'data';
if (options.dataPropertyName) {
dataPropertyName = options.dataPropertyName as string;
}
const files = await glob(fileSelector);
const newItems: INodeExecutionData[] = [];
for (const filePath of files) {
const stream = await this.helpers.createReadStream(filePath);
const binaryData = await this.helpers.prepareBinaryData(stream, filePath);
if (options.fileName !== undefined) {
binaryData.fileName = options.fileName as string;
}
if (options.fileExtension !== undefined) {
binaryData.fileExtension = options.fileExtension as string;
}
if (options.mimeType !== undefined) {
binaryData.mimeType = options.mimeType as string;
}
newItems.push({
binary: {
[dataPropertyName]: binaryData,
},
json: {
mimeType: binaryData.mimeType,
fileType: binaryData.fileType,
fileName: binaryData.fileName,
directory: binaryData.directory,
fileExtension: binaryData.fileExtension,
fileSize: binaryData.fileSize,
},
pairedItem: {
item: itemIndex,
},
});
}
returnData.push(...newItems);
} catch (error) {
const nodeOperatioinError = errorMapper.call(this, error, itemIndex, {
filePath: fileSelector,
operation: 'read',
});
if (this.continueOnFail()) {
returnData.push({
json: {
error: nodeOperatioinError.message,
},
pairedItem: {
item: itemIndex,
},
});
continue;
}
throw nodeOperatioinError;
}
}
return returnData;
}

View File

@@ -0,0 +1,123 @@
import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
import { BINARY_ENCODING } from 'n8n-workflow';
import type { Readable } from 'stream';
import { updateDisplayOptions } from '@utils/utilities';
import { errorMapper } from '../helpers/utils';
export const properties: INodeProperties[] = [
{
displayName: 'File Path and Name',
name: 'fileName',
type: 'string',
default: '',
required: true,
placeholder: 'e.g. /data/example.jpg',
description:
'Path and name of the file that should be written. Also include the file extension.',
},
{
displayName: 'Input Binary Field',
name: 'dataPropertyName',
type: 'string',
default: 'data',
placeholder: 'e.g. data',
required: true,
hint: 'The name of the input binary field containing the file to be written',
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Append',
name: 'append',
type: 'boolean',
default: false,
description:
"Whether to append to an existing file. While it's commonly used with text files, it's not limited to them, however, it wouldn't be applicable for file types that have a specific structure like most binary formats.",
},
],
},
];
const displayOptions = {
show: {
operation: ['write'],
},
};
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions, items: INodeExecutionData[]) {
const returnData: INodeExecutionData[] = [];
let fileName;
let item: INodeExecutionData;
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
try {
const dataPropertyName = this.getNodeParameter('dataPropertyName', itemIndex);
fileName = this.getNodeParameter('fileName', itemIndex) as string;
const options = this.getNodeParameter('options', itemIndex, {});
const flag: string = options.append ? 'a' : 'w';
item = items[itemIndex];
const newItem: INodeExecutionData = {
json: {},
pairedItem: {
item: itemIndex,
},
};
Object.assign(newItem.json, item.json);
const binaryData = this.helpers.assertBinaryData(itemIndex, dataPropertyName);
let fileContent: Buffer | Readable;
if (binaryData.id) {
fileContent = await this.helpers.getBinaryStream(binaryData.id);
} else {
fileContent = Buffer.from(binaryData.data, BINARY_ENCODING);
}
// Write the file to disk
await this.helpers.writeContentToFile(fileName, fileContent, flag);
if (item.binary !== undefined) {
// Create a shallow copy of the binary data so that the old
// data references which do not get changed still stay behind
// but the incoming data does not get changed.
newItem.binary = {};
Object.assign(newItem.binary, item.binary);
}
// Add the file name to data
newItem.json.fileName = fileName;
returnData.push(newItem);
} catch (error) {
const nodeOperatioinError = errorMapper.call(this, error, itemIndex, {
filePath: fileName,
operation: 'write',
});
if (this.continueOnFail()) {
returnData.push({
json: {
error: nodeOperatioinError.message,
},
pairedItem: {
item: itemIndex,
},
});
continue;
}
throw nodeOperatioinError;
}
}
return returnData;
}

View File

@@ -0,0 +1,32 @@
import type { IDataObject, IExecuteFunctions } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
export function errorMapper(
this: IExecuteFunctions,
error: Error,
itemIndex: number,
context?: IDataObject,
) {
let message;
let description;
if (error.message.includes('Cannot create a string longer than')) {
message = 'The file is too large';
description =
'The binary file you are attempting to read exceeds 512MB, which is limit when using default binary data mode, try using the filesystem binary mode. More information <a href="https://docs.n8n.io/hosting/scaling/binary-data/" target="_blank">here</a>.';
} else if (error.message.includes('EACCES') && context?.operation === 'read') {
const path =
((error as unknown as IDataObject).path as string) || (context?.filePath as string);
message = `You don't have the permissions to access ${path}`;
description =
"Verify that the path specified in 'File(s) Selector' is correct, or change the file(s) permissions if needed";
} else if (error.message.includes('EACCES') && context?.operation === 'write') {
const path =
((error as unknown as IDataObject).path as string) || (context?.filePath as string);
message = `You don't have the permissions to write the file ${path}`;
description =
"Specify another destination folder in 'File Path and Name', or change the permissions of the parent folder";
}
return new NodeOperationError(this.getNode(), error, { itemIndex, message, description });
}

View File

@@ -0,0 +1,13 @@
<svg width="512" height="512" viewBox="0 0 512 512" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_1141_1547)">
<path d="M0 12C0 5.37258 5.37258 0 12 0H159V154C159 160.627 164.373 166 171 166H325V242H228.562C210.895 242 194.656 251.705 186.288 267.264L129.203 373.407C125.131 380.978 123 389.44 123 398.037V434H12C5.37257 434 0 428.627 0 422V12Z" fill="#44AA44"/>
<path d="M325 134V127.401C325 124.223 323.74 121.175 321.495 118.925L206.369 3.52481C204.118 1.2682 201.061 0 197.873 0H191V134H325Z" fill="#44AA44"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M228.563 274C222.674 274 217.261 277.235 214.472 282.421L172.211 361H492.64L444.67 281.717C441.772 276.927 436.58 274 430.981 274H228.563Z" fill="#44AA44"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M155 409C155 400.163 162.163 393 171 393H496C504.837 393 512 400.163 512 409V496C512 504.837 504.837 512 496 512H171C162.163 512 155 504.837 155 496V409ZM397 453C397 466.255 386.255 477 373 477C359.745 477 349 466.255 349 453C349 439.745 359.745 429 373 429C386.255 429 397 439.745 397 453ZM445 477C458.255 477 469 466.255 469 453C469 439.745 458.255 429 445 429C431.745 429 421 439.745 421 453C421 466.255 431.745 477 445 477Z" fill="#44AA44"/>
</g>
<defs>
<clipPath id="clip0_1141_1547">
<rect width="512" height="512" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 1.3 KiB

View File

@@ -0,0 +1,101 @@
/* eslint-disable @typescript-eslint/no-loop-func */
import * as Helpers from '@test/nodes/Helpers';
import type { WorkflowTestData } from '@test/nodes/types';
import { executeWorkflow } from '@test/nodes/ExecuteWorkflow';
describe('Test ReadWriteFile Node', () => {
beforeEach(async () => {
await Helpers.initBinaryDataService();
});
const temporaryDir = Helpers.createTemporaryDir();
const directory = __dirname.replace(/\\/gi, '/');
const workflow = Helpers.readJsonFileSync(
'nodes/Files/ReadWriteFile/test/ReadWriteFile.workflow.json',
);
const readFileNode = workflow.nodes.find((n: any) => n.name === 'Read from Disk');
readFileNode.parameters.fileSelector = `${directory}/image.jpg`;
const writeFileNode = workflow.nodes.find((n: any) => n.name === 'Write to Disk');
writeFileNode.parameters.fileName = `${temporaryDir}/image-written.jpg`;
const tests: WorkflowTestData[] = [
{
description: 'nodes/Files/ReadWriteFile/test/ReadWriteFile.workflow.json',
input: {
workflowData: workflow,
},
output: {
nodeData: {
'Read from Disk': [
[
{
json: {
directory,
fileExtension: 'jpg',
fileName: 'image.jpg',
fileSize: '1.04 kB',
fileType: 'image',
mimeType: 'image/jpeg',
},
binary: {
data: {
mimeType: 'image/jpeg',
fileType: 'image',
fileExtension: 'jpg',
data: '/9j/4AAQSkZJRgABAQEASABIAAD/4QBmRXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAAExAAIAAAAQAAAATgAAAAAAARlJAAAD6AABGUkAAAPocGFpbnQubmV0IDUuMC4xAP/bAEMAIBYYHBgUIBwaHCQiICYwUDQwLCwwYkZKOlB0Znp4cmZwboCQuJyAiK6KbnCg2qKuvsTO0M58muLy4MjwuMrOxv/bAEMBIiQkMCowXjQ0XsaEcITGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxv/AABEIAB8AOwMBEgACEQEDEQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEAAhEDEQA/AOgqgrXF2zNHJ5aKcD3oNPZ23di/VKG82bkuTh1OMgdaAdOSLtZ6G5ut0iSeWoOAKAdO27NCqUN8oQrcHDqccDrQDpyRNPdRwEKcsx7CobIebPLORwThc0inGMF724jagNpxG4OOM1dIDAgjIPBpkqUOxnR2pmh85pW3nJB9KkNi4yqTssZ6rSNXNX0ehHFfusYDLuI7+tXY4I40ChQcdzQRKcL7Fb7PcQO32cqUY5we1XqZPtH11KsFoFDGYK7sckkZxVqgTnJlEQXMBZYGUoTkZ7VeoH7RvcqwWaIh80K7k5JIq1QJzkyhbMtvdSxMdqnlc1amgjmx5i5I70inNSVpFdrmaWRltkBVerHvUW57B2AUNGxyOaC+VW9xXLVrcGbcjrtkXqKZZxvveeTAL9APSgiooq1ty3RTMj//2Q==',
directory,
fileName: 'image.jpg',
fileSize: '1.04 kB',
},
},
},
],
],
'Write to Disk': [
[
{
json: {
directory,
fileExtension: 'jpg',
fileName: writeFileNode.parameters.fileName,
fileSize: '1.04 kB',
fileType: 'image',
mimeType: 'image/jpeg',
},
binary: {
data: {
mimeType: 'image/jpeg',
fileType: 'image',
fileExtension: 'jpg',
data: '/9j/4AAQSkZJRgABAQEASABIAAD/4QBmRXhpZgAATU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAAExAAIAAAAQAAAATgAAAAAAARlJAAAD6AABGUkAAAPocGFpbnQubmV0IDUuMC4xAP/bAEMAIBYYHBgUIBwaHCQiICYwUDQwLCwwYkZKOlB0Znp4cmZwboCQuJyAiK6KbnCg2qKuvsTO0M58muLy4MjwuMrOxv/bAEMBIiQkMCowXjQ0XsaEcITGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxv/AABEIAB8AOwMBEgACEQEDEQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEAAhEDEQA/AOgqgrXF2zNHJ5aKcD3oNPZ23di/VKG82bkuTh1OMgdaAdOSLtZ6G5ut0iSeWoOAKAdO27NCqUN8oQrcHDqccDrQDpyRNPdRwEKcsx7CobIebPLORwThc0inGMF724jagNpxG4OOM1dIDAgjIPBpkqUOxnR2pmh85pW3nJB9KkNi4yqTssZ6rSNXNX0ehHFfusYDLuI7+tXY4I40ChQcdzQRKcL7Fb7PcQO32cqUY5we1XqZPtH11KsFoFDGYK7sckkZxVqgTnJlEQXMBZYGUoTkZ7VeoH7RvcqwWaIh80K7k5JIq1QJzkyhbMtvdSxMdqnlc1amgjmx5i5I70inNSVpFdrmaWRltkBVerHvUW57B2AUNGxyOaC+VW9xXLVrcGbcjrtkXqKZZxvveeTAL9APSgiooq1ty3RTMj//2Q==',
directory,
fileName: 'image.jpg',
fileSize: '1.04 kB',
},
},
},
],
],
},
},
},
];
const nodeTypes = Helpers.setup(tests);
for (const testData of tests) {
test(testData.description, async () => {
const { result } = await executeWorkflow(testData, nodeTypes);
const resultNodeData = Helpers.getResultNodeData(result, testData);
resultNodeData.forEach(({ nodeName, resultData }) => {
expect(resultData).toEqual(testData.output.nodeData[nodeName]);
});
expect(result.finished).toEqual(true);
});
}
});

View File

@@ -0,0 +1,72 @@
{
"meta": {
"instanceId": "104a4d08d8897b8bdeb38aaca515021075e0bd8544c983c2bb8c86e6a8e6081c"
},
"nodes": [
{
"parameters": {},
"id": "01b8609f-a345-41de-80bf-6d84276b5e7a",
"name": "When clicking \"Execute Workflow\"",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [
700,
320
]
},
{
"parameters": {
"fileSelector": "C:/Test/image.jpg",
"options": {}
},
"id": "a1ea0fd0-cc95-4de2-bc58-bc980cb1d97e",
"name": "Read from Disk",
"type": "n8n-nodes-base.readWriteFile",
"typeVersion": 1,
"position": [
920,
320
]
},
{
"parameters": {
"operation": "write",
"fileName": "C:/Test/image-written.jpg",
"options": {}
},
"id": "94abac52-bd10-4b57-85b0-691c70989137",
"name": "Write to Disk",
"type": "n8n-nodes-base.readWriteFile",
"typeVersion": 1,
"position": [
1140,
320
]
}
],
"connections": {
"When clicking \"Execute Workflow\"": {
"main": [
[
{
"node": "Read from Disk",
"type": "main",
"index": 0
}
]
]
},
"Read from Disk": {
"main": [
[
{
"node": "Write to Disk",
"type": "main",
"index": 0
}
]
]
}
},
"pinData": {}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.0 KiB