feat(core): Improvements/overhaul for nodes working with binary data (#7651)

Github issue / Community forum post (link here to close automatically):

---------

Co-authored-by: Giulio Andreini <andreini@netseven.it>
Co-authored-by: Marcus <marcus@n8n.io>
This commit is contained in:
Michael Kret
2024-01-03 13:08:16 +02:00
committed by GitHub
parent 259323b97e
commit 5e16dd4ab4
119 changed files with 4477 additions and 1201 deletions

View File

@@ -50,7 +50,7 @@ export class Compression implements INodeType {
icon: 'fa:file-archive',
group: ['transform'],
subtitle: '={{$parameter["operation"]}}',
version: 1,
version: [1, 1.1],
description: 'Compress and decompress files',
defaults: {
name: 'Compression',
@@ -68,28 +68,49 @@ export class Compression implements INodeType {
{
name: 'Compress',
value: 'compress',
action: 'Compress file(s)',
description: 'Compress files into a zip or gzip archive',
},
{
name: 'Decompress',
value: 'decompress',
action: 'Decompress file(s)',
description: 'Decompress zip or gzip archives',
},
],
default: 'decompress',
},
{
displayName: 'Binary Property',
displayName: 'Input Binary Field(s)',
name: 'binaryPropertyName',
type: 'string',
default: 'data',
required: true,
displayOptions: {
show: {
operation: ['compress', 'decompress'],
operation: ['compress'],
},
},
placeholder: '',
placeholder: 'e.g. data,data2,data3',
hint: 'The name of the input binary field(s) containing the file(s) to be compressed',
description:
'Name of the binary property which contains the data for the file(s) to be compress/decompress. Multiple can be used separated by a comma (,).',
'To process more than one file, use a comma-separated list of the binary fields names',
},
{
displayName: 'Input Binary Field(s)',
name: 'binaryPropertyName',
type: 'string',
default: 'data',
required: true,
displayOptions: {
show: {
operation: ['decompress'],
},
},
placeholder: 'e.g. data',
hint: 'The name of the input binary field(s) containing the file(s) to decompress',
description:
'To process more than one file, use a comma-separated list of the binary fields names',
},
{
displayName: 'Output Format',
@@ -109,16 +130,42 @@ export class Compression implements INodeType {
displayOptions: {
show: {
operation: ['compress'],
'@version': [1],
},
},
description: 'Format of the output file',
description: 'Format of the output',
},
{
displayName: 'Output Format',
name: 'outputFormat',
type: 'options',
default: 'zip',
options: [
{
name: 'Gzip',
value: 'gzip',
},
{
name: 'Zip',
value: 'zip',
},
],
displayOptions: {
show: {
operation: ['compress'],
},
hide: {
'@version': [1],
},
},
description: 'Format of the output',
},
{
displayName: 'File Name',
name: 'fileName',
type: 'string',
default: '',
placeholder: 'data.zip',
placeholder: 'e.g. data.zip',
required: true,
displayOptions: {
show: {
@@ -126,10 +173,10 @@ export class Compression implements INodeType {
outputFormat: ['zip'],
},
},
description: 'Name of the file to be compressed',
description: 'Name of the output file',
},
{
displayName: 'Binary Property Output',
displayName: 'Put Output File in Field',
name: 'binaryPropertyOutput',
type: 'string',
default: 'data',
@@ -139,12 +186,43 @@ export class Compression implements INodeType {
operation: ['compress'],
},
},
placeholder: '',
description:
'Name of the binary property to which to write the data of the compressed files',
hint: 'The name of the output binary field to put the file in',
},
{
displayName: 'Output Prefix',
displayName: 'File Name',
name: 'fileName',
type: 'string',
default: '',
placeholder: 'e.g. data.txt',
displayOptions: {
show: {
operation: ['compress'],
outputFormat: ['gzip'],
},
hide: {
'@version': [1],
},
},
description: 'Name of the output file',
},
{
displayName: 'Put Output File in Field',
name: 'binaryPropertyOutput',
type: 'string',
default: 'data',
displayOptions: {
show: {
outputFormat: ['gzip'],
operation: ['compress'],
},
hide: {
'@version': [1],
},
},
hint: 'The name of the output binary field to put the file in',
},
{
displayName: 'Output File Prefix',
name: 'outputPrefix',
type: 'string',
default: 'data',
@@ -153,9 +231,10 @@ export class Compression implements INodeType {
show: {
operation: ['compress'],
outputFormat: ['gzip'],
'@version': [1],
},
},
description: 'Prefix use for all gzip compressed files',
description: 'Prefix to add to the gzip file',
},
{
displayName: 'Output Prefix',
@@ -168,7 +247,7 @@ export class Compression implements INodeType {
operation: ['decompress'],
},
},
description: 'Prefix use for all decompressed files',
description: 'Prefix to add to the decompressed files',
},
],
};
@@ -178,6 +257,7 @@ export class Compression implements INodeType {
const length = items.length;
const returnData: INodeExecutionData[] = [];
const operation = this.getNodeParameter('operation', 0);
const nodeVersion = this.getNode().typeVersion;
for (let i = 0; i < length; i++) {
try {
@@ -212,20 +292,37 @@ export class Compression implements INodeType {
binaryObject[`${outputPrefix}${zipIndex++}`] = data;
}
} else if (binaryData.fileExtension?.toLowerCase() === 'gz') {
} else if (['gz', 'gzip'].includes(binaryData.fileExtension?.toLowerCase() as string)) {
const file = await gunzip(binaryDataBuffer);
const fileName = binaryData.fileName?.split('.')[0];
let fileExtension;
let mimeType;
if (binaryData.fileName?.endsWith('.gz')) {
const extractedFileExtension = binaryData.fileName.replace('.gz', '').split('.');
if (extractedFileExtension.length > 1) {
fileExtension = extractedFileExtension[extractedFileExtension.length - 1];
mimeType = mime.lookup(fileExtension) as string;
}
}
const propertyName = `${outputPrefix}${index}`;
binaryObject[propertyName] = await this.helpers.prepareBinaryData(
Buffer.from(file.buffer),
fileName,
mimeType,
);
const fileExtension = mime.extension(binaryObject[propertyName].mimeType) as string;
if (!fileExtension) {
mimeType = binaryObject[propertyName].mimeType;
fileExtension = mime.extension(mimeType) as string;
}
binaryObject[propertyName].fileName = `${fileName}.${fileExtension}`;
binaryObject[propertyName].fileExtension = fileExtension;
binaryObject[propertyName].mimeType = mimeType as string;
}
}
@@ -239,14 +336,21 @@ export class Compression implements INodeType {
}
if (operation === 'compress') {
const binaryPropertyNames = this.getNodeParameter('binaryPropertyName', 0)
let binaryPropertyNameIndex = 0;
if (nodeVersion > 1) {
binaryPropertyNameIndex = i;
}
const binaryPropertyNames = this.getNodeParameter(
'binaryPropertyName',
binaryPropertyNameIndex,
)
.split(',')
.map((key) => key.trim());
const outputFormat = this.getNodeParameter('outputFormat', 0) as string;
const zipData: fflate.Zippable = {};
const binaryObject: IBinaryKeyData = {};
for (const [index, binaryPropertyName] of binaryPropertyNames.entries()) {
@@ -261,26 +365,53 @@ export class Compression implements INodeType {
},
];
} else if (outputFormat === 'gzip') {
const outputPrefix = this.getNodeParameter('outputPrefix', 0) as string;
let outputPrefix;
let fileName;
let binaryProperty;
let filePath;
if (nodeVersion > 1) {
outputPrefix = this.getNodeParameter('binaryPropertyOutput', i, 'data');
binaryProperty = `${outputPrefix}${index ? index : ''}`;
fileName = this.getNodeParameter('fileName', i, '') as string;
if (!fileName) {
fileName = binaryData.fileName?.split('.')[0];
} else {
fileName = fileName.replace('.gz', '').replace('.gzip', '');
}
const fileExtension = binaryData.fileExtension
? `.${binaryData.fileExtension.toLowerCase()}`
: '';
filePath = `${fileName}${fileExtension}.gz`;
} else {
outputPrefix = this.getNodeParameter('outputPrefix', 0) as string;
binaryProperty = `${outputPrefix}${index}`;
fileName = binaryData.fileName?.split('.')[0];
filePath = `${fileName}.gzip`;
}
const data = await gzip(binaryDataBuffer);
const fileName = binaryData.fileName?.split('.')[0];
binaryObject[`${outputPrefix}${index}`] = await this.helpers.prepareBinaryData(
binaryObject[binaryProperty] = await this.helpers.prepareBinaryData(
Buffer.from(data),
`${fileName}.gzip`,
filePath,
);
}
}
if (outputFormat === 'zip') {
const fileName = this.getNodeParameter('fileName', 0) as string;
const binaryPropertyOutput = this.getNodeParameter('binaryPropertyOutput', 0);
let zipOptionsIndex = 0;
if (nodeVersion > 1) {
zipOptionsIndex = i;
}
const fileName = this.getNodeParameter('fileName', zipOptionsIndex) as string;
const binaryPropertyOutput = this.getNodeParameter(
'binaryPropertyOutput',
zipOptionsIndex,
);
const buffer = await zip(zipData);
const data = await this.helpers.prepareBinaryData(Buffer.from(buffer), fileName);
returnData.push({