From a66e483a74e5a56fb9407000026629b8344fdbc7 Mon Sep 17 00:00:00 2001 From: Rupenieks <32895755+Rupenieks@users.noreply.github.com> Date: Tue, 4 Aug 2020 08:01:13 +0100 Subject: [PATCH] :sparkles: FileTransfer Node (FTP/SFTP) (#779) * :construction: Setup * :construction: SFTP finished * :white_check_mark: FTP Finished / Tested * Tab indentation * :bug: Fix some issues on FTP Node * :zap: Changed dependencies * Update FileTransfer.node.ts * :zap: Fix issues on FTP-Node * :zap: Fixed uploading to subdirectories Co-authored-by: Jan Oberhauser Co-authored-by: Jan --- .../nodes-base/credentials/Ftp.credentials.ts | 41 ++ .../credentials/Sftp.credentials.ts | 41 ++ packages/nodes-base/nodes/Ftp.node.ts | 434 ++++++++++++++++++ packages/nodes-base/package.json | 7 + 4 files changed, 523 insertions(+) create mode 100644 packages/nodes-base/credentials/Ftp.credentials.ts create mode 100644 packages/nodes-base/credentials/Sftp.credentials.ts create mode 100644 packages/nodes-base/nodes/Ftp.node.ts diff --git a/packages/nodes-base/credentials/Ftp.credentials.ts b/packages/nodes-base/credentials/Ftp.credentials.ts new file mode 100644 index 0000000000..50be96dfa8 --- /dev/null +++ b/packages/nodes-base/credentials/Ftp.credentials.ts @@ -0,0 +1,41 @@ +import { + ICredentialType, + NodePropertyTypes, +} from 'n8n-workflow'; + +export class Ftp implements ICredentialType { + name = 'ftp'; + displayName = 'FTP'; + properties = [ + { + displayName: 'Host', + name: 'host', + required: true, + type: 'string' as NodePropertyTypes, + default: '', + placeholder: 'localhost' + }, + { + displayName: 'Port', + name: 'port', + required: true, + type: 'number' as NodePropertyTypes, + default: 21, + }, + { + displayName: 'Username', + name: 'username', + type: 'string' as NodePropertyTypes, + default: '', + }, + { + displayName: 'Password', + name: 'password', + type: 'string' as NodePropertyTypes, + typeOptions: { + password: true, + }, + default: '', + }, + ]; +} diff --git a/packages/nodes-base/credentials/Sftp.credentials.ts b/packages/nodes-base/credentials/Sftp.credentials.ts new file mode 100644 index 0000000000..a0d307b28a --- /dev/null +++ b/packages/nodes-base/credentials/Sftp.credentials.ts @@ -0,0 +1,41 @@ +import { + ICredentialType, + NodePropertyTypes, +} from 'n8n-workflow'; + +export class Sftp implements ICredentialType { + name = 'sftp'; + displayName = 'SFTP'; + properties = [ + { + displayName: 'Host', + name: 'host', + required: true, + type: 'string' as NodePropertyTypes, + default: '', + }, + { + displayName: 'Port', + name: 'port', + required: true, + type: 'number' as NodePropertyTypes, + default: 22, + }, + { + displayName: 'Username', + name: 'username', + required: true, + type: 'string' as NodePropertyTypes, + default: '', + }, + { + displayName: 'Password', + name: 'password', + type: 'string' as NodePropertyTypes, + typeOptions: { + password: true, + }, + default: '', + }, + ]; +} diff --git a/packages/nodes-base/nodes/Ftp.node.ts b/packages/nodes-base/nodes/Ftp.node.ts new file mode 100644 index 0000000000..165484c0d0 --- /dev/null +++ b/packages/nodes-base/nodes/Ftp.node.ts @@ -0,0 +1,434 @@ +import { + BINARY_ENCODING, + IExecuteFunctions +} from 'n8n-core'; +import { + ICredentialDataDecryptedObject, + IDataObject, + INodeExecutionData, + INodeType, + INodeTypeDescription +} from 'n8n-workflow'; +import { + basename, + dirname, +} from 'path'; + +import * as ftpClient from 'promise-ftp'; +import * as sftpClient from 'ssh2-sftp-client'; + +export class Ftp implements INodeType { + description: INodeTypeDescription = { + displayName: 'FTP', + name: 'ftp', + icon: 'fa:server', + group: ['input'], + version: 1, + subtitle: '={{$parameter["protocol"] + ": " + $parameter["operation"]}}', + description: 'Transfers files via FTP or SFTP.', + defaults: { + name: 'FTP', + color: '#000000', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'ftp', + required: true, + displayOptions: { + show: { + protocol: [ + 'ftp', + ], + }, + }, + }, + { + name: 'sftp', + required: true, + displayOptions: { + show: { + protocol: [ + 'sftp', + ], + }, + }, + }, + ], + properties: [ + { + displayName: 'Protocol', + name: 'protocol', + type: 'options', + options: [ + { + name: 'FTP', + value: 'ftp', + }, + { + name: 'SFTP', + value: 'sftp', + }, + ], + default: 'ftp', + description: 'File transfer protocol.', + }, + { + displayName: 'Operation', + name: 'operation', + type: 'options', + options: [ + { + name: 'Download', + value: 'download', + description: 'Download a file.', + }, + { + name: 'List', + value: 'list', + description: 'List folder content.', + }, + { + name: 'Upload', + value: 'upload', + description: 'Upload a file.', + }, + ], + default: 'download', + description: 'Operation to perform.', + }, + + // ---------------------------------- + // download + // ---------------------------------- + { + displayName: 'Path', + displayOptions: { + show: { + operation: [ + 'download', + ], + }, + }, + name: 'path', + type: 'string', + default: '', + placeholder: '/documents/invoice.txt', + description: 'The file path of the file to download. Has to contain the full path.', + required: true, + }, + { + displayName: 'Binary Property', + displayOptions: { + show: { + operation: [ + 'download', + ], + }, + }, + name: 'binaryPropertyName', + type: 'string', + default: 'data', + description: 'Object property name which holds binary data.', + required: true, + }, + + // ---------------------------------- + // upload + // ---------------------------------- + { + displayName: 'Path', + displayOptions: { + show: { + operation: [ + 'upload', + ], + }, + }, + name: 'path', + type: 'string', + default: '', + description: 'The file path of the file to upload. Has to contain the full path.', + required: true, + }, + { + displayName: 'Binary Data', + displayOptions: { + show: { + operation: [ + 'upload', + ], + }, + }, + name: 'binaryData', + type: 'boolean', + default: true, + description: 'The text content of the file to upload.', + }, + { + displayName: 'Binary Property', + displayOptions: { + show: { + operation: [ + 'upload', + ], + binaryData: [ + true, + ] + }, + }, + name: 'binaryPropertyName', + type: 'string', + default: 'data', + description: 'Object property name which holds binary data.', + required: true, + }, + { + displayName: 'File Content', + displayOptions: { + show: { + operation: [ + 'upload', + ], + binaryData: [ + false, + ] + }, + }, + name: 'fileContent', + type: 'string', + default: '', + description: 'The text content of the file to upload.', + }, + + // ---------------------------------- + // list + // ---------------------------------- + { + displayName: 'Path', + displayOptions: { + show: { + operation: [ + 'list', + ], + }, + }, + name: 'path', + type: 'string', + default: '/', + description: 'Path of directory to list contents of.', + required: true, + }, + ], + }; + + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + // const returnData: IDataObject[] = []; + const returnItems: INodeExecutionData[] = []; + const qs: IDataObject = {}; + let responseData; + const operation = this.getNodeParameter('operation', 0) as string; + + let credentials: ICredentialDataDecryptedObject | undefined = undefined; + const protocol = this.getNodeParameter('protocol', 0) as string; + if (protocol === 'sftp') { + credentials = this.getCredentials('sftp'); + } else { + credentials = this.getCredentials('ftp'); + } + + if (credentials === undefined) { + throw new Error('Failed to get credentials!'); + } + + let ftp: ftpClient; + let sftp: sftpClient; + if (protocol === 'sftp') { + sftp = new sftpClient(); + + await sftp.connect({ + host: credentials.host as string, + port: credentials.port as number, + username: credentials.username as string, + password: credentials.password as string, + }); + + } else { + ftp = new ftpClient(); + + await ftp.connect({ + host: credentials.host as string, + port: credentials.port as number, + user: credentials.username as string, + password: credentials.password as string + }); + } + + for (let i = 0; i < items.length; i++) { + const newItem: INodeExecutionData = { + json: items[i].json, + binary: {}, + }; + + if (items[i].binary !== undefined) { + // Create a shallow copy of the binary data so that the old + // data references which do not get changed still stay behind + // but the incoming data does not get changed. + Object.assign(newItem.binary, items[i].binary); + } + + items[i] = newItem; + + if (protocol === 'sftp') { + const path = this.getNodeParameter('path', i) as string; + + if (operation === 'list') { + responseData = await sftp!.list(path); + returnItems.push.apply(returnItems, this.helpers.returnJsonArray(responseData as unknown as IDataObject[])); + } + + if (operation === 'download') { + responseData = await sftp!.get(path); + + const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i) as string; + + const filePathDownload = this.getNodeParameter('path', i) as string; + items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData(responseData as Buffer, filePathDownload); + + returnItems.push(items[i]); + } + + if (operation === 'upload') { + const remotePath = this.getNodeParameter('path', i) as string; + + // Check if dir path exists + const dirExists = await sftp!.exists(dirname(remotePath)); + + // If dir does not exist, create all recursively in path + if (!dirExists) { + // Separate filename from dir path + const fileName = basename(remotePath); + const dirPath = remotePath.replace(fileName, ''); + // Create directory + await sftp!.mkdir(dirPath, true); + } + + if (this.getNodeParameter('binaryData', i) === true) { + // Is binary file to upload + const item = items[i]; + + if (item.binary === undefined) { + throw new Error('No binary data exists on item!'); + } + + const propertyNameUpload = this.getNodeParameter('binaryPropertyName', i) as string; + + if (item.binary[propertyNameUpload] === undefined) { + throw new Error(`No binary data property "${propertyNameUpload}" does not exists on item!`); + } + + const buffer = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING) as Buffer; + await sftp!.put(buffer, remotePath); + } else { + // Is text file + const buffer = Buffer.from(this.getNodeParameter('fileContent', i) as string, 'utf8') as Buffer; + await sftp!.put(buffer, remotePath); + } + + returnItems.push(items[i]); + } + } + + if (protocol === 'ftp') { + + const path = this.getNodeParameter('path', i) as string; + + if (operation === 'list') { + responseData = await ftp!.list(path); + returnItems.push.apply(returnItems, this.helpers.returnJsonArray(responseData as unknown as IDataObject[])); + } + + if (operation === 'download') { + responseData = await ftp!.get(path); + + // Convert readable stream to buffer so that can be displayed properly + const chunks = []; + for await (const chunk of responseData) { + chunks.push(chunk); + } + + // @ts-ignore + responseData = Buffer.concat(chunks); + + const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i) as string; + + const filePathDownload = this.getNodeParameter('path', i) as string; + items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData(responseData, filePathDownload); + + returnItems.push(items[i]); + } + + if (operation === 'upload') { + const remotePath = this.getNodeParameter('path', i) as string; + const fileName = basename(remotePath); + const dirPath = remotePath.replace(fileName, ''); + + if (this.getNodeParameter('binaryData', i) === true) { + // Is binary file to upload + const item = items[i]; + + if (item.binary === undefined) { + throw new Error('No binary data exists on item!'); + } + + const propertyNameUpload = this.getNodeParameter('binaryPropertyName', i) as string; + + if (item.binary[propertyNameUpload] === undefined) { + throw new Error(`No binary data property "${propertyNameUpload}" does not exists on item!`); + } + + const buffer = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING) as Buffer; + + try { + await ftp!.put(buffer, remotePath); + } catch (error) { + if (error.code === 553) { + // Create directory + await ftp!.mkdir(dirPath, true); + await ftp!.put(buffer, remotePath); + } else { + throw new Error(error); + } + } + } else { + // Is text file + const buffer = Buffer.from(this.getNodeParameter('fileContent', i) as string, 'utf8') as Buffer; + try { + await ftp!.put(buffer, remotePath); + } catch (error) { + if (error.code === 553) { + // Create directory + await ftp!.mkdir(dirPath, true); + await ftp!.put(buffer, remotePath); + } else { + throw new Error(error); + } + } + } + returnItems.push(items[i]); + } + } + } + + if (protocol === 'sftp') { + await sftp!.end(); + } else { + await ftp!.end(); + } + + return [returnItems]; + } +} diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index bca833fb84..1a0c4905c2 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -60,6 +60,7 @@ "dist/credentials/FreshdeskApi.credentials.js", "dist/credentials/FileMaker.credentials.js", "dist/credentials/FlowApi.credentials.js", + "dist/credentials/Ftp.credentials.js", "dist/credentials/GithubApi.credentials.js", "dist/credentials/GithubOAuth2Api.credentials.js", "dist/credentials/GitlabApi.credentials.js", @@ -133,6 +134,7 @@ "dist/credentials/StripeApi.credentials.js", "dist/credentials/SalesmateApi.credentials.js", "dist/credentials/SegmentApi.credentials.js", + "dist/credentials/Sftp.credentials.js", "dist/credentials/Signl4Api.credentials.js", "dist/credentials/SpotifyOAuth2Api.credentials.js", "dist/credentials/SurveyMonkeyApi.credentials.js", @@ -209,6 +211,7 @@ "dist/nodes/ExecuteWorkflow.node.js", "dist/nodes/Facebook/FacebookGraphApi.node.js", "dist/nodes/FileMaker/FileMaker.node.js", + "dist/nodes/Ftp.node.js", "dist/nodes/Freshdesk/Freshdesk.node.js", "dist/nodes/Flow/Flow.node.js", "dist/nodes/Flow/FlowTrigger.node.js", @@ -347,6 +350,7 @@ "@types/nodemailer": "^6.4.0", "@types/redis": "^2.8.11", "@types/request-promise-native": "~1.0.15", + "@types/ssh2-sftp-client": "^5.1.0", "@types/uuid": "^3.4.6", "@types/xml2js": "^0.4.3", "gulp": "^4.0.0", @@ -357,6 +361,7 @@ "typescript": "~3.7.4" }, "dependencies": { + "@types/promise-ftp": "^1.3.4", "aws4": "^1.8.0", "basic-auth": "^2.0.1", "change-case": "^4.1.1", @@ -382,10 +387,12 @@ "pdf-parse": "^1.1.1", "pg": "^8.3.0", "pg-promise": "^10.5.8", + "promise-ftp": "^1.3.5", "redis": "^2.8.0", "request": "^2.88.2", "rhea": "^1.0.11", "rss-parser": "^3.7.0", + "ssh2-sftp-client": "^5.2.1", "uuid": "^3.4.0", "vm2": "^3.6.10", "xlsx": "^0.14.3",