Merge branch 'master' into static-stateless-webhooks

packages/cli/src/Server.ts

@@ -5,6 +5,7 @@ import {
 import {
 	dirname as pathDirname,
 	join as pathJoin,
+	resolve as pathResolve,
 } from 'path';
 import {
 	getConnectionManager,
@@ -12,13 +13,21 @@ import {
 import * as bodyParser from 'body-parser';
 require('body-parser-xml')(bodyParser);
 import * as history from 'connect-history-api-fallback';
-import * as requestPromise from 'request-promise-native';
+import * as _ from 'lodash';
+import * as clientOAuth2 from 'client-oauth2';
+import * as clientOAuth1 from 'oauth-1.0a';
+import { RequestOptions } from 'oauth-1.0a';
+import * as csrf from 'csrf';
+import * as requestPromise  from 'request-promise-native';
+import { createHmac } from 'crypto';
 
 import {
 	ActiveExecutions,
 	ActiveWorkflowRunner,
+	CredentialsHelper,
 	CredentialTypes,
 	Db,
+	ExternalHooks,
 	IActivationError,
 	ICustomRequest,
 	ICredentialsDb,
@@ -33,6 +42,7 @@ import {
 	IExecutionsListResponse,
 	IExecutionsStopData,
 	IExecutionsSummary,
+	IExternalHooksClass,
 	IN8nUISettings,
 	IPackageVersions,
 	IWorkflowBase,
@@ -57,6 +67,7 @@ import {
 } from 'n8n-core';
 
 import {
+	ICredentialsEncrypted,
 	ICredentialType,
 	IDataObject,
 	INodeCredentials,
@@ -64,6 +75,7 @@ import {
 	INodeParameters,
 	INodePropertyOptions,
 	IRunData,
+	IWorkflowCredentials,
 	Workflow,
 } from 'n8n-workflow';
 
@@ -83,7 +95,8 @@ import * as jwks from 'jwks-rsa';
 // @ts-ignore
 import * as timezones from 'google-timezones-json';
 import * as parseUrl from 'parseurl';
-
+import * as querystring from 'querystring';
+import { OptionsWithUrl } from 'request-promise-native';
 
 class App {
 
@@ -92,6 +105,7 @@ class App {
 	testWebhooks: TestWebhooks.TestWebhooks;
 	endpointWebhook: string;
 	endpointWebhookTest: string;
+	externalHooks: IExternalHooksClass;
 	saveDataErrorExecution: string;
 	saveDataSuccessExecution: string;
 	saveManualExecutions: boolean;
@@ -123,6 +137,8 @@ class App {
 		this.protocol = config.get('protocol');
 		this.sslKey  = config.get('ssl_key');
 		this.sslCert = config.get('ssl_cert');
+
+		this.externalHooks = ExternalHooks();
 	}
 
 
@@ -340,7 +356,7 @@ class App {
 		// Creates a new workflow
 		this.app.post('/rest/workflows', ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IWorkflowResponse> => {
 
-			const newWorkflowData = req.body;
+			const newWorkflowData = req.body as IWorkflowBase;
 
 			newWorkflowData.name = newWorkflowData.name.trim();
 			newWorkflowData.createdAt = this.getCurrentDate();
@@ -348,6 +364,8 @@ class App {
 
 			newWorkflowData.id = undefined;
 
+			await this.externalHooks.run('workflow.create', [newWorkflowData]);
+
 			// Save the workflow in DB
 			const result = await Db.collections.Workflow!.save(newWorkflowData);
 
@@ -423,9 +441,11 @@ class App {
 		// Updates an existing workflow
 		this.app.patch('/rest/workflows/:id', ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<IWorkflowResponse> => {
 
-			const newWorkflowData = req.body;
+			const newWorkflowData = req.body as IWorkflowBase;
 			const id = req.params.id;
 
+            await this.externalHooks.run('workflow.update', [newWorkflowData]);
+
 			const isActive = await this.activeWorkflowRunner.isActive(id);
 
 			if (isActive) {
@@ -469,6 +489,8 @@ class App {
 			if (responseData.active === true) {
 				// When the workflow is supposed to be active add it again
 				try {
+					await this.externalHooks.run('workflow.activate', [responseData]);
+
 					await this.activeWorkflowRunner.add(id);
 				} catch (error) {
 					// If workflow could not be activated set it again to inactive
@@ -493,6 +515,8 @@ class App {
 		this.app.delete('/rest/workflows/:id', ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<boolean> => {
 			const id = req.params.id;
 
+            await this.externalHooks.run('workflow.delete', [id]);
+
 			const isActive = await this.activeWorkflowRunner.isActive(id);
 
 			if (isActive) {
@@ -567,7 +591,7 @@ class App {
 
 			const nodeTypes = NodeTypes();
 
-			const loadDataInstance = new LoadNodeParameterOptions(nodeType, nodeTypes, credentials);
+			const loadDataInstance = new LoadNodeParameterOptions(nodeType, nodeTypes, JSON.parse('' + req.query.currentNodeParameters), credentials!);
 
 			const workflowData = loadDataInstance.getWorkflowData() as IWorkflowBase;
 			const workflowCredentials = await WorkflowCredentials(workflowData.nodes);
@@ -601,8 +625,8 @@ class App {
 
 
 		// Returns the node icon
-		this.app.get('/rest/node-icon/:nodeType', async (req: express.Request, res: express.Response): Promise<void> => {
-			const nodeTypeName = req.params.nodeType;
+		this.app.get(['/rest/node-icon/:nodeType', '/rest/node-icon/:scope/:nodeType'], async (req: express.Request, res: express.Response): Promise<void> => {
+			const nodeTypeName = `${req.params.scope ? `${req.params.scope}/` : ''}${req.params.nodeType}`;
 
 			const nodeTypes = NodeTypes();
 			const nodeType = nodeTypes.getByName(nodeTypeName);
@@ -658,6 +682,8 @@ class App {
 		this.app.delete('/rest/credentials/:id', ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<boolean> => {
 			const id = req.params.id;
 
+			await this.externalHooks.run('credentials.delete', [id]);
+
 			await Db.collections.Credentials!.delete({ id });
 
 			return true;
@@ -667,6 +693,10 @@ class App {
 		this.app.post('/rest/credentials', ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<ICredentialsResponse> => {
 			const incomingData = req.body;
 
+			if (!incomingData.name || incomingData.name.length < 3) {
+				throw new ResponseHelper.ResponseError(`Credentials name must be at least 3 characters long.`, undefined, 400);
+			}
+
 			// Add the added date for node access permissions
 			for (const nodeAccess of incomingData.nodesAccess) {
 				nodeAccess.date = this.getCurrentDate();
@@ -699,6 +729,8 @@ class App {
 			credentials.setData(incomingData.data, encryptionKey);
 			const newCredentialsData = credentials.getDataToSave() as ICredentialsDb;
 
+			await this.externalHooks.run('credentials.create', [newCredentialsData]);
+
 			// Add special database related data
 			newCredentialsData.createdAt = this.getCurrentDate();
 			newCredentialsData.updatedAt = this.getCurrentDate();
@@ -707,6 +739,7 @@ class App {
 
 			// Save the credentials in DB
 			const result = await Db.collections.Credentials!.save(newCredentialsData);
+			result.data = incomingData.data;
 
 			// Convert to response format in which the id is a string
 			(result as unknown as ICredentialsResponse).id = result.id.toString();
@@ -750,6 +783,21 @@ class App {
 				throw new Error('No encryption key got found to encrypt the credentials!');
 			}
 
+			// Load the currently saved credentials to be able to persist some of the data if
+			const result = await Db.collections.Credentials!.findOne(id);
+			if (result === undefined) {
+				throw new ResponseHelper.ResponseError(`Credentials with the id "${id}" do not exist.`, undefined, 400);
+			}
+
+			const currentlySavedCredentials = new Credentials(result.name, result.type, result.nodesAccess, result.data);
+			const decryptedData = currentlySavedCredentials.getData(encryptionKey!);
+
+			// Do not overwrite the oauth data else data like the access or refresh token would get lost
+			// everytime anybody changes anything on the credentials even if it is just the name.
+			if (decryptedData.oauthTokenData) {
+				incomingData.data.oauthTokenData = decryptedData.oauthTokenData;
+			}
+
 			// Encrypt the data
 			const credentials = new Credentials(incomingData.name, incomingData.type, incomingData.nodesAccess);
 			credentials.setData(incomingData.data, encryptionKey);
@@ -758,6 +806,8 @@ class App {
 			// Add special database related data
 			newCredentialsData.updatedAt = this.getCurrentDate();
 
+			await this.externalHooks.run('credentials.update', [newCredentialsData]);
+
 			// Update the credentials in DB
 			await Db.collections.Credentials!.update(id, newCredentialsData);
 
@@ -869,6 +919,331 @@ class App {
 			return returnData;
 		}));
 
+		// ----------------------------------------
+		// OAuth1-Credential/Auth
+		// ----------------------------------------
+
+		// Authorize OAuth Data
+		this.app.get('/rest/oauth1-credential/auth', ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<string> => {
+			if (req.query.id === undefined) {
+				throw new Error('Required credential id is missing!');
+			}
+
+			const result = await Db.collections.Credentials!.findOne(req.query.id as string);
+			if (result === undefined) {
+				res.status(404).send('The credential is not known.');
+				return '';
+			}
+
+			let encryptionKey = undefined;
+			encryptionKey = await UserSettings.getEncryptionKey();
+			if (encryptionKey === undefined) {
+				throw new Error('No encryption key got found to decrypt the credentials!');
+			}
+
+			// Decrypt the currently saved credentials
+			const workflowCredentials: IWorkflowCredentials = {
+				[result.type as string]: {
+					[result.name as string]: result as ICredentialsEncrypted,
+				},
+			};
+			const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey);
+			const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, true);
+			const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type);
+
+			const signatureMethod = _.get(oauthCredentials, 'signatureMethod') as string;
+
+			const oauth = new clientOAuth1({
+				consumer: {
+					key: _.get(oauthCredentials, 'consumerKey') as string,
+					secret: _.get(oauthCredentials, 'consumerSecret') as string,
+				},
+				signature_method: signatureMethod,
+				hash_function(base, key) {
+					const algorithm = (signatureMethod === 'HMAC-SHA1') ? 'sha1' : 'sha256';
+					return createHmac(algorithm, key)
+							.update(base)
+							.digest('base64');
+				},
+			});
+
+			const callback = `${WebhookHelpers.getWebhookBaseUrl()}rest/oauth1-credential/callback?cid=${req.query.id}`;
+
+			const options: RequestOptions  = {
+				method: 'POST',
+				url: (_.get(oauthCredentials, 'requestTokenUrl') as string),
+				data: {
+					oauth_callback: callback,
+				},
+			};
+
+			const data = oauth.toHeader(oauth.authorize(options as RequestOptions));
+
+			//@ts-ignore
+			options.headers = data;
+
+			const response = await requestPromise(options);
+
+			// Response comes as x-www-form-urlencoded string so convert it to JSON
+
+			const responseJson = querystring.parse(response);
+
+			const returnUri = `${_.get(oauthCredentials, 'authUrl')}?oauth_token=${responseJson.oauth_token}`;
+
+			// Encrypt the data
+			const credentials = new Credentials(result.name, result.type, result.nodesAccess);
+
+			credentials.setData(decryptedDataOriginal, encryptionKey);
+			const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
+
+			// Add special database related data
+			newCredentialsData.updatedAt = this.getCurrentDate();
+
+			// Update the credentials in DB
+			await Db.collections.Credentials!.update(req.query.id as string, newCredentialsData);
+
+			return returnUri;
+		}));
+
+		// Verify and store app code. Generate access tokens and store for respective credential.
+		this.app.get('/rest/oauth1-credential/callback', async (req: express.Request, res: express.Response) => {
+			const { oauth_verifier, oauth_token, cid } = req.query;
+
+			if (oauth_verifier === undefined || oauth_token === undefined) {
+				throw new Error('Insufficient parameters for OAuth1 callback');
+			}
+
+			const result = await Db.collections.Credentials!.findOne(cid as any); // tslint:disable-line:no-any
+			if (result === undefined) {
+				const errorResponse = new ResponseHelper.ResponseError('The credential is not known.', undefined, 404);
+				return ResponseHelper.sendErrorResponse(res, errorResponse);
+			}
+
+			let encryptionKey = undefined;
+			encryptionKey = await UserSettings.getEncryptionKey();
+			if (encryptionKey === undefined) {
+				const errorResponse = new ResponseHelper.ResponseError('No encryption key got found to decrypt the credentials!', undefined, 503);
+				return ResponseHelper.sendErrorResponse(res, errorResponse);
+			}
+
+			// Decrypt the currently saved credentials
+			const workflowCredentials: IWorkflowCredentials = {
+				[result.type as string]: {
+					[result.name as string]: result as ICredentialsEncrypted,
+				},
+			};
+			const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey);
+			const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, true);
+			const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type);
+
+			const options: OptionsWithUrl  = {
+				method: 'POST',
+				url: _.get(oauthCredentials, 'accessTokenUrl') as string,
+				qs: {
+					oauth_token,
+					oauth_verifier,
+				}
+			};
+
+			let oauthToken;
+
+			try {
+				oauthToken = await requestPromise(options);
+			} catch (error) {
+				const errorResponse = new ResponseHelper.ResponseError('Unable to get access tokens!', undefined, 404);
+				return ResponseHelper.sendErrorResponse(res, errorResponse);
+			}
+
+			// Response comes as x-www-form-urlencoded string so convert it to JSON
+
+			const oauthTokenJson = querystring.parse(oauthToken);
+
+			decryptedDataOriginal.oauthTokenData = oauthTokenJson;
+
+			const credentials = new Credentials(result.name, result.type, result.nodesAccess);
+			credentials.setData(decryptedDataOriginal, encryptionKey);
+			const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
+			// Add special database related data
+			newCredentialsData.updatedAt = this.getCurrentDate();
+			// Save the credentials in DB
+			await Db.collections.Credentials!.update(cid as any, newCredentialsData); // tslint:disable-line:no-any
+
+			res.sendFile(pathResolve(__dirname, '../../templates/oauth-callback.html'));
+		});
+
+
+		// ----------------------------------------
+		// OAuth2-Credential/Auth
+		// ----------------------------------------
+
+
+		// Authorize OAuth Data
+		this.app.get('/rest/oauth2-credential/auth', ResponseHelper.send(async (req: express.Request, res: express.Response): Promise<string> => {
+			if (req.query.id === undefined) {
+				throw new Error('Required credential id is missing!');
+			}
+
+			const result = await Db.collections.Credentials!.findOne(req.query.id as string);
+			if (result === undefined) {
+				res.status(404).send('The credential is not known.');
+				return '';
+			}
+
+			let encryptionKey = undefined;
+			encryptionKey = await UserSettings.getEncryptionKey();
+			if (encryptionKey === undefined) {
+				throw new Error('No encryption key got found to decrypt the credentials!');
+			}
+
+			// Decrypt the currently saved credentials
+			const workflowCredentials: IWorkflowCredentials = {
+				[result.type as string]: {
+					[result.name as string]: result as ICredentialsEncrypted,
+				},
+			};
+			const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey);
+			const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, true);
+			const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type);
+
+			const token = new csrf();
+			// Generate a CSRF prevention token and send it as a OAuth2 state stringma/ERR
+			const csrfSecret = token.secretSync();
+			const state = {
+				token: token.create(csrfSecret),
+				cid: req.query.id
+			};
+			const stateEncodedStr = Buffer.from(JSON.stringify(state)).toString('base64') as string;
+
+			const oAuthObj = new clientOAuth2({
+				clientId: _.get(oauthCredentials, 'clientId') as string,
+				clientSecret: _.get(oauthCredentials, 'clientSecret', '') as string,
+				accessTokenUri: _.get(oauthCredentials, 'accessTokenUrl', '') as string,
+				authorizationUri: _.get(oauthCredentials, 'authUrl', '') as string,
+				redirectUri: `${WebhookHelpers.getWebhookBaseUrl()}rest/oauth2-credential/callback`,
+				scopes: _.split(_.get(oauthCredentials, 'scope', 'openid,') as string, ','),
+				state: stateEncodedStr,
+			});
+
+			// Encrypt the data
+			const credentials = new Credentials(result.name, result.type, result.nodesAccess);
+			decryptedDataOriginal.csrfSecret = csrfSecret;
+
+			credentials.setData(decryptedDataOriginal, encryptionKey);
+			const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
+
+			// Add special database related data
+			newCredentialsData.updatedAt = this.getCurrentDate();
+
+			// Update the credentials in DB
+			await Db.collections.Credentials!.update(req.query.id as string, newCredentialsData);
+
+			const authQueryParameters = _.get(oauthCredentials, 'authQueryParameters', '') as string;
+			let returnUri = oAuthObj.code.getUri();
+
+			if (authQueryParameters) {
+				returnUri += '&' + authQueryParameters;
+			}
+
+			return returnUri;
+		}));
+
+		// ----------------------------------------
+		// OAuth2-Credential/Callback
+		// ----------------------------------------
+
+		// Verify and store app code. Generate access tokens and store for respective credential.
+		this.app.get('/rest/oauth2-credential/callback', async (req: express.Request, res: express.Response) => {
+			const {code, state: stateEncoded } = req.query;
+
+			if (code === undefined || stateEncoded === undefined) {
+				throw new Error('Insufficient parameters for OAuth2 callback');
+			}
+
+			let state;
+			try {
+				state = JSON.parse(Buffer.from(stateEncoded as string, 'base64').toString());
+			} catch (error) {
+				const errorResponse = new ResponseHelper.ResponseError('Invalid state format returned', undefined, 503);
+				return ResponseHelper.sendErrorResponse(res, errorResponse);
+			}
+
+			const result = await Db.collections.Credentials!.findOne(state.cid);
+			if (result === undefined) {
+				const errorResponse = new ResponseHelper.ResponseError('The credential is not known.', undefined, 404);
+				return ResponseHelper.sendErrorResponse(res, errorResponse);
+			}
+
+			let encryptionKey = undefined;
+			encryptionKey = await UserSettings.getEncryptionKey();
+			if (encryptionKey === undefined) {
+				const errorResponse = new ResponseHelper.ResponseError('No encryption key got found to decrypt the credentials!', undefined, 503);
+				return ResponseHelper.sendErrorResponse(res, errorResponse);
+			}
+
+			// Decrypt the currently saved credentials
+			const workflowCredentials: IWorkflowCredentials = {
+				[result.type as string]: {
+					[result.name as string]: result as ICredentialsEncrypted,
+				},
+			};
+			const credentialsHelper = new CredentialsHelper(workflowCredentials, encryptionKey);
+			const decryptedDataOriginal = credentialsHelper.getDecrypted(result.name, result.type, true);
+			const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites(decryptedDataOriginal, result.type);
+
+			const token = new csrf();
+			if (decryptedDataOriginal.csrfSecret === undefined || !token.verify(decryptedDataOriginal.csrfSecret as string, state.token)) {
+				const errorResponse = new ResponseHelper.ResponseError('The OAuth2 callback state is invalid!', undefined, 404);
+				return ResponseHelper.sendErrorResponse(res, errorResponse);
+			}
+
+			let options = {};
+
+			if (_.get(oauthCredentials, 'authentication', 'header') as string === 'body') {
+				options = {
+					body: {
+						client_id: _.get(oauthCredentials, 'clientId') as string,
+						client_secret: _.get(oauthCredentials, 'clientSecret', '') as string,
+					},
+				};
+			}
+
+			const oAuthObj = new clientOAuth2({
+				clientId: _.get(oauthCredentials, 'clientId') as string,
+				clientSecret: _.get(oauthCredentials, 'clientSecret', '') as string,
+				accessTokenUri: _.get(oauthCredentials, 'accessTokenUrl', '') as string,
+				authorizationUri: _.get(oauthCredentials, 'authUrl', '') as string,
+				redirectUri: `${WebhookHelpers.getWebhookBaseUrl()}rest/oauth2-credential/callback`,
+				scopes: _.split(_.get(oauthCredentials, 'scope', 'openid,') as string, ',')
+			});
+
+			const oauthToken = await oAuthObj.code.getToken(req.originalUrl, options);
+
+			if (oauthToken === undefined) {
+				const errorResponse = new ResponseHelper.ResponseError('Unable to get access tokens!', undefined, 404);
+				return ResponseHelper.sendErrorResponse(res, errorResponse);
+			}
+
+			if (decryptedDataOriginal.oauthTokenData) {
+				// Only overwrite supplied data as some providers do for example just return the
+				// refresh_token on the very first request and not on subsequent ones.
+				Object.assign(decryptedDataOriginal.oauthTokenData, oauthToken.data);
+			} else {
+				// No data exists so simply set
+				decryptedDataOriginal.oauthTokenData = oauthToken.data;
+			}
+
+			_.unset(decryptedDataOriginal, 'csrfSecret');
+
+			const credentials = new Credentials(result.name, result.type, result.nodesAccess);
+			credentials.setData(decryptedDataOriginal, encryptionKey);
+			const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb;
+			// Add special database related data
+			newCredentialsData.updatedAt = this.getCurrentDate();
+			// Save the credentials in DB
+			await Db.collections.Credentials!.update(state.cid, newCredentialsData);
+
+			res.sendFile(pathResolve(__dirname, '../../templates/oauth-callback.html'));
+		});
 
 
 		// ----------------------------------------
@@ -1299,6 +1674,7 @@ class App {
 
 export async function start(): Promise<void> {
 	const PORT = config.get('port');
+	const ADDRESS = config.get('listen_address');
 
 	const app = new App();
 
@@ -1317,9 +1693,9 @@ export async function start(): Promise<void> {
 		server = http.createServer(app.app);
 	}
 
-	server.listen(PORT, async () => {
+	server.listen(PORT, ADDRESS, async () => {
 		const versions = await GenericHelpers.getVersions();
-		console.log(`n8n ready on port ${PORT}`);
+		console.log(`n8n ready on ${ADDRESS}, port ${PORT}`);
 		console.log(`Version: ${versions.cli}`);
 	});
 }