✨ Separate webhooks from core (#1408)

* Unify execution ID across executions

* Fix indentation and improved comments

* WIP: saving data after each node execution

* Added on/off to save data after each step, saving initial data and retries working

* Fixing lint issues

* Fixing more lint issues

* ✨ Add bull to execute workflows

* 👕 Fix lint issue

* ⚡ Add graceful shutdown to worker

* ⚡ Add loading staticData to worker

* 👕 Fix lint issue

* ⚡ Fix import

* Changed tables metadata to add nullable to stoppedAt

* Reload database on migration run

* Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration

* Added checks to Redis and exiting process if connection is unavailable

* Fixing error with new installations

* Fix issue with data not being sent back to browser on manual executions with defined destination

* Merging bull and unify execution id branch fixes

* Main process will now get execution success from database instead of redis

* Omit execution duration if execution did not stop

* Fix issue with execution list displaying inconsistant information information while a workflow is running

* Remove unused hooks to clarify for developers that these wont run in queue mode

* Added active pooling to help recover from Redis crashes

* Lint issues

* Changing default polling interval to 60 seconds

* Removed unnecessary attributes from bull job

* Added webhooks service and setting to disable webhooks from main process

* Fixed executions list when running with queues. Now we get the list of actively running workflows from bull.

* Add option to disable deregistration of webhooks on shutdown

* Rename WEBHOOK_TUNNEL_URL to WEBHOOK_URL keeping backwards compat.

* Added auto refresh to executions list

* Improvements to workflow stop process when running with queues

* Refactor queue system to use a singleton and avoid code duplication

* Improve comments and remove unnecessary commits

* Remove console.log from vue file

* Blocking webhook process to run without queues

* Handling execution stop graciously when possible

* Removing initialization of all workflows from webhook process

* Refactoring code to remove code duplication for job stop

* Improved execution list to be more fluid and less intrusive

* Fixing workflow name for current executions when auto updating

* ⚡ Right align autorefresh checkbox

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>

packages/editor-ui/src/Interface.ts

@@ -125,7 +125,7 @@ export interface IRestApi {
 	getActiveWorkflows(): Promise<string[]>;
 	getActivationError(id: string): Promise<IActivationError | undefined >;
 	getCurrentExecutions(filter: object): Promise<IExecutionsCurrentSummaryExtended[]>;
-	getPastExecutions(filter: object, limit: number, lastId?: string | number): Promise<IExecutionsListResponse>;
+	getPastExecutions(filter: object, limit: number, lastId?: string | number, firstId?: string | number): Promise<IExecutionsListResponse>;
 	stopCurrentExecution(executionId: string): Promise<IExecutionsStopData>;
 	makeRestApiRequest(method: string, endpoint: string, data?: any): Promise<any>; // tslint:disable-line:no-any
 	getSettings(): Promise<IN8nUISettings>;

packages/editor-ui/src/components/ExecutionsList.vue

@@ -28,7 +28,10 @@
 							</el-option>
 						</el-select>
 					</el-col>
-					<el-col :span="8">&nbsp;
+					<el-col :span="4">&nbsp;
+					</el-col>
+					<el-col :span="4" class="autorefresh">
+						<el-checkbox v-model="autoRefresh" @change="handleAutoRefreshToggle">Auto refresh</el-checkbox>
 					</el-col>
 				</el-row>
 			</div>
@@ -191,6 +194,8 @@ export default mixins(
 			finishedExecutionsCount: 0,
 
 			checkAll: false,
+			autoRefresh: true,
+			autoRefreshInterval: undefined as undefined | NodeJS.Timer,
 
 			filter: {
 				status: 'ALL',
@@ -292,6 +297,10 @@ export default mixins(
 			// Handle the close externally as the visible parameter is an external prop
 			// and is so not allowed to be changed here.
 			this.$emit('closeDialog');
+			if (this.autoRefreshInterval) {
+				clearInterval(this.autoRefreshInterval);
+				this.autoRefreshInterval = undefined;
+			}
 			return false;
 		},
 		displayExecution (execution: IExecutionShortResponse) {
@@ -301,6 +310,18 @@ export default mixins(
 			});
 			this.closeDialog();
 		},
+		handleAutoRefreshToggle () {
+			if (this.autoRefreshInterval) {
+				// Clear any previously existing intervals (if any - there shouldn't)
+				clearInterval(this.autoRefreshInterval);
+				this.autoRefreshInterval = undefined;
+			}
+
+
+			if (this.autoRefresh) {
+				this.autoRefreshInterval = setInterval(this.loadAutoRefresh, 4 * 1000); // refresh data every 4 secs
+			}
+		},
 		handleCheckAllChange () {
 			if (this.checkAll === false) {
 				Vue.set(this, 'selectedItems', {});
@@ -389,6 +410,27 @@ export default mixins(
 
 			this.$store.commit('setActiveExecutions', activeExecutions);
 		},
+		async loadAutoRefresh () : Promise<void> {
+			let firstId: string | number | undefined = 0;
+			if (this.finishedExecutions.length !== 0) {
+				firstId = this.finishedExecutions[0].id;
+			}
+			const activeExecutionsPromise: Promise<IExecutionsListResponse> = this.restApi().getPastExecutions({}, 100, undefined, firstId);
+			const currentExecutionsPromise: Promise<IExecutionsCurrentSummaryExtended[]> = this.restApi().getCurrentExecutions({});
+
+			const results = await Promise.all([activeExecutionsPromise, currentExecutionsPromise]);
+
+			for (const activeExecution of results[1]) {
+				if (activeExecution.workflowId !== undefined && activeExecution.workflowName === undefined) {
+					activeExecution.workflowName = this.getWorkflowName(activeExecution.workflowId);
+				}
+			}
+
+			this.$store.commit('setActiveExecutions', results[1]);
+
+			this.finishedExecutions.unshift.apply(this.finishedExecutions, results[0].results);
+			this.finishedExecutionsCount = results[0].count;
+		},
 		async loadFinishedExecutions (): Promise<void> {
 			if (this.filter.status === 'running') {
 				this.finishedExecutions = [];
@@ -459,6 +501,7 @@ export default mixins(
 
 			await this.loadWorkflows();
 			await this.refreshData();
+			this.handleAutoRefreshToggle();
 		},
 		async retryExecution (execution: IExecutionShortResponse, loadWorkflow?: boolean) {
 			this.isDataLoading = true;
@@ -544,6 +587,11 @@ export default mixins(
 
 <style scoped lang="scss">
 
+.autorefresh {
+	padding-right: 0.5em;
+	text-align: right;
+}
+
 .filters {
 	line-height: 2em;
 	.refresh-button {

packages/editor-ui/src/components/mixins/restApi.ts

@@ -300,11 +300,12 @@ export const restApi = Vue.extend({
 
 				// Returns all saved executions
 				// TODO: For sure needs some kind of default filter like last day, with max 10 results, ...
-				getPastExecutions: (filter: object, limit: number, lastId?: string | number): Promise<IExecutionsListResponse> => {
+				getPastExecutions: (filter: object, limit: number, lastId?: string | number, firstId?: string | number): Promise<IExecutionsListResponse> => {
 					let sendData = {};
 					if (filter) {
 						sendData = {
 							filter,
+							firstId,
 							lastId,
 							limit,
 						};

packages/editor-ui/src/views/NodeView.vue

@@ -147,6 +147,7 @@ import {
 	NodeInputConnections,
 	NodeHelpers,
 	Workflow,
+	IRun,
 } from 'n8n-workflow';
 import {
 	IConnectionsUi,
@@ -161,6 +162,7 @@ import {
 	IUpdateInformation,
 	IWorkflowDataUpdate,
 	XYPositon,
+	IPushDataExecutionFinished,
 } from '../Interface';
 
 export default mixins(
@@ -728,7 +730,37 @@ export default mixins(
 						type: 'success',
 					});
 				} catch (error) {
-					this.$showError(error, 'Problem stopping execution', 'There was a problem stopping the execuction:');
+					// Execution stop might fail when the execution has already finished. Let's treat this here.
+					const execution = await this.restApi().getExecution(executionId) as IExecutionResponse;
+					if (execution.finished) {
+						const executedData = {
+							data: execution.data,
+							finished: execution.finished,
+							mode: execution.mode,
+							startedAt: execution.startedAt,
+							stoppedAt: execution.stoppedAt,
+						} as IRun;
+						const pushData = {
+							data: executedData,
+							executionIdActive: executionId,
+							executionIdDb: executionId,
+							retryOf: execution.retryOf,
+						} as IPushDataExecutionFinished;
+						this.$store.commit('finishActiveExecution', pushData);
+						this.$titleSet(execution.workflowData.name, 'IDLE');
+						this.$store.commit('setExecutingNode', null);
+						this.$store.commit('setWorkflowExecutionData', executedData);
+						this.$store.commit('removeActiveAction', 'workflowRunning');
+						this.$showMessage({
+							title: 'Workflow finished executing',
+							message: 'Unable to stop operation in time. Workflow finished executing already.',
+							type: 'success',
+						});
+					} else {
+						this.$showError(error, 'Problem stopping execution', 'There was a problem stopping the execuction:');
+					}
+
+					
 				}
 				this.stopExecutionInProgress = false;
 			},