Unify execution id + Queue system (#1340)

* Unify execution ID across executions

* Fix indentation and improved comments

* WIP: saving data after each node execution

* Added on/off to save data after each step, saving initial data and retries working

* Fixing lint issues

* Fixing more lint issues

*  Add bull to execute workflows

* 👕 Fix lint issue

*  Add graceful shutdown to worker

*  Add loading staticData to worker

* 👕 Fix lint issue

*  Fix import

* Changed tables metadata to add nullable to stoppedAt

* Reload database on migration run

* Fixed reloading database schema for sqlite by reconnecting and fixing postgres migration

* Added checks to Redis and exiting process if connection is unavailable

* Fixing error with new installations

* Fix issue with data not being sent back to browser on manual executions with defined destination

* Merging bull and unify execution id branch fixes

* Main process will now get execution success from database instead of redis

* Omit execution duration if execution did not stop

* Fix issue with execution list displaying inconsistant information information while a workflow is running

* Remove unused hooks to clarify for developers that these wont run in queue mode

* Added active pooling to help recover from Redis crashes

* Lint issues

* Changing default polling interval to 60 seconds

* Removed unnecessary attributes from bull job

*  Improved output on worker job start

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
This commit is contained in:
Omar Ajoue
2021-02-09 08:59:32 +01:00
committed by GitHub
parent 9c67c893e7
commit 7a3aaf8a24
26 changed files with 952 additions and 60 deletions

View File

@@ -97,6 +97,25 @@
</el-select>
</el-col>
</el-row>
<el-row>
<el-col :span="10" class="setting-name">
Save Execution Progress:
<el-tooltip class="setting-info" placement="top" effect="light">
<div slot="content" v-html="helpTexts.saveExecutionProgress"></div>
<font-awesome-icon icon="question-circle" />
</el-tooltip>
</el-col>
<el-col :span="14" class="ignore-key-press">
<el-select v-model="workflowSettings.saveExecutionProgress" placeholder="Select Option" size="small" filterable>
<el-option
v-for="option of saveExecutionProgressOptions"
:key="option.key"
:label="option.value"
:value="option.key">
</el-option>
</el-select>
</el-col>
</el-row>
<el-row>
<el-col :span="10" class="setting-name">
Timeout Workflow:
@@ -135,7 +154,6 @@
</el-col>
</el-row>
</div>
<div class="action-buttons">
<el-button type="success" @click="saveSettings">
Save
@@ -178,6 +196,7 @@ export default mixins(
timezone: 'The timezone in which the workflow should run. Gets for example used by "Cron" node.',
saveDataErrorExecution: 'If data data of executions should be saved in case they failed.',
saveDataSuccessExecution: 'If data data of executions should be saved in case they succeed.',
saveExecutionProgress: 'If data should be saved after each node, allowing you to resume in case of errors from where it stopped. May increase latency.',
saveManualExecutions: 'If data data of executions should be saved when started manually from the editor.',
executionTimeoutToggle: 'Cancel workflow execution after defined time',
executionTimeout: 'After what time the workflow should timeout.',
@@ -186,10 +205,12 @@ export default mixins(
timezone: 'America/New_York',
saveDataErrorExecution: 'all',
saveDataSuccessExecution: 'all',
saveExecutionProgress: false,
saveManualExecutions: false,
},
saveDataErrorExecutionOptions: [] as Array<{ key: string, value: string }>,
saveDataSuccessExecutionOptions: [] as Array<{ key: string, value: string }>,
saveExecutionProgressOptions: [] as Array<{ key: string | boolean, value: string }>,
saveManualOptions: [] as Array<{ key: string | boolean, value: string }>,
timezones: [] as Array<{ key: string, value: string }>,
workflowSettings: {} as IWorkflowSettings,
@@ -251,6 +272,25 @@ export default mixins(
],
);
},
async loadSaveExecutionProgressOptions () {
this.saveExecutionProgressOptions.length = 0;
this.saveExecutionProgressOptions.push.apply( // eslint-disable-line no-useless-call
this.saveExecutionProgressOptions, [
{
key: 'DEFAULT',
value: 'Default - ' + (this.defaultValues.saveExecutionProgress === true ? 'Yes' : 'No'),
},
{
key: true,
value: 'Yes',
},
{
key: false,
value: 'No',
},
],
);
},
async loadSaveManualOptions () {
this.saveManualOptions.length = 0;
this.saveManualOptions.push({
@@ -334,6 +374,7 @@ export default mixins(
promises.push(this.loadWorkflows());
promises.push(this.loadSaveDataErrorExecutionOptions());
promises.push(this.loadSaveDataSuccessExecutionOptions());
promises.push(this.loadSaveExecutionProgressOptions());
promises.push(this.loadSaveManualOptions());
promises.push(this.loadTimezones());
@@ -354,6 +395,9 @@ export default mixins(
if (workflowSettings.saveDataSuccessExecution === undefined) {
workflowSettings.saveDataSuccessExecution = 'DEFAULT';
}
if (workflowSettings.saveExecutionProgress === undefined) {
workflowSettings.saveExecutionProgress = 'DEFAULT';
}
if (workflowSettings.saveManualExecutions === undefined) {
workflowSettings.saveManualExecutions = 'DEFAULT';
}