admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-18 02:21:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

⚡ Run migration in chunks (#2393)

Browse Source
This commit is contained in:
Ben Hesseldieck
2021-11-03 16:12:48 +01:00
committed by GitHub
parent 4f9aee14b5
commit 0877f485d9
4 changed files with 560 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

261
packages/cli/src/databases/sqlite/migrations/1630330987096-UpdateWorkflowCredentials.ts
View File

@@ -1,5 +1,6 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
import config = require('../../../../config');
import { MigrationHelpers } from '../../MigrationHelpers';
// replacing the credentials in workflows and execution
// `nodeType: name` changes to `nodeType: { id, name }`
@@ -8,58 +9,101 @@ export class UpdateWorkflowCredentials1630330987096 implements MigrationInterfac
name = 'UpdateWorkflowCredentials1630330987096';
public async up(queryRunner: QueryRunner): Promise<void> {
console.log('Start migration', this.name);
console.time(this.name);
const tablePrefix = config.get('database.tablePrefix');
const helpers = new MigrationHelpers(queryRunner);
const credentialsEntities = await queryRunner.query(`
SELECT id, name, type
FROM "${tablePrefix}credentials_entity"
`);
const workflows = await queryRunner.query(`
const workflowsQuery = `
SELECT id, nodes
FROM "${tablePrefix}workflow_entity"
`);
`;
// @ts-ignore
workflows.forEach(async (workflow) => {
const nodes = JSON.parse(workflow.nodes);
let credentialsUpdated = false;
// @ts-ignore
nodes.forEach((node) => {
if (node.credentials) {
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, name] of allNodeCredentials) {
if (typeof name === 'string') {
// @ts-ignore
const matchingCredentials = credentialsEntities.find(
// @ts-ignore
(credentials) => credentials.name === name && credentials.type === type,
);
node.credentials[type] = { id: matchingCredentials?.id || null, name };
credentialsUpdated = true;
await helpers.runChunked(workflowsQuery, (workflows) => {
workflows.forEach(async (workflow) => {
const nodes = JSON.parse(workflow.nodes);
let credentialsUpdated = false;
// @ts-ignore
nodes.forEach((node) => {
if (node.credentials) {
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, name] of allNodeCredentials) {
if (typeof name === 'string') {
const matchingCredentials = credentialsEntities.find(
// @ts-ignore
(credentials) => credentials.name === name && credentials.type === type,
);
node.credentials[type] = { id: matchingCredentials?.id || null, name };
credentialsUpdated = true;
}
}
}
});
if (credentialsUpdated) {
const [updateQuery, updateParams] =
queryRunner.connection.driver.escapeQueryWithParameters(
`
UPDATE "${tablePrefix}workflow_entity"
SET nodes = :nodes
WHERE id = '${workflow.id}'
`,
{ nodes: JSON.stringify(nodes) },
{},
);
queryRunner.query(updateQuery, updateParams);
}
});
if (credentialsUpdated) {
const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
`
UPDATE "${tablePrefix}workflow_entity"
SET nodes = :nodes
WHERE id = '${workflow.id}'
`,
{ nodes: JSON.stringify(nodes) },
{},
);
await queryRunner.query(updateQuery, updateParams);
}
});
const waitingExecutions = await queryRunner.query(`
const waitingExecutionsQuery = `
SELECT id, "workflowData"
FROM "${tablePrefix}execution_entity"
WHERE "waitTill" IS NOT NULL AND finished = 0
`);
`;
// @ts-ignore
await helpers.runChunked(waitingExecutionsQuery, (waitingExecutions) => {
waitingExecutions.forEach(async (execution) => {
const data = JSON.parse(execution.workflowData);
let credentialsUpdated = false;
// @ts-ignore
data.nodes.forEach((node) => {
if (node.credentials) {
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, name] of allNodeCredentials) {
if (typeof name === 'string') {
const matchingCredentials = credentialsEntities.find(
// @ts-ignore
(credentials) => credentials.name === name && credentials.type === type,
);
node.credentials[type] = { id: matchingCredentials?.id || null, name };
credentialsUpdated = true;
}
}
}
});
if (credentialsUpdated) {
const [updateQuery, updateParams] =
queryRunner.connection.driver.escapeQueryWithParameters(
`
UPDATE "${tablePrefix}execution_entity"
SET "workflowData" = :data
WHERE id = '${execution.id}'
`,
{ data: JSON.stringify(data) },
{},
);
queryRunner.query(updateQuery, updateParams);
}
});
});
const retryableExecutions = await queryRunner.query(`
SELECT id, "workflowData"
@@ -68,8 +112,8 @@ export class UpdateWorkflowCredentials1630330987096 implements MigrationInterfac
ORDER BY "startedAt" DESC
LIMIT 200
`);
[...waitingExecutions, ...retryableExecutions].forEach(async (execution) => {
// @ts-ignore
retryableExecutions.forEach(async (execution) => {
const data = JSON.parse(execution.workflowData);
let credentialsUpdated = false;
// @ts-ignore
@@ -78,7 +122,6 @@ export class UpdateWorkflowCredentials1630330987096 implements MigrationInterfac
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, name] of allNodeCredentials) {
if (typeof name === 'string') {
// @ts-ignore
const matchingCredentials = credentialsEntities.find(
// @ts-ignore
(credentials) => credentials.name === name && credentials.type === type,
@@ -92,77 +135,127 @@ export class UpdateWorkflowCredentials1630330987096 implements MigrationInterfac
if (credentialsUpdated) {
const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
`
UPDATE "${tablePrefix}execution_entity"
SET "workflowData" = :data
WHERE id = '${execution.id}'
`,
UPDATE "${tablePrefix}execution_entity"
SET "workflowData" = :data
WHERE id = '${execution.id}'
`,
{ data: JSON.stringify(data) },
{},
);
await queryRunner.query(updateQuery, updateParams);
queryRunner.query(updateQuery, updateParams);
}
});
console.timeEnd(this.name);
}
public async down(queryRunner: QueryRunner): Promise<void> {
const tablePrefix = config.get('database.tablePrefix');
const helpers = new MigrationHelpers(queryRunner);
const credentialsEntities = await queryRunner.query(`
SELECT id, name, type
FROM "${tablePrefix}credentials_entity"
`);
const workflows = await queryRunner.query(`
const workflowsQuery = `
SELECT id, nodes
FROM "${tablePrefix}workflow_entity"
`);
`;
// @ts-ignore
workflows.forEach(async (workflow) => {
const nodes = JSON.parse(workflow.nodes);
let credentialsUpdated = false;
await helpers.runChunked(workflowsQuery, (workflows) => {
// @ts-ignore
nodes.forEach((node) => {
if (node.credentials) {
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, creds] of allNodeCredentials) {
if (typeof creds === 'object') {
// @ts-ignore
const matchingCredentials = credentialsEntities.find(
// @ts-ignore
(credentials) => credentials.id === creds.id && credentials.type === type,
);
if (matchingCredentials) {
node.credentials[type] = matchingCredentials.name;
} else {
// @ts-ignore
node.credentials[type] = creds.name;
workflows.forEach(async (workflow) => {
const nodes = JSON.parse(workflow.nodes);
let credentialsUpdated = false;
// @ts-ignore
nodes.forEach((node) => {
if (node.credentials) {
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, creds] of allNodeCredentials) {
if (typeof creds === 'object') {
const matchingCredentials = credentialsEntities.find(
// @ts-ignore
(credentials) => credentials.id === creds.id && credentials.type === type,
);
if (matchingCredentials) {
node.credentials[type] = matchingCredentials.name;
} else {
// @ts-ignore
node.credentials[type] = creds.name;
}
credentialsUpdated = true;
}
credentialsUpdated = true;
}
}
});
if (credentialsUpdated) {
const [updateQuery, updateParams] =
queryRunner.connection.driver.escapeQueryWithParameters(
`
UPDATE "${tablePrefix}workflow_entity"
SET nodes = :nodes
WHERE id = '${workflow.id}'
`,
{ nodes: JSON.stringify(nodes) },
{},
);
queryRunner.query(updateQuery, updateParams);
}
});
if (credentialsUpdated) {
const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
`
UPDATE "${tablePrefix}workflow_entity"
SET nodes = :nodes
WHERE id = '${workflow.id}'
`,
{ nodes: JSON.stringify(nodes) },
{},
);
await queryRunner.query(updateQuery, updateParams);
}
});
const waitingExecutions = await queryRunner.query(`
const waitingExecutionsQuery = `
SELECT id, "workflowData"
FROM "${tablePrefix}execution_entity"
WHERE "waitTill" IS NOT NULL AND finished = 0
`);
`;
// @ts-ignore
await helpers.runChunked(waitingExecutionsQuery, (waitingExecutions) => {
// @ts-ignore
waitingExecutions.forEach(async (execution) => {
const data = JSON.parse(execution.workflowData);
let credentialsUpdated = false;
// @ts-ignore
data.nodes.forEach((node) => {
if (node.credentials) {
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, creds] of allNodeCredentials) {
if (typeof creds === 'object') {
const matchingCredentials = credentialsEntities.find(
// @ts-ignore
(credentials) => credentials.id === creds.id && credentials.type === type,
);
if (matchingCredentials) {
node.credentials[type] = matchingCredentials.name;
} else {
// @ts-ignore
node.credentials[type] = creds.name;
}
credentialsUpdated = true;
}
}
}
});
if (credentialsUpdated) {
const [updateQuery, updateParams] =
queryRunner.connection.driver.escapeQueryWithParameters(
`
UPDATE "${tablePrefix}execution_entity"
SET "workflowData" = :data
WHERE id = '${execution.id}'
`,
{ data: JSON.stringify(data) },
{},
);
await queryRunner.query(updateQuery, updateParams);
}
});
});
const retryableExecutions = await queryRunner.query(`
SELECT id, "workflowData"
@@ -172,7 +265,8 @@ export class UpdateWorkflowCredentials1630330987096 implements MigrationInterfac
LIMIT 200
`);
[...waitingExecutions, ...retryableExecutions].forEach(async (execution) => {
// @ts-ignore
retryableExecutions.forEach(async (execution) => {
const data = JSON.parse(execution.workflowData);
let credentialsUpdated = false;
// @ts-ignore
@@ -181,7 +275,6 @@ export class UpdateWorkflowCredentials1630330987096 implements MigrationInterfac
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, creds] of allNodeCredentials) {
if (typeof creds === 'object') {
// @ts-ignore
const matchingCredentials = credentialsEntities.find(
// @ts-ignore
(credentials) => credentials.id === creds.id && credentials.type === type,
@@ -200,15 +293,15 @@ export class UpdateWorkflowCredentials1630330987096 implements MigrationInterfac
if (credentialsUpdated) {
const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters(
`
UPDATE "${tablePrefix}execution_entity"
SET "workflowData" = :data
WHERE id = '${execution.id}'
`,
UPDATE "${tablePrefix}execution_entity"
SET "workflowData" = :data
WHERE id = '${execution.id}'
`,
{ data: JSON.stringify(data) },
{},
);
await queryRunner.query(updateQuery, updateParams);
queryRunner.query(updateQuery, updateParams);
}
});
}