Iván Ovejero
2023-12-13 10:00:21 +01:00
committed by GitHub
parent 70f0755278
commit 7b5d0a9546
4 changed files with 355 additions and 205 deletions

View File

@@ -1,26 +1,18 @@
import { flags } from '@oclif/command';
import type { INode, INodeCredentialsDetails } from 'n8n-workflow';
import { ApplicationError, jsonParse } from 'n8n-workflow';
import fs from 'fs';
import glob from 'fast-glob';
import { Container } from 'typedi';
import type { EntityManager } from 'typeorm';
import { v4 as uuid } from 'uuid';
import * as Db from '@/Db';
import { SharedWorkflow } from '@db/entities/SharedWorkflow';
import { WorkflowEntity } from '@db/entities/WorkflowEntity';
import type { Role } from '@db/entities/Role';
import type { User } from '@db/entities/User';
import { disableAutoGeneratedIds } from '@db/utils/commandHelpers';
import type { ICredentialsDb, IWorkflowToImport } from '@/Interfaces';
import { replaceInvalidCredentials } from '@/WorkflowHelpers';
import type { IWorkflowToImport } from '@/Interfaces';
import { BaseCommand } from '../BaseCommand';
import { generateNanoId } from '@db/utils/generators';
import { RoleService } from '@/services/role.service';
import { TagService } from '@/services/tag.service';
import { UM_FIX_INSTRUCTION } from '@/constants';
import { UserRepository } from '@db/repositories/user.repository';
import { CredentialsRepository } from '@db/repositories/credentials.repository';
import { ImportService } from '@/services/import.service';
import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
function assertHasWorkflowsToImport(workflows: unknown): asserts workflows is IWorkflowToImport[] {
if (!Array.isArray(workflows)) {
@@ -64,16 +56,9 @@ export class ImportWorkflowsCommand extends BaseCommand {
}),
};
private ownerWorkflowRole: Role;
private transactionManager: EntityManager;
private tagService: TagService;
async init() {
disableAutoGeneratedIds(WorkflowEntity);
await super.init();
this.tagService = Container.get(TagService);
}
async run(): Promise<void> {
@@ -94,12 +79,8 @@ export class ImportWorkflowsCommand extends BaseCommand {
}
}
await this.initOwnerWorkflowRole();
const user = flags.userId ? await this.getAssignee(flags.userId) : await this.getOwner();
const credentials = await Container.get(CredentialsRepository).find();
const tags = await this.tagService.getAll();
let totalImported = 0;
if (flags.separate) {
@@ -116,41 +97,17 @@ export class ImportWorkflowsCommand extends BaseCommand {
totalImported = files.length;
this.logger.info(`Importing ${totalImported} workflows...`);
await Db.getConnection().transaction(async (transactionManager) => {
this.transactionManager = transactionManager;
for (const file of files) {
const workflow = jsonParse<IWorkflowToImport>(
fs.readFileSync(file, { encoding: 'utf8' }),
);
if (!workflow.id) {
workflow.id = generateNanoId();
}
if (credentials.length > 0) {
workflow.nodes.forEach((node: INode) => {
this.transformCredentials(node, credentials);
if (!node.id) {
node.id = uuid();
}
});
}
if (Object.prototype.hasOwnProperty.call(workflow, 'tags')) {
await this.tagService.setTagsForImport(transactionManager, workflow, tags);
}
if (workflow.active) {
this.logger.info(
`Deactivating workflow "${workflow.name}" during import, remember to activate it later.`,
);
workflow.active = false;
}
await this.storeWorkflow(workflow, user);
for (const file of files) {
const workflow = jsonParse<IWorkflowToImport>(fs.readFileSync(file, { encoding: 'utf8' }));
if (!workflow.id) {
workflow.id = generateNanoId();
}
});
const _workflow = Container.get(WorkflowRepository).create(workflow);
await Container.get(ImportService).importWorkflows([_workflow], user.id);
}
this.reportSuccess(totalImported);
process.exit();
@@ -160,46 +117,13 @@ export class ImportWorkflowsCommand extends BaseCommand {
fs.readFileSync(flags.input, { encoding: 'utf8' }),
);
const _workflows = workflows.map((w) => Container.get(WorkflowRepository).create(w));
assertHasWorkflowsToImport(workflows);
totalImported = workflows.length;
await Db.getConnection().transaction(async (transactionManager) => {
this.transactionManager = transactionManager;
for (const workflow of workflows) {
let oldCredentialFormat = false;
if (credentials.length > 0) {
workflow.nodes.forEach((node: INode) => {
this.transformCredentials(node, credentials);
if (!node.id) {
node.id = uuid();
}
if (!node.credentials?.id) {
oldCredentialFormat = true;
}
});
}
if (oldCredentialFormat) {
try {
await replaceInvalidCredentials(workflow as unknown as WorkflowEntity);
} catch (error) {
this.logger.error('Failed to replace invalid credential', error as Error);
}
}
if (Object.prototype.hasOwnProperty.call(workflow, 'tags')) {
await this.tagService.setTagsForImport(transactionManager, workflow, tags);
}
if (workflow.active) {
this.logger.info(
`Deactivating workflow "${workflow.name}" during import, remember to activate it later.`,
);
workflow.active = false;
}
await this.storeWorkflow(workflow, user);
}
});
await Container.get(ImportService).importWorkflows(_workflows, user.id);
this.reportSuccess(totalImported);
}
@@ -213,29 +137,6 @@ export class ImportWorkflowsCommand extends BaseCommand {
this.logger.info(`Successfully imported ${total} ${total === 1 ? 'workflow.' : 'workflows.'}`);
}
private async initOwnerWorkflowRole() {
const ownerWorkflowRole = await Container.get(RoleService).findWorkflowOwnerRole();
if (!ownerWorkflowRole) {
throw new ApplicationError(`Failed to find owner workflow role. ${UM_FIX_INSTRUCTION}`);
}
this.ownerWorkflowRole = ownerWorkflowRole;
}
private async storeWorkflow(workflow: object, user: User) {
const result = await this.transactionManager.upsert(WorkflowEntity, workflow, ['id']);
await this.transactionManager.upsert(
SharedWorkflow,
{
workflowId: result.identifiers[0].id as string,
userId: user.id,
roleId: this.ownerWorkflowRole.id,
},
['workflowId', 'userId'],
);
}
private async getOwner() {
const ownerGlobalRole = await Container.get(RoleService).findGlobalOwnerRole();
@@ -259,28 +160,4 @@ export class ImportWorkflowsCommand extends BaseCommand {
return user;
}
private transformCredentials(node: INode, credentialsEntities: ICredentialsDb[]) {
if (node.credentials) {
const allNodeCredentials = Object.entries(node.credentials);
for (const [type, name] of allNodeCredentials) {
if (typeof name === 'string') {
const nodeCredentials: INodeCredentialsDetails = {
id: null,
name,
};
const matchingCredentials = credentialsEntities.filter(
(credentials) => credentials.name === name && credentials.type === type,
);
if (matchingCredentials.length === 1) {
nodeCredentials.id = matchingCredentials[0].id;
}
node.credentials[type] = nodeCredentials;
}
}
}
}
}

View File

@@ -0,0 +1,153 @@
import { Service } from 'typedi';
import { v4 as uuid } from 'uuid';
import { type INode, type INodeCredentialsDetails } from 'n8n-workflow';
import type { EntityManager } from 'typeorm';
import { Logger } from '@/Logger';
import * as Db from '@/Db';
import { CredentialsRepository } from '@/databases/repositories/credentials.repository';
import { TagRepository } from '@/databases/repositories/tag.repository';
import { SharedWorkflow } from '@/databases/entities/SharedWorkflow';
import { RoleService } from '@/services/role.service';
import { replaceInvalidCredentials } from '@/WorkflowHelpers';
import { WorkflowEntity } from '@/databases/entities/WorkflowEntity';
import { WorkflowTagMapping } from '@/databases/entities/WorkflowTagMapping';
import type { TagEntity } from '@/databases/entities/TagEntity';
import type { Role } from '@/databases/entities/Role';
import type { ICredentialsDb } from '@/Interfaces';
@Service()
export class ImportService {
private dbCredentials: ICredentialsDb[] = [];
private dbTags: TagEntity[] = [];
private workflowOwnerRole: Role;
constructor(
private readonly logger: Logger,
private readonly credentialsRepository: CredentialsRepository,
private readonly tagRepository: TagRepository,
private readonly roleService: RoleService,
) {}
async initRecords() {
this.dbCredentials = await this.credentialsRepository.find();
this.dbTags = await this.tagRepository.find();
this.workflowOwnerRole = await this.roleService.findWorkflowOwnerRole();
}
async importWorkflows(workflows: WorkflowEntity[], userId: string) {
await this.initRecords();
for (const workflow of workflows) {
workflow.nodes.forEach((node) => {
this.toNewCredentialFormat(node);
if (!node.id) node.id = uuid();
});
const hasInvalidCreds = workflow.nodes.some((node) => !node.credentials?.id);
if (hasInvalidCreds) await this.replaceInvalidCreds(workflow);
}
await Db.transaction(async (tx) => {
for (const workflow of workflows) {
if (workflow.active) {
workflow.active = false;
this.logger.info(`Deactivating workflow "${workflow.name}". Remember to activate later.`);
}
const upsertResult = await tx.upsert(WorkflowEntity, workflow, ['id']);
const workflowId = upsertResult.identifiers.at(0)?.id as string;
await tx.upsert(SharedWorkflow, { workflowId, userId, roleId: this.workflowOwnerRole.id }, [
'workflowId',
'userId',
]);
if (!workflow.tags?.length) continue;
await this.setTags(tx, workflow);
for (const tag of workflow.tags) {
await tx.upsert(WorkflowTagMapping, { tagId: tag.id, workflowId }, [
'tagId',
'workflowId',
]);
}
}
});
}
async replaceInvalidCreds(workflow: WorkflowEntity) {
try {
await replaceInvalidCredentials(workflow);
} catch (e) {
const error = e instanceof Error ? e : new Error(`${e}`);
this.logger.error('Failed to replace invalid credential', error);
}
}
/**
* Convert a node's credentials from old format `{ <nodeType>: <credentialName> }`
* to new format: `{ <nodeType>: { id: string | null, name: <credentialName> } }`
*/
private toNewCredentialFormat(node: INode) {
if (!node.credentials) return;
for (const [type, name] of Object.entries(node.credentials)) {
if (typeof name !== 'string') continue;
const nodeCredential: INodeCredentialsDetails = { id: null, name };
const match = this.dbCredentials.find((c) => c.name === name && c.type === type);
if (match) nodeCredential.id = match.id;
node.credentials[type] = nodeCredential;
}
}
/**
* Set tags on workflow to import while ensuring all tags exist in the database,
* either by matching incoming to existing tags or by creating them first.
*/
private async setTags(tx: EntityManager, workflow: WorkflowEntity) {
if (!workflow?.tags?.length) return;
for (let i = 0; i < workflow.tags.length; i++) {
const importTag = workflow.tags[i];
if (!importTag.name) continue;
const identicalMatch = this.dbTags.find(
(dbTag) =>
dbTag.id === importTag.id &&
dbTag.createdAt &&
importTag.createdAt &&
dbTag.createdAt.getTime() === new Date(importTag.createdAt).getTime(),
);
if (identicalMatch) {
workflow.tags[i] = identicalMatch;
continue;
}
const nameMatch = this.dbTags.find((dbTag) => dbTag.name === importTag.name);
if (nameMatch) {
workflow.tags[i] = nameMatch;
continue;
}
const tagEntity = this.tagRepository.create(importTag);
workflow.tags[i] = await tx.save<TagEntity>(tagEntity);
}
}
}

View File

@@ -1,9 +1,9 @@
import { TagRepository } from '@db/repositories/tag.repository';
import { Service } from 'typedi';
import { validateEntity } from '@/GenericHelpers';
import type { ITagToImport, ITagWithCountDb, IWorkflowToImport } from '@/Interfaces';
import type { ITagWithCountDb } from '@/Interfaces';
import type { TagEntity } from '@db/entities/TagEntity';
import type { EntityManager, FindManyOptions, FindOneOptions } from 'typeorm';
import type { FindManyOptions, FindOneOptions } from 'typeorm';
import type { UpsertOptions } from 'typeorm/repository/UpsertOptions';
import { ExternalHooks } from '@/ExternalHooks';
@@ -89,69 +89,4 @@ export class TagService {
return requestOrder.map((tagId) => tagMap[tagId]);
}
/**
* Set tag IDs to use existing tags, creates a new tag if no matching tag could be found
*/
async setTagsForImport(
transactionManager: EntityManager,
workflow: IWorkflowToImport,
tags: TagEntity[],
) {
if (!this.hasTags(workflow)) return;
const workflowTags = workflow.tags;
const tagLookupPromises = [];
for (let i = 0; i < workflowTags.length; i++) {
if (workflowTags[i]?.name) {
const lookupPromise = this.findOrCreateTag(transactionManager, workflowTags[i], tags).then(
(tag) => {
workflowTags[i] = {
id: tag.id,
name: tag.name,
};
},
);
tagLookupPromises.push(lookupPromise);
}
}
await Promise.all(tagLookupPromises);
}
private hasTags(workflow: IWorkflowToImport) {
return 'tags' in workflow && Array.isArray(workflow.tags) && workflow.tags.length > 0;
}
private async findOrCreateTag(
transactionManager: EntityManager,
importTag: ITagToImport,
tagsEntities: TagEntity[],
) {
// Assume tag is identical if createdAt date is the same to preserve a changed tag name
const identicalMatch = tagsEntities.find(
(existingTag) =>
existingTag.id === importTag.id &&
existingTag.createdAt &&
importTag.createdAt &&
existingTag.createdAt.getTime() === new Date(importTag.createdAt).getTime(),
);
if (identicalMatch) {
return identicalMatch;
}
const nameMatch = tagsEntities.find((existingTag) => existingTag.name === importTag.name);
if (nameMatch) {
return nameMatch;
}
const created = await this.txCreateTag(transactionManager, importTag.name);
tagsEntities.push(created);
return created;
}
private async txCreateTag(transactionManager: EntityManager, name: string) {
const tag = this.tagRepository.create({ name: name.trim() });
return transactionManager.save<TagEntity>(tag);
}
}

View File

@@ -0,0 +1,185 @@
import Container from 'typedi';
import { mock } from 'jest-mock-extended';
import { v4 as uuid } from 'uuid';
import type { INode } from 'n8n-workflow';
import { CredentialsRepository } from '@/databases/repositories/credentials.repository';
import { TagRepository } from '@/databases/repositories/tag.repository';
import { ImportService } from '@/services/import.service';
import { RoleService } from '@/services/role.service';
import { TagEntity } from '@/databases/entities/TagEntity';
import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
import { SharedWorkflowRepository } from '@/databases/repositories/sharedWorkflow.repository';
import * as testDb from './shared/testDb';
import { mockInstance } from '../shared/mocking';
import { createOwner } from './shared/db/users';
import { createWorkflow, getWorkflowById } from './shared/db/workflows';
import type { User } from '@db/entities/User';
describe('ImportService', () => {
let importService: ImportService;
let tagRepository: TagRepository;
let owner: User;
beforeAll(async () => {
await testDb.init();
owner = await createOwner();
tagRepository = Container.get(TagRepository);
const credentialsRepository = mockInstance(CredentialsRepository);
credentialsRepository.find.mockResolvedValue([]);
importService = new ImportService(
mock(),
credentialsRepository,
tagRepository,
Container.get(RoleService),
);
});
afterEach(async () => {
await testDb.truncate(['Workflow', 'SharedWorkflow', 'Tag', 'WorkflowTagMapping']);
});
afterAll(async () => {
await testDb.terminate();
});
test('should import credless and tagless workflow', async () => {
const workflowToImport = await createWorkflow();
await importService.importWorkflows([workflowToImport], owner.id);
const dbWorkflow = await getWorkflowById(workflowToImport.id);
if (!dbWorkflow) fail('Expected to find workflow');
expect(dbWorkflow.id).toBe(workflowToImport.id);
});
test('should make user owner of imported workflow', async () => {
const workflowToImport = await createWorkflow();
await importService.importWorkflows([workflowToImport], owner.id);
const workflowOwnerRole = await Container.get(RoleService).findWorkflowOwnerRole();
const dbSharing = await Container.get(SharedWorkflowRepository).findOneOrFail({
where: { workflowId: workflowToImport.id, userId: owner.id, roleId: workflowOwnerRole.id },
});
expect(dbSharing.userId).toBe(owner.id);
});
test('should deactivate imported workflow if active', async () => {
const workflowToImport = await createWorkflow({ active: true });
await importService.importWorkflows([workflowToImport], owner.id);
const dbWorkflow = await getWorkflowById(workflowToImport.id);
if (!dbWorkflow) fail('Expected to find workflow');
expect(dbWorkflow.active).toBe(false);
});
test('should leave intact new-format credentials', async () => {
const credential = {
n8nApi: { id: '123', name: 'n8n API' },
};
const nodes: INode[] = [
{
id: uuid(),
name: 'n8n',
parameters: {},
position: [0, 0],
type: 'n8n-nodes-base.n8n',
typeVersion: 1,
credentials: credential,
},
];
const workflowToImport = await createWorkflow({ nodes });
await importService.importWorkflows([workflowToImport], owner.id);
const dbWorkflow = await getWorkflowById(workflowToImport.id);
if (!dbWorkflow) fail('Expected to find workflow');
expect(dbWorkflow.nodes.at(0)?.credentials).toMatchObject(credential);
});
test('should set tag by identical match', async () => {
const tag = Object.assign(new TagEntity(), {
id: '123',
createdAt: new Date(),
name: 'Test',
});
await tagRepository.save(tag); // tag stored
const workflowToImport = await createWorkflow({ tags: [tag] });
await importService.importWorkflows([workflowToImport], owner.id);
const dbWorkflow = await Container.get(WorkflowRepository).findOneOrFail({
where: { id: workflowToImport.id },
relations: ['tags'],
});
expect(dbWorkflow.tags).toStrictEqual([tag]); // workflow tagged
const dbTags = await tagRepository.find();
expect(dbTags).toStrictEqual([tag]); // tag matched
});
test('should set tag by name match', async () => {
const tag = Object.assign(new TagEntity(), { name: 'Test' });
await tagRepository.save(tag); // tag stored
const workflowToImport = await createWorkflow({ tags: [tag] });
await importService.importWorkflows([workflowToImport], owner.id);
const dbWorkflow = await Container.get(WorkflowRepository).findOneOrFail({
where: { id: workflowToImport.id },
relations: ['tags'],
});
expect(dbWorkflow.tags).toStrictEqual([tag]); // workflow tagged
const dbTags = await tagRepository.find();
expect(dbTags).toStrictEqual([tag]); // tag matched
});
test('should set tag by creating if no match', async () => {
const tag = Object.assign(new TagEntity(), { name: 'Test' }); // tag not stored
const workflowToImport = await createWorkflow({ tags: [tag] });
await importService.importWorkflows([workflowToImport], owner.id);
const dbWorkflow = await Container.get(WorkflowRepository).findOneOrFail({
where: { id: workflowToImport.id },
relations: ['tags'],
});
if (!dbWorkflow.tags) fail('No tags found on workflow');
expect(dbWorkflow.tags.at(0)?.name).toBe(tag.name); // workflow tagged
const dbTag = await tagRepository.findOneOrFail({ where: { name: tag.name } });
expect(dbTag.name).toBe(tag.name); // tag created
});
});