ci(core): Reduce memory usage in tests (part-1) (no-changelog) (#7654)

packages/cli/test/integration/shared/db/credentials.ts

@@ -0,0 +1,67 @@
+import { CredentialsEntity } from '@db/entities/CredentialsEntity';
+import type { User } from '@db/entities/User';
+import type { Role } from '@db/entities/Role';
+import type { ICredentialsDb } from '@/Interfaces';
+import { RoleService } from '@/services/role.service';
+import type { CredentialPayload } from '../types';
+import Container from 'typedi';
+import { CredentialsRepository, SharedCredentialsRepository } from '@/databases/repositories';
+
+async function encryptCredentialData(credential: CredentialsEntity) {
+	const { createCredentialsFromCredentialsEntity } = await import('@/CredentialsHelper');
+	const coreCredential = createCredentialsFromCredentialsEntity(credential, true);
+
+	// @ts-ignore
+	coreCredential.setData(credential.data);
+
+	return coreCredential.getDataToSave() as ICredentialsDb;
+}
+
+/**
+ * Save a credential to the test DB, sharing it with a user.
+ */
+export async function saveCredential(
+	credentialPayload: CredentialPayload,
+	{ user, role }: { user: User; role: Role },
+) {
+	const newCredential = new CredentialsEntity();
+
+	Object.assign(newCredential, credentialPayload);
+
+	const encryptedData = await encryptCredentialData(newCredential);
+
+	Object.assign(newCredential, encryptedData);
+
+	const savedCredential = await Container.get(CredentialsRepository).save(newCredential);
+
+	savedCredential.data = newCredential.data;
+
+	await Container.get(SharedCredentialsRepository).save({
+		user,
+		credentials: savedCredential,
+		role,
+	});
+
+	return savedCredential;
+}
+
+export async function shareCredentialWithUsers(credential: CredentialsEntity, users: User[]) {
+	const role = await Container.get(RoleService).findCredentialUserRole();
+	const newSharedCredentials = users.map((user) =>
+		Container.get(SharedCredentialsRepository).create({
+			userId: user.id,
+			credentialsId: credential.id,
+			roleId: role?.id,
+		}),
+	);
+	return Container.get(SharedCredentialsRepository).save(newSharedCredentials);
+}
+
+export function affixRoleToSaveCredential(role: Role) {
+	return async (credentialPayload: CredentialPayload, { user }: { user: User }) =>
+		saveCredential(credentialPayload, { user, role });
+}
+
+export async function getAllCredentials() {
+	return Container.get(CredentialsRepository).find();
+}

packages/cli/test/integration/shared/db/executions.ts

@@ -0,0 +1,68 @@
+import Container from 'typedi';
+import type { ExecutionData } from '@db/entities/ExecutionData';
+import type { ExecutionEntity } from '@db/entities/ExecutionEntity';
+import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
+import { ExecutionDataRepository, ExecutionRepository } from '@db/repositories';
+
+export async function createManyExecutions(
+	amount: number,
+	workflow: WorkflowEntity,
+	callback: (workflow: WorkflowEntity) => Promise<ExecutionEntity>,
+) {
+	const executionsRequests = [...Array(amount)].map(async (_) => callback(workflow));
+	return Promise.all(executionsRequests);
+}
+
+/**
+ * Store a execution in the DB and assign it to a workflow.
+ */
+export async function createExecution(
+	attributes: Partial<ExecutionEntity & ExecutionData>,
+	workflow: WorkflowEntity,
+) {
+	const { data, finished, mode, startedAt, stoppedAt, waitTill, status, deletedAt } = attributes;
+
+	const execution = await Container.get(ExecutionRepository).save({
+		finished: finished ?? true,
+		mode: mode ?? 'manual',
+		startedAt: startedAt ?? new Date(),
+		...(workflow !== undefined && { workflowId: workflow.id }),
+		stoppedAt: stoppedAt ?? new Date(),
+		waitTill: waitTill ?? null,
+		status,
+		deletedAt,
+	});
+
+	await Container.get(ExecutionDataRepository).save({
+		data: data ?? '[]',
+		workflowData: workflow ?? {},
+		executionId: execution.id,
+	});
+
+	return execution;
+}
+
+/**
+ * Store a successful execution in the DB and assign it to a workflow.
+ */
+export async function createSuccessfulExecution(workflow: WorkflowEntity) {
+	return createExecution({ finished: true, status: 'success' }, workflow);
+}
+
+/**
+ * Store an error execution in the DB and assign it to a workflow.
+ */
+export async function createErrorExecution(workflow: WorkflowEntity) {
+	return createExecution({ finished: false, stoppedAt: new Date(), status: 'failed' }, workflow);
+}
+
+/**
+ * Store a waiting execution in the DB and assign it to a workflow.
+ */
+export async function createWaitingExecution(workflow: WorkflowEntity) {
+	return createExecution({ finished: false, waitTill: new Date(), status: 'waiting' }, workflow);
+}
+
+export async function getAllExecutions() {
+	return Container.get(ExecutionRepository).find();
+}

packages/cli/test/integration/shared/db/roles.ts

@@ -0,0 +1,31 @@
+import Container from 'typedi';
+import { RoleService } from '@/services/role.service';
+
+export async function getGlobalOwnerRole() {
+	return Container.get(RoleService).findGlobalOwnerRole();
+}
+
+export async function getGlobalMemberRole() {
+	return Container.get(RoleService).findGlobalMemberRole();
+}
+
+export async function getWorkflowOwnerRole() {
+	return Container.get(RoleService).findWorkflowOwnerRole();
+}
+
+export async function getWorkflowEditorRole() {
+	return Container.get(RoleService).findWorkflowEditorRole();
+}
+
+export async function getCredentialOwnerRole() {
+	return Container.get(RoleService).findCredentialOwnerRole();
+}
+
+export async function getAllRoles() {
+	return Promise.all([
+		getGlobalOwnerRole(),
+		getGlobalMemberRole(),
+		getWorkflowOwnerRole(),
+		getCredentialOwnerRole(),
+	]);
+}

packages/cli/test/integration/shared/db/tags.ts

@@ -0,0 +1,25 @@
+import Container from 'typedi';
+import type { TagEntity } from '@db/entities/TagEntity';
+import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
+import { TagRepository, WorkflowTagMappingRepository } from '@db/repositories';
+import { generateNanoId } from '@db/utils/generators';
+
+import { randomName } from '../random';
+
+export async function createTag(attributes: Partial<TagEntity> = {}, workflow?: WorkflowEntity) {
+	const { name } = attributes;
+
+	const tag = await Container.get(TagRepository).save({
+		id: generateNanoId(),
+		name: name ?? randomName(),
+		...attributes,
+	});
+
+	if (workflow) {
+		const mappingRepository = Container.get(WorkflowTagMappingRepository);
+		const mapping = mappingRepository.create({ tagId: tag.id, workflowId: workflow.id });
+		await mappingRepository.save(mapping);
+	}
+
+	return tag;
+}

packages/cli/test/integration/shared/db/users.ts

@@ -0,0 +1,134 @@
+import Container from 'typedi';
+import { hash } from 'bcryptjs';
+import { AuthIdentity } from '@db/entities/AuthIdentity';
+import type { Role } from '@db/entities/Role';
+import type { User } from '@db/entities/User';
+import { AuthIdentityRepository, UserRepository } from '@db/repositories';
+import { TOTPService } from '@/Mfa/totp.service';
+import { MfaService } from '@/Mfa/mfa.service';
+
+import { randomApiKey, randomEmail, randomName, randomValidPassword } from '../random';
+import { getGlobalMemberRole, getGlobalOwnerRole } from './roles';
+
+/**
+ * Store a user in the DB, defaulting to a `member`.
+ */
+export async function createUser(attributes: Partial<User> = {}): Promise<User> {
+	const { email, password, firstName, lastName, globalRole, ...rest } = attributes;
+	const user: Partial<User> = {
+		email: email ?? randomEmail(),
+		password: await hash(password ?? randomValidPassword(), 10),
+		firstName: firstName ?? randomName(),
+		lastName: lastName ?? randomName(),
+		globalRoleId: (globalRole ?? (await getGlobalMemberRole())).id,
+		globalRole,
+		...rest,
+	};
+
+	return Container.get(UserRepository).save(user);
+}
+
+export async function createLdapUser(attributes: Partial<User>, ldapId: string): Promise<User> {
+	const user = await createUser(attributes);
+	await Container.get(AuthIdentityRepository).save(AuthIdentity.create(user, ldapId, 'ldap'));
+	return user;
+}
+
+export async function createUserWithMfaEnabled(
+	data: { numberOfRecoveryCodes: number } = { numberOfRecoveryCodes: 10 },
+) {
+	const email = randomEmail();
+	const password = randomValidPassword();
+
+	const toptService = new TOTPService();
+
+	const secret = toptService.generateSecret();
+
+	const mfaService = Container.get(MfaService);
+
+	const recoveryCodes = mfaService.generateRecoveryCodes(data.numberOfRecoveryCodes);
+
+	const { encryptedSecret, encryptedRecoveryCodes } = mfaService.encryptSecretAndRecoveryCodes(
+		secret,
+		recoveryCodes,
+	);
+
+	return {
+		user: await createUser({
+			mfaEnabled: true,
+			password,
+			email,
+			mfaSecret: encryptedSecret,
+			mfaRecoveryCodes: encryptedRecoveryCodes,
+		}),
+		rawPassword: password,
+		rawSecret: secret,
+		rawRecoveryCodes: recoveryCodes,
+	};
+}
+
+export async function createOwner() {
+	return createUser({ globalRole: await getGlobalOwnerRole() });
+}
+
+export async function createMember() {
+	return createUser({ globalRole: await getGlobalMemberRole() });
+}
+
+export async function createUserShell(globalRole: Role): Promise<User> {
+	if (globalRole.scope !== 'global') {
+		throw new Error(`Invalid role received: ${JSON.stringify(globalRole)}`);
+	}
+
+	const shell: Partial<User> = { globalRoleId: globalRole.id };
+
+	if (globalRole.name !== 'owner') {
+		shell.email = randomEmail();
+	}
+
+	return Container.get(UserRepository).save(shell);
+}
+
+/**
+ * Create many users in the DB, defaulting to a `member`.
+ */
+export async function createManyUsers(
+	amount: number,
+	attributes: Partial<User> = {},
+): Promise<User[]> {
+	let { email, password, firstName, lastName, globalRole, ...rest } = attributes;
+	if (!globalRole) {
+		globalRole = await getGlobalMemberRole();
+	}
+
+	const users = await Promise.all(
+		[...Array(amount)].map(async () =>
+			Container.get(UserRepository).create({
+				email: email ?? randomEmail(),
+				password: await hash(password ?? randomValidPassword(), 10),
+				firstName: firstName ?? randomName(),
+				lastName: lastName ?? randomName(),
+				globalRole,
+				...rest,
+			}),
+		),
+	);
+
+	return Container.get(UserRepository).save(users);
+}
+
+export async function addApiKey(user: User): Promise<User> {
+	user.apiKey = randomApiKey();
+	return Container.get(UserRepository).save(user);
+}
+
+export const getAllUsers = async () =>
+	Container.get(UserRepository).find({
+		relations: ['globalRole', 'authIdentities'],
+	});
+
+export const getLdapIdentities = async () =>
+	Container.get(AuthIdentityRepository).find({
+		where: { providerType: 'ldap' },
+		relations: ['user'],
+	});

packages/cli/test/integration/shared/db/workflowHistory.ts

@@ -0,0 +1,43 @@
+import Container from 'typedi';
+import { v4 as uuid } from 'uuid';
+import type { WorkflowHistory } from '@db/entities/WorkflowHistory';
+import { WorkflowHistoryRepository } from '@db/repositories';
+
+export async function createWorkflowHistoryItem(
+	workflowId: string,
+	data?: Partial<WorkflowHistory>,
+) {
+	return Container.get(WorkflowHistoryRepository).save({
+		authors: 'John Smith',
+		connections: {},
+		nodes: [
+			{
+				id: 'uuid-1234',
+				name: 'Start',
+				parameters: {},
+				position: [-20, 260],
+				type: 'n8n-nodes-base.start',
+				typeVersion: 1,
+			},
+		],
+		versionId: uuid(),
+		...(data ?? {}),
+		workflowId,
+	});
+}
+
+export async function createManyWorkflowHistoryItems(
+	workflowId: string,
+	count: number,
+	time?: Date,
+) {
+	const baseTime = (time ?? new Date()).valueOf();
+	return Promise.all(
+		[...Array(count)].map(async (_, i) =>
+			createWorkflowHistoryItem(workflowId, {
+				createdAt: new Date(baseTime + i),
+				updatedAt: new Date(baseTime + i),
+			}),
+		),
+	);
+}

packages/cli/test/integration/shared/db/workflows.ts

@@ -0,0 +1,118 @@
+import Container from 'typedi';
+import { v4 as uuid } from 'uuid';
+import type { User } from '@db/entities/User';
+import type { WorkflowEntity } from '@db/entities/WorkflowEntity';
+import { SharedWorkflowRepository, WorkflowRepository } from '@db/repositories';
+import { getWorkflowEditorRole, getWorkflowOwnerRole } from './roles';
+
+export async function createManyWorkflows(
+	amount: number,
+	attributes: Partial<WorkflowEntity> = {},
+	user?: User,
+) {
+	const workflowRequests = [...Array(amount)].map(async (_) => createWorkflow(attributes, user));
+	return Promise.all(workflowRequests);
+}
+
+/**
+ * Store a workflow in the DB (without a trigger) and optionally assign it to a user.
+ * @param attributes workflow attributes
+ * @param user user to assign the workflow to
+ */
+export async function createWorkflow(attributes: Partial<WorkflowEntity> = {}, user?: User) {
+	const { active, name, nodes, connections, versionId } = attributes;
+
+	const workflowEntity = Container.get(WorkflowRepository).create({
+		active: active ?? false,
+		name: name ?? 'test workflow',
+		nodes: nodes ?? [
+			{
+				id: 'uuid-1234',
+				name: 'Schedule Trigger',
+				parameters: {},
+				position: [-20, 260],
+				type: 'n8n-nodes-base.scheduleTrigger',
+				typeVersion: 1,
+			},
+		],
+		connections: connections ?? {},
+		versionId: versionId ?? uuid(),
+		...attributes,
+	});
+
+	const workflow = await Container.get(WorkflowRepository).save(workflowEntity);
+
+	if (user) {
+		await Container.get(SharedWorkflowRepository).save({
+			user,
+			workflow,
+			role: await getWorkflowOwnerRole(),
+		});
+	}
+	return workflow;
+}
+
+export async function shareWorkflowWithUsers(workflow: WorkflowEntity, users: User[]) {
+	const role = await getWorkflowEditorRole();
+	const sharedWorkflows = users.map((user) => ({
+		user,
+		workflow,
+		role,
+	}));
+	return Container.get(SharedWorkflowRepository).save(sharedWorkflows);
+}
+
+export async function getWorkflowSharing(workflow: WorkflowEntity) {
+	return Container.get(SharedWorkflowRepository).findBy({
+		workflowId: workflow.id,
+	});
+}
+
+/**
+ * Store a workflow in the DB (with a trigger) and optionally assign it to a user.
+ * @param user user to assign the workflow to
+ */
+export async function createWorkflowWithTrigger(
+	attributes: Partial<WorkflowEntity> = {},
+	user?: User,
+) {
+	const workflow = await createWorkflow(
+		{
+			nodes: [
+				{
+					id: 'uuid-1',
+					parameters: {},
+					name: 'Start',
+					type: 'n8n-nodes-base.start',
+					typeVersion: 1,
+					position: [240, 300],
+				},
+				{
+					id: 'uuid-2',
+					parameters: { triggerTimes: { item: [{ mode: 'everyMinute' }] } },
+					name: 'Cron',
+					type: 'n8n-nodes-base.cron',
+					typeVersion: 1,
+					position: [500, 300],
+				},
+				{
+					id: 'uuid-3',
+					parameters: { options: {} },
+					name: 'Set',
+					type: 'n8n-nodes-base.set',
+					typeVersion: 1,
+					position: [780, 300],
+				},
+			],
+			connections: { Cron: { main: [[{ node: 'Set', type: 'main', index: 0 }]] } },
+			...attributes,
+		},
+		user,
+	);
+
+	return workflow;
+}
+
+export async function getAllWorkflows() {
+	return Container.get(WorkflowRepository).find();
+}