feat(core): Add folder synchronization to environments feature (#14005)

packages/cli/src/environments.ee/source-control/source-control-import.service.ee.ts

@@ -17,6 +17,7 @@ import type { User } from '@/databases/entities/user';
 import type { Variables } from '@/databases/entities/variables';
 import type { WorkflowTagMapping } from '@/databases/entities/workflow-tag-mapping';
 import { CredentialsRepository } from '@/databases/repositories/credentials.repository';
+import { FolderRepository } from '@/databases/repositories/folder.repository';
 import { ProjectRepository } from '@/databases/repositories/project.repository';
 import { SharedCredentialsRepository } from '@/databases/repositories/shared-credentials.repository';
 import { SharedWorkflowRepository } from '@/databases/repositories/shared-workflow.repository';
@@ -33,6 +34,7 @@ import { WorkflowService } from '@/workflows/workflow.service';
 
 import {
 	SOURCE_CONTROL_CREDENTIAL_EXPORT_FOLDER,
+	SOURCE_CONTROL_FOLDERS_EXPORT_FILE,
 	SOURCE_CONTROL_GIT_FOLDER,
 	SOURCE_CONTROL_TAGS_EXPORT_FILE,
 	SOURCE_CONTROL_VARIABLES_EXPORT_FILE,
@@ -40,6 +42,7 @@ import {
 } from './constants';
 import { getCredentialExportPath, getWorkflowExportPath } from './source-control-helper.ee';
 import type { ExportableCredential } from './types/exportable-credential';
+import type { ExportableFolder } from './types/exportable-folders';
 import type { ResourceOwner } from './types/resource-owner';
 import type { SourceControlWorkflowVersionId } from './types/source-control-workflow-version-id';
 import { VariablesService } from '../variables/variables.service.ee';
@@ -69,6 +72,7 @@ export class SourceControlImportService {
 		private readonly workflowService: WorkflowService,
 		private readonly credentialsService: CredentialsService,
 		private readonly tagService: TagService,
+		private readonly folderRepository: FolderRepository,
 		instanceSettings: InstanceSettings,
 	) {
 		this.gitFolder = path.join(instanceSettings.n8nFolder, SOURCE_CONTROL_GIT_FOLDER);
@@ -88,6 +92,7 @@ export class SourceControlImportService {
 			remoteWorkflowFiles.map(async (file) => {
 				this.logger.debug(`Parsing workflow file ${file}`);
 				const remote = jsonParse<IWorkflowToImport>(await fsReadFile(file, { encoding: 'utf8' }));
+
 				if (!remote?.id) {
 					return undefined;
 				}
@@ -95,6 +100,7 @@ export class SourceControlImportService {
 					id: remote.id,
 					versionId: remote.versionId,
 					name: remote.name,
+					parentFolderId: remote.parentFolderId,
 					remoteId: remote.id,
 					filename: getWorkflowExportPath(remote.id, this.workflowExportFolder),
 				} as SourceControlWorkflowVersionId;
@@ -107,7 +113,16 @@ export class SourceControlImportService {
 
 	async getLocalVersionIdsFromDb(): Promise<SourceControlWorkflowVersionId[]> {
 		const localWorkflows = await this.workflowRepository.find({
-			select: ['id', 'name', 'versionId', 'updatedAt'],
+			relations: ['parentFolder'],
+			select: {
+				id: true,
+				versionId: true,
+				name: true,
+				updatedAt: true,
+				parentFolder: {
+					id: true,
+				},
+			},
 		});
 		return localWorkflows.map((local) => {
 			let updatedAt: Date;
@@ -127,6 +142,7 @@ export class SourceControlImportService {
 				versionId: local.versionId,
 				name: local.name,
 				localId: local.id,
+				parentFolderId: local.parentFolder?.id ?? null,
 				filename: getWorkflowExportPath(local.id, this.workflowExportFolder),
 				updatedAt: updatedAt.toISOString(),
 			};
@@ -190,6 +206,52 @@ export class SourceControlImportService {
 		return await this.variablesService.getAllCached();
 	}
 
+	async getRemoteFoldersAndMappingsFromFile(): Promise<{
+		folders: ExportableFolder[];
+	}> {
+		const foldersFile = await glob(SOURCE_CONTROL_FOLDERS_EXPORT_FILE, {
+			cwd: this.gitFolder,
+			absolute: true,
+		});
+		if (foldersFile.length > 0) {
+			this.logger.debug(`Importing folders from file ${foldersFile[0]}`);
+			const mappedFolders = jsonParse<{
+				folders: ExportableFolder[];
+			}>(await fsReadFile(foldersFile[0], { encoding: 'utf8' }), {
+				fallbackValue: { folders: [] },
+			});
+			return mappedFolders;
+		}
+		return { folders: [] };
+	}
+
+	async getLocalFoldersAndMappingsFromDb(): Promise<{
+		folders: ExportableFolder[];
+	}> {
+		const localFolders = await this.folderRepository.find({
+			relations: ['parentFolder', 'homeProject'],
+			select: {
+				id: true,
+				name: true,
+				createdAt: true,
+				updatedAt: true,
+				parentFolder: { id: true },
+				homeProject: { id: true },
+			},
+		});
+
+		return {
+			folders: localFolders.map((f) => ({
+				id: f.id,
+				name: f.name,
+				parentFolderId: f.parentFolder?.id ?? null,
+				homeProjectId: f.homeProject.id,
+				createdAt: f.createdAt.toISOString(),
+				updatedAt: f.updatedAt.toISOString(),
+			})),
+		};
+	}
+
 	async getRemoteTagsAndMappingsFromFile(): Promise<{
 		tags: TagEntity[];
 		mappings: WorkflowTagMapping[];
@@ -229,6 +291,10 @@ export class SourceControlImportService {
 		const existingWorkflows = await this.workflowRepository.findByIds(candidateIds, {
 			fields: ['id', 'name', 'versionId', 'active'],
 		});
+
+		const folders = await this.folderRepository.find({ select: ['id'] });
+		const existingFolderIds = folders.map((f) => f.id);
+
 		const allSharedWorkflows = await this.sharedWorkflowRepository.findWithFields(candidateIds, {
 			select: ['workflowId', 'role', 'projectId'],
 		});
@@ -239,7 +305,7 @@ export class SourceControlImportService {
 		// We must iterate over the array and run the whole process workflow by workflow
 		for (const candidate of candidates) {
 			this.logger.debug(`Parsing workflow file ${candidate.file}`);
-			const importedWorkflow = jsonParse<IWorkflowToImport & { owner: string }>(
+			const importedWorkflow = jsonParse<IWorkflowToImport>(
 				await fsReadFile(candidate.file, { encoding: 'utf8' }),
 			);
 			if (!importedWorkflow?.id) {
@@ -247,8 +313,18 @@ export class SourceControlImportService {
 			}
 			const existingWorkflow = existingWorkflows.find((e) => e.id === importedWorkflow.id);
 			importedWorkflow.active = existingWorkflow?.active ?? false;
+
+			const parentFolderId = importedWorkflow.parentFolderId ?? '';
+
 			this.logger.debug(`Updating workflow id ${importedWorkflow.id ?? 'new'}`);
-			const upsertResult = await this.workflowRepository.upsert({ ...importedWorkflow }, ['id']);
+
+			const upsertResult = await this.workflowRepository.upsert(
+				{
+					...importedWorkflow,
+					parentFolder: existingFolderIds.includes(parentFolderId) ? { id: parentFolderId } : null,
+				},
+				['id'],
+			);
 			if (upsertResult?.identifiers?.length !== 1) {
 				throw new UnexpectedError('Failed to upsert workflow', {
 					extra: { workflowId: importedWorkflow.id ?? 'new' },
@@ -440,6 +516,62 @@ export class SourceControlImportService {
 		return mappedTags;
 	}
 
+	async importFoldersFromWorkFolder(user: User, candidate: SourceControlledFile) {
+		let mappedFolders;
+		const projects = await this.projectRepository.find();
+		const personalProject = await this.projectRepository.getPersonalProjectForUserOrFail(user.id);
+
+		try {
+			this.logger.debug(`Importing folders from file ${candidate.file}`);
+			mappedFolders = jsonParse<{
+				folders: ExportableFolder[];
+			}>(await fsReadFile(candidate.file, { encoding: 'utf8' }), {
+				fallbackValue: { folders: [] },
+			});
+		} catch (e) {
+			const error = ensureError(e);
+			this.logger.error(`Failed to import folders from file ${candidate.file}`, { error });
+			return;
+		}
+
+		if (mappedFolders.folders.length === 0) {
+			return;
+		}
+
+		await Promise.all(
+			mappedFolders.folders.map(async (folder) => {
+				const folderCopy = this.folderRepository.create({
+					id: folder.id,
+					name: folder.name,
+					homeProject: {
+						id: projects.find((p) => p.id === folder.homeProjectId)?.id ?? personalProject.id,
+					},
+				});
+
+				await this.folderRepository.upsert(folderCopy, {
+					skipUpdateIfNoValuesChanged: true,
+					conflictPaths: { id: true },
+				});
+			}),
+		);
+
+		// After folders are created, setup the parentFolder relationship
+		await Promise.all(
+			mappedFolders.folders.map(async (folder) => {
+				await this.folderRepository.update(
+					{ id: folder.id },
+					{
+						parentFolder: folder.parentFolderId ? { id: folder.parentFolderId } : null,
+						createdAt: folder.createdAt,
+						updatedAt: folder.updatedAt,
+					},
+				);
+			}),
+		);
+
+		return mappedFolders;
+	}
+
 	async importVariablesFromWorkFolder(
 		candidate: SourceControlledFile,
 		valueOverrides?: {
@@ -531,6 +663,12 @@ export class SourceControlImportService {
 		}
 	}
 
+	async deleteFoldersNotInWorkfolder(candidates: SourceControlledFile[]) {
+		for (const candidate of candidates) {
+			await this.folderRepository.delete(candidate.id);
+		}
+	}
+
 	private async findOrCreateOwnerProject(owner: ResourceOwner): Promise<Project | null> {
 		if (typeof owner === 'string' || owner.type === 'personal') {
 			const email = typeof owner === 'string' ? owner : owner.personalEmail;