feat(core): Add folder synchronization to environments feature (#14005)

packages/cli/src/environments.ee/source-control/source-control.service.ee.ts

@@ -13,6 +13,7 @@ import type { PushResult } from 'simple-git';
 import type { TagEntity } from '@/databases/entities/tag-entity';
 import type { User } from '@/databases/entities/user';
 import type { Variables } from '@/databases/entities/variables';
+import { FolderRepository } from '@/databases/repositories/folder.repository';
 import { TagRepository } from '@/databases/repositories/tag.repository';
 import { BadRequestError } from '@/errors/response-errors/bad-request.error';
 import { EventService } from '@/events/event.service';
@@ -25,6 +26,7 @@ import {
 import { SourceControlExportService } from './source-control-export.service.ee';
 import { SourceControlGitService } from './source-control-git.service.ee';
 import {
+	getFoldersPath,
 	getTagsPath,
 	getTrackingInformationFromPostPushResult,
 	getTrackingInformationFromPrePushResult,
@@ -36,6 +38,7 @@ import {
 import { SourceControlImportService } from './source-control-import.service.ee';
 import { SourceControlPreferencesService } from './source-control-preferences.service.ee';
 import type { ExportableCredential } from './types/exportable-credential';
+import type { ExportableFolder } from './types/exportable-folders';
 import type { ImportResult } from './types/import-result';
 import type { SourceControlGetStatus } from './types/source-control-get-status';
 import type { SourceControlPreferences } from './types/source-control-preferences';
@@ -57,6 +60,7 @@ export class SourceControlService {
 		private sourceControlExportService: SourceControlExportService,
 		private sourceControlImportService: SourceControlImportService,
 		private tagRepository: TagRepository,
+		private folderRepository: FolderRepository,
 		private readonly eventService: EventService,
 	) {
 		const { gitFolder, sshFolder, sshKeyName } = sourceControlPreferencesService;
@@ -255,13 +259,20 @@ export class SourceControlService {
 
 		const filesToBePushed = new Set<string>();
 		const filesToBeDeleted = new Set<string>();
-		filesToPush.forEach((e) => {
-			if (e.status !== 'deleted') {
-				filesToBePushed.add(e.file);
-			} else {
-				filesToBeDeleted.add(e.file);
-			}
-		});
+
+		/*
+			Exclude tags, variables and folders JSON file from being deleted as
+			we keep track of them in a single file unlike workflows and credentials
+		*/
+		filesToPush
+			.filter((f) => ['workflow', 'credential'].includes(f.type))
+			.forEach((e) => {
+				if (e.status !== 'deleted') {
+					filesToBePushed.add(e.file);
+				} else {
+					filesToBeDeleted.add(e.file);
+				}
+			});
 
 		this.sourceControlExportService.rmFilesFromExportFolder(filesToBeDeleted);
 
@@ -284,11 +295,21 @@ export class SourceControlService {
 			});
 		}
 
-		if (filesToPush.find((e) => e.type === 'tags')) {
+		const tagChanges = filesToPush.find((e) => e.type === 'tags');
+		if (tagChanges) {
+			filesToBePushed.add(tagChanges.file);
 			await this.sourceControlExportService.exportTagsToWorkFolder();
 		}
 
-		if (filesToPush.find((e) => e.type === 'variables')) {
+		const folderChanges = filesToPush.find((e) => e.type === 'folders');
+		if (folderChanges) {
+			filesToBePushed.add(folderChanges.file);
+			await this.sourceControlExportService.exportFoldersToWorkFolder();
+		}
+
+		const variablesChanges = filesToPush.find((e) => e.type === 'variables');
+		if (variablesChanges) {
+			filesToBePushed.add(variablesChanges.file);
 			await this.sourceControlExportService.exportVariablesToWorkFolder();
 		}
 
@@ -354,6 +375,14 @@ export class SourceControlService {
 		return files.find((e) => e.type === 'variables' && e.status !== 'deleted');
 	}
 
+	private getFoldersToImport(files: SourceControlledFile[]): SourceControlledFile | undefined {
+		return files.find((e) => e.type === 'folders' && e.status !== 'deleted');
+	}
+
+	private getFoldersToDelete(files: SourceControlledFile[]): SourceControlledFile[] {
+		return files.filter((e) => e.type === 'folders' && e.status === 'deleted');
+	}
+
 	private getVariablesToDelete(files: SourceControlledFile[]): SourceControlledFile[] {
 		return files.filter((e) => e.type === 'variables' && e.status === 'deleted');
 	}
@@ -381,6 +410,12 @@ export class SourceControlService {
 			}
 		}
 
+		// Make sure the folders get processed first as the workflows depend on them
+		const foldersToBeImported = this.getFoldersToImport(statusResult);
+		if (foldersToBeImported) {
+			await this.sourceControlImportService.importFoldersFromWorkFolder(user, foldersToBeImported);
+		}
+
 		const workflowsToBeImported = this.getWorkflowsToImport(statusResult);
 		await this.sourceControlImportService.importWorkflowFromWorkFolder(
 			workflowsToBeImported,
@@ -416,6 +451,9 @@ export class SourceControlService {
 		const variablesToBeDeleted = this.getVariablesToDelete(statusResult);
 		await this.sourceControlImportService.deleteVariablesNotInWorkfolder(variablesToBeDeleted);
 
+		const foldersToBeDeleted = this.getFoldersToDelete(statusResult);
+		await this.sourceControlImportService.deleteFoldersNotInWorkfolder(foldersToBeDeleted);
+
 		// #region Tracking Information
 		this.eventService.emit(
 			'source-control-user-finished-pull-ui',
@@ -469,6 +507,9 @@ export class SourceControlService {
 			mappingsMissingInRemote,
 		} = await this.getStatusTagsMappings(options, sourceControlledFiles);
 
+		const { foldersMissingInLocal, foldersMissingInRemote, foldersModifiedInEither } =
+			await this.getStatusFoldersMapping(options, sourceControlledFiles);
+
 		// #region Tracking Information
 		if (options.direction === 'push') {
 			this.eventService.emit(
@@ -501,6 +542,9 @@ export class SourceControlService {
 				tagsModifiedInEither,
 				mappingsMissingInLocal,
 				mappingsMissingInRemote,
+				foldersMissingInLocal,
+				foldersMissingInRemote,
+				foldersModifiedInEither,
 				sourceControlledFiles,
 			};
 		} else {
@@ -526,7 +570,9 @@ export class SourceControlService {
 		const wfModifiedInEither: SourceControlWorkflowVersionId[] = [];
 		wfLocalVersionIds.forEach((local) => {
 			const mismatchingIds = wfRemoteVersionIds.find(
-				(remote) => remote.id === local.id && remote.versionId !== local.versionId,
+				(remote) =>
+					remote.id === local.id &&
+					(remote.versionId !== local.versionId || remote.parentFolderId !== local.parentFolderId),
 			);
 			let name = (options?.preferLocalVersion ? local?.name : mismatchingIds?.name) ?? 'Workflow';
 			if (local.name && mismatchingIds?.name && local.name !== mismatchingIds.name) {
@@ -840,6 +886,88 @@ export class SourceControlService {
 		};
 	}
 
+	private async getStatusFoldersMapping(
+		options: SourceControlGetStatus,
+		sourceControlledFiles: SourceControlledFile[],
+	) {
+		const lastUpdatedFolder = await this.folderRepository.find({
+			order: { updatedAt: 'DESC' },
+			take: 1,
+			select: ['updatedAt'],
+		});
+
+		const foldersMappingsRemote =
+			await this.sourceControlImportService.getRemoteFoldersAndMappingsFromFile();
+		const foldersMappingsLocal =
+			await this.sourceControlImportService.getLocalFoldersAndMappingsFromDb();
+
+		const foldersMissingInLocal = foldersMappingsRemote.folders.filter(
+			(remote) => foldersMappingsLocal.folders.findIndex((local) => local.id === remote.id) === -1,
+		);
+
+		const foldersMissingInRemote = foldersMappingsLocal.folders.filter(
+			(local) => foldersMappingsRemote.folders.findIndex((remote) => remote.id === local.id) === -1,
+		);
+
+		const foldersModifiedInEither: ExportableFolder[] = [];
+		foldersMappingsLocal.folders.forEach((local) => {
+			const mismatchingIds = foldersMappingsRemote.folders.find(
+				(remote) =>
+					remote.id === local.id &&
+					(remote.name !== local.name || remote.parentFolderId !== local.parentFolderId),
+			);
+
+			if (!mismatchingIds) {
+				return;
+			}
+			foldersModifiedInEither.push(options.preferLocalVersion ? local : mismatchingIds);
+		});
+
+		foldersMissingInLocal.forEach((item) => {
+			sourceControlledFiles.push({
+				id: item.id,
+				name: item.name,
+				type: 'folders',
+				status: options.direction === 'push' ? 'deleted' : 'created',
+				location: options.direction === 'push' ? 'local' : 'remote',
+				conflict: false,
+				file: getFoldersPath(this.gitFolder),
+				updatedAt: lastUpdatedFolder[0]?.updatedAt.toISOString(),
+			});
+		});
+		foldersMissingInRemote.forEach((item) => {
+			sourceControlledFiles.push({
+				id: item.id,
+				name: item.name,
+				type: 'folders',
+				status: options.direction === 'push' ? 'created' : 'deleted',
+				location: options.direction === 'push' ? 'local' : 'remote',
+				conflict: options.direction === 'push' ? false : true,
+				file: getFoldersPath(this.gitFolder),
+				updatedAt: lastUpdatedFolder[0]?.updatedAt.toISOString(),
+			});
+		});
+
+		foldersModifiedInEither.forEach((item) => {
+			sourceControlledFiles.push({
+				id: item.id,
+				name: item.name,
+				type: 'folders',
+				status: 'modified',
+				location: options.direction === 'push' ? 'local' : 'remote',
+				conflict: true,
+				file: getFoldersPath(this.gitFolder),
+				updatedAt: lastUpdatedFolder[0]?.updatedAt.toISOString(),
+			});
+		});
+
+		return {
+			foldersMissingInLocal,
+			foldersMissingInRemote,
+			foldersModifiedInEither,
+		};
+	}
+
 	async setGitUserDetails(
 		name = SOURCE_CONTROL_DEFAULT_NAME,
 		email = SOURCE_CONTROL_DEFAULT_EMAIL,