feat: Version control mvp (#6271)

* implement basic git service

* cleanup connected prop

* add skeleton of git functions

* initial import/export setup

* split out export service

* refactor and improve export

* begin import

* more commands and basic import

* clean up imports with transactions

* work folder import functions

* reintroduce versionid

* add missing import to pull workfolder

* add get-status endpoint

* add cleanup to disconnect

* add initRepo options

* add more checks and cleanup

* minor cleanup

* refactor prefs

* fix server.ts

* fix sending deleted files

* rename files to ee

* add variable override and fix critical cred import bug

* fix mkdir race condition

* make initRepo default to true

* fix front back integration

* improve connect flow

* add comment to generated ssh key

* fix(editor): use useToast composable

* fix buttons size

* commenting out repo init for now

* fix(editor): update UI logic

* fix(editor): remove console.log

* fix(editor): remove unused ref

* adjust endpoints for improved UI

* fix(editor): add push and pull buttons

* keep or not ssh key

* switching file name to id

* fix(editor): add success messages, fix save button

* fixed faulty diff preventing pull

* fix build

* fix(editor): adding loader to VC components

* removing duplicate exports

* improve conflict finding on push pull

* manage pull conflict

* alternate push pull

* fix pull confirmation

* fix rm and credential export/import

* switch to alternative pull implementation

* fix initial commit

* fix(editor): subscribing to VC store action to refresh lists

* fix(editor): wrap VC store actions with try

* feat: add fine-grained file selection for push action

* fix: close modal after successful push

* fix(editor): VC preferences validation

* fix confirm

* fix: update endpoint to /get-status

* feat: update pull modal override changes message

* fix missing wf error

* undo

* removing connect endpoint

* fix(editor): add button titles

* fix(editor): cleaning up store action

* add version-control/set-read-only protection

* fix(editor): adding set branch readonly

* fix(editor): remove Push button if branch set to readonly

* fix(editor): fix some styles

* fix(editor): remove duplicate and delete actions in WF list when branch is readonly

* fix: load status before opening selective push modal

* fix(editor): extend readonly logic

* add cleanup after failed initRepo

* fix deleted files crashing get-status

* fix n8n-checkbox in staging dialog

* fix(editor): fix loading

* fix(editor): resize buttons

* fix(editor): fix translation

* fix(editor): fix copy text size

* fix(editor): fix copy text size

* fix(editor): add disconnection confirmation

* fix(editor): add disconnection confirmation

* fix(editor): set large buttons

* add public api Pull endpoint

* feat: add refresh ssh key

* return prefs when new keys are generated

* fix(editor): adding readOnly mode to main header

* fix(editor): adding readOnly mode to workflow settings

* improve credential owner import

* add middleware to endpoints

* improve public api error/doc

* do not create branch if one already exists

* update wordings for connect toasts

* fix(editor): updating and separating readonly modes

* fix(editor): fix readonly mode in WF list

* fix(editor): disable elements dragging on canvas in readonly mode (WIP: not working when NodeView page is loaded first)

* fix(editor): fix canvas draggables in readonly env

* fix(editor): remove unused variables

* fix(editor): hide actions in node connections when readonly

* fix(editor): hide actions in node connections when readonly

* fix(editor): disable Save button when readonly

* fix(editor): disable Save settings if no branch is selected

* fix(editor): lint fix

* fix(editor): update snapshots

* fix(editor): replace Loading... text

* fix(editor): reset Loading... text

* reset branchname on disconnect

* fix(editor): adding some translations

* fix(editor): fix unit test

* fix(editor): fix loading

* fix(editor): set settings saved message

* fix(editor): update connection flag

* fix branchName not returning after connect

* temporary (but still breaking) fix for postgres

* fix(editor): adding tooltip to Push/Pull buttons when they're collapsed

* fix(editor): enabled activator in readonly mode

* fix test

* fix(editor): disabling new item addition for workflows in readonly mode

* fix(editor): modify Pull/Push button tooltips

* do not commit empty variables file

---------

Co-authored-by: Michael Auerswald <michael.auerswald@gmail.com>
Co-authored-by: Romain Minaud <romain.minaud@gmail.com>
Co-authored-by: Alex Grozav <alex@grozav.com>
This commit is contained in:
Csaba Tuncsik
2023-05-31 15:01:57 +02:00
committed by GitHub
parent 04cfa548af
commit 1b321416c0
75 changed files with 3720 additions and 460 deletions

View File

@@ -1,108 +1,491 @@
import { Service } from 'typedi';
import { generateSshKeyPair } from './versionControlHelper';
import { VersionControlPreferences } from './types/versionControlPreferences';
import { VERSION_CONTROL_PREFERENCES_DB_KEY } from './constants';
import path from 'path';
import * as Db from '@/Db';
import { jsonParse, LoggerProxy } from 'n8n-workflow';
import type { ValidationError } from 'class-validator';
import { validate } from 'class-validator';
import { versionControlFoldersExistCheck } from './versionControlHelper.ee';
import type { VersionControlPreferences } from './types/versionControlPreferences';
import {
VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER,
VERSION_CONTROL_GIT_FOLDER,
VERSION_CONTROL_README,
VERSION_CONTROL_SSH_FOLDER,
VERSION_CONTROL_SSH_KEY_NAME,
VERSION_CONTROL_TAGS_EXPORT_FILE,
VERSION_CONTROL_VARIABLES_EXPORT_FILE,
VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER,
} from './constants';
import { LoggerProxy } from 'n8n-workflow';
import { VersionControlGitService } from './versionControlGit.service.ee';
import { UserSettings } from 'n8n-core';
import type {
CommitResult,
DiffResult,
FetchResult,
PullResult,
PushResult,
StatusResult,
} from 'simple-git';
import type { ExportResult } from './types/exportResult';
import { VersionControlExportService } from './versionControlExport.service.ee';
import { BadRequestError } from '../../ResponseHelper';
import type { ImportResult } from './types/importResult';
import type { VersionControlPushWorkFolder } from './types/versionControlPushWorkFolder';
import type { VersionControllPullOptions } from './types/versionControlPullWorkFolder';
import type {
VersionControlledFileLocation,
VersionControlledFile,
VersionControlledFileStatus,
VersionControlledFileType,
} from './types/versionControlledFile';
import { VersionControlPreferencesService } from './versionControlPreferences.service.ee';
import { writeFileSync } from 'fs';
@Service()
export class VersionControlService {
private _versionControlPreferences: VersionControlPreferences = new VersionControlPreferences();
private sshKeyName: string;
private sshFolder: string;
private gitFolder: string;
constructor(
private gitService: VersionControlGitService,
private versionControlPreferencesService: VersionControlPreferencesService,
private versionControlExportService: VersionControlExportService,
) {
const userFolder = UserSettings.getUserN8nFolderPath();
this.sshFolder = path.join(userFolder, VERSION_CONTROL_SSH_FOLDER);
this.gitFolder = path.join(userFolder, VERSION_CONTROL_GIT_FOLDER);
this.sshKeyName = path.join(this.sshFolder, VERSION_CONTROL_SSH_KEY_NAME);
}
async init(): Promise<void> {
await this.loadFromDbAndApplyVersionControlPreferences();
this.gitService.resetService();
versionControlFoldersExistCheck([this.gitFolder, this.sshFolder]);
await this.versionControlPreferencesService.loadFromDbAndApplyVersionControlPreferences();
await this.gitService.initService({
versionControlPreferences: this.versionControlPreferencesService.getPreferences(),
gitFolder: this.gitFolder,
sshKeyName: this.sshKeyName,
sshFolder: this.sshFolder,
});
}
public get versionControlPreferences(): VersionControlPreferences {
return {
...this._versionControlPreferences,
privateKey: '(redacted)',
};
async disconnect(options: { keepKeyPair?: boolean } = {}) {
try {
await this.versionControlPreferencesService.setPreferences({
connected: false,
branchName: '',
});
await this.versionControlExportService.deleteRepositoryFolder();
if (!options.keepKeyPair) {
await this.versionControlPreferencesService.deleteKeyPairFiles();
}
this.gitService.resetService();
return this.versionControlPreferencesService.versionControlPreferences;
} catch (error) {
throw Error(`Failed to disconnect from version control: ${(error as Error).message}`);
}
}
public set versionControlPreferences(preferences: Partial<VersionControlPreferences>) {
this._versionControlPreferences = {
connected: preferences.connected ?? this._versionControlPreferences.connected,
authorEmail: preferences.authorEmail ?? this._versionControlPreferences.authorEmail,
authorName: preferences.authorName ?? this._versionControlPreferences.authorName,
branchName: preferences.branchName ?? this._versionControlPreferences.branchName,
branchColor: preferences.branchColor ?? this._versionControlPreferences.branchColor,
branchReadOnly: preferences.branchReadOnly ?? this._versionControlPreferences.branchReadOnly,
privateKey: preferences.privateKey ?? this._versionControlPreferences.privateKey,
publicKey: preferences.publicKey ?? this._versionControlPreferences.publicKey,
repositoryUrl: preferences.repositoryUrl ?? this._versionControlPreferences.repositoryUrl,
};
}
async generateAndSaveKeyPair() {
const keyPair = generateSshKeyPair('ed25519');
if (keyPair.publicKey && keyPair.privateKey) {
await this.setPreferences({ ...keyPair });
async initializeRepository(preferences: VersionControlPreferences) {
if (!this.gitService.git) {
await this.init();
}
LoggerProxy.debug('Initializing repository...');
await this.gitService.initRepository(preferences);
let getBranchesResult;
try {
getBranchesResult = await this.getBranches();
} catch (error) {
if ((error as Error).message.includes('Warning: Permanently added')) {
LoggerProxy.debug('Added repository host to the list of known hosts. Retrying...');
getBranchesResult = await this.getBranches();
} else {
throw error;
}
}
if (getBranchesResult.branches.includes(preferences.branchName)) {
await this.gitService.setBranch(preferences.branchName);
} else {
LoggerProxy.error('Failed to generate key pair');
}
return keyPair;
}
if (getBranchesResult.branches?.length === 0) {
try {
writeFileSync(path.join(this.gitFolder, '/README.md'), VERSION_CONTROL_README);
async validateVersionControlPreferences(
preferences: Partial<VersionControlPreferences>,
): Promise<ValidationError[]> {
const preferencesObject = new VersionControlPreferences(preferences);
const validationResult = await validate(preferencesObject, {
forbidUnknownValues: false,
skipMissingProperties: true,
stopAtFirstError: false,
validationError: { target: false },
});
if (validationResult.length > 0) {
throw new Error(`Invalid version control preferences: ${JSON.stringify(validationResult)}`);
}
// TODO: if repositoryUrl is changed, check if it is valid
// TODO: if branchName is changed, check if it is valid
return validationResult;
}
async setPreferences(
preferences: Partial<VersionControlPreferences>,
saveToDb = true,
): Promise<VersionControlPreferences> {
this.versionControlPreferences = preferences;
if (saveToDb) {
const settingsValue = JSON.stringify(this._versionControlPreferences);
try {
await Db.collections.Settings.save({
key: VERSION_CONTROL_PREFERENCES_DB_KEY,
value: settingsValue,
loadOnStartup: true,
});
} catch (error) {
throw new Error(`Failed to save version control preferences: ${(error as Error).message}`);
}
}
return this.versionControlPreferences;
}
async loadFromDbAndApplyVersionControlPreferences(): Promise<
VersionControlPreferences | undefined
> {
const loadedPreferences = await Db.collections.Settings.findOne({
where: { key: VERSION_CONTROL_PREFERENCES_DB_KEY },
});
if (loadedPreferences) {
try {
const preferences = jsonParse<VersionControlPreferences>(loadedPreferences.value);
if (preferences) {
await this.setPreferences(preferences, false);
return preferences;
await this.gitService.stage(new Set<string>(['README.md']));
await this.gitService.commit('Initial commit');
await this.gitService.push({
branch: preferences.branchName,
force: true,
});
getBranchesResult = await this.getBranches();
} catch (fileError) {
LoggerProxy.error(`Failed to create initial commit: ${(fileError as Error).message}`);
}
} catch (error) {
LoggerProxy.warn(
`Could not parse Version Control settings from database: ${(error as Error).message}`,
);
} else {
await this.versionControlPreferencesService.setPreferences({
branchName: '',
connected: true,
});
}
}
return getBranchesResult;
}
async export() {
const result: {
tags: ExportResult | undefined;
credentials: ExportResult | undefined;
variables: ExportResult | undefined;
workflows: ExportResult | undefined;
} = {
credentials: undefined,
tags: undefined,
variables: undefined,
workflows: undefined,
};
try {
// comment next line if needed
await this.versionControlExportService.cleanWorkFolder();
result.tags = await this.versionControlExportService.exportTagsToWorkFolder();
result.variables = await this.versionControlExportService.exportVariablesToWorkFolder();
result.workflows = await this.versionControlExportService.exportWorkflowsToWorkFolder();
result.credentials = await this.versionControlExportService.exportCredentialsToWorkFolder();
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}
return result;
}
async import(options: VersionControllPullOptions): Promise<ImportResult | undefined> {
try {
return await this.versionControlExportService.importFromWorkFolder(options);
} catch (error) {
throw new BadRequestError((error as { message: string }).message);
}
}
async getBranches(): Promise<{ branches: string[]; currentBranch: string }> {
// fetch first to get include remote changes
await this.gitService.fetch();
return this.gitService.getBranches();
}
async setBranch(branch: string): Promise<{ branches: string[]; currentBranch: string }> {
await this.versionControlPreferencesService.setPreferences({
branchName: branch,
connected: true,
});
return this.gitService.setBranch(branch);
}
// will reset the branch to the remote branch and pull
// this will discard all local changes
async resetWorkfolder(options: VersionControllPullOptions): Promise<ImportResult | undefined> {
const currentBranch = await this.gitService.getCurrentBranch();
await this.versionControlExportService.cleanWorkFolder();
await this.gitService.resetBranch({
hard: true,
target: currentBranch.remote,
});
await this.gitService.pull();
if (options.importAfterPull) {
return this.import(options);
}
return;
}
async pushWorkfolder(
options: VersionControlPushWorkFolder,
): Promise<PushResult | VersionControlledFile[]> {
if (this.versionControlPreferencesService.isBranchReadOnly()) {
throw new BadRequestError('Cannot push onto read-only branch.');
}
if (!options.skipDiff) {
const diffResult = await this.getStatus();
const possibleConflicts = diffResult?.filter((file) => file.conflict);
if (possibleConflicts?.length > 0 && options.force !== true) {
await this.unstage();
return diffResult;
}
}
await this.unstage();
await this.stage(options);
await this.gitService.commit(options.message ?? 'Updated Workfolder');
return this.gitService.push({
branch: this.versionControlPreferencesService.getBranchName(),
force: options.force ?? false,
});
}
// async pushWorkfolder(
// options: VersionControlPushWorkFolder,
// ): Promise<PushResult | VersionControlledFile[]> {
// await this.gitService.fetch();
// await this.export(); // refresh workfolder
// await this.stage(options);
// await this.gitService.commit(options.message ?? 'Updated Workfolder');
// return this.gitService.push({
// branch: this.versionControlPreferencesService.getBranchName(),
// force: options.force ?? false,
// });
// }
// TODO: Alternate implementation for pull
// async pullWorkfolder(
// options: VersionControllPullOptions,
// ): Promise<ImportResult | VersionControlledFile[] | PullResult | undefined> {
// const diffResult = await this.getStatus();
// const possibleConflicts = diffResult?.filter((file) => file.conflict);
// if (possibleConflicts?.length > 0 || options.force === true) {
// await this.unstage();
// if (options.force === true) {
// return this.resetWorkfolder(options);
// } else {
// return diffResult;
// }
// }
// const pullResult = await this.gitService.pull();
// if (options.importAfterPull) {
// return this.import(options);
// }
// return pullResult;
// }
async pullWorkfolder(
options: VersionControllPullOptions,
): Promise<ImportResult | StatusResult | undefined> {
await this.resetWorkfolder({
importAfterPull: false,
userId: options.userId,
force: false,
});
await this.export(); // refresh workfolder
const status = await this.gitService.status();
if (status.modified.length > 0 && options.force !== true) {
return status;
}
await this.resetWorkfolder({ ...options, importAfterPull: false });
if (options.importAfterPull) {
return this.import(options);
}
return;
}
async stage(
options: Pick<VersionControlPushWorkFolder, 'fileNames' | 'credentialIds' | 'workflowIds'>,
): Promise<{ staged: string[] } | string> {
const { fileNames, credentialIds, workflowIds } = options;
const status = await this.gitService.status();
let mergedFileNames = new Set<string>();
fileNames?.forEach((e) => mergedFileNames.add(e));
credentialIds?.forEach((e) =>
mergedFileNames.add(this.versionControlExportService.getCredentialsPath(e)),
);
workflowIds?.forEach((e) =>
mergedFileNames.add(this.versionControlExportService.getWorkflowPath(e)),
);
if (mergedFileNames.size === 0) {
mergedFileNames = new Set<string>([
...status.not_added,
...status.created,
...status.modified,
]);
}
const deletedFiles = new Set<string>(status.deleted);
deletedFiles.forEach((e) => mergedFileNames.delete(e));
await this.unstage();
const stageResult = await this.gitService.stage(mergedFileNames, deletedFiles);
if (!stageResult) {
const statusResult = await this.gitService.status();
return { staged: statusResult.staged };
}
return stageResult;
}
async unstage(): Promise<StatusResult | string> {
const stageResult = await this.gitService.resetBranch();
if (!stageResult) {
return this.gitService.status();
}
return stageResult;
}
async status(): Promise<StatusResult> {
return this.gitService.status();
}
private async fileNameToVersionControlledFile(
fileName: string,
location: VersionControlledFileLocation,
statusResult: StatusResult,
): Promise<VersionControlledFile | undefined> {
let id: string | undefined = undefined;
let name = '';
let conflict = false;
let status: VersionControlledFileStatus = 'unknown';
let type: VersionControlledFileType = 'file';
// initialize status from git status result
if (statusResult.not_added.find((e) => e === fileName)) status = 'new';
else if (statusResult.conflicted.find((e) => e === fileName)) {
status = 'conflicted';
conflict = true;
} else if (statusResult.created.find((e) => e === fileName)) status = 'created';
else if (statusResult.deleted.find((e) => e === fileName)) status = 'deleted';
else if (statusResult.modified.find((e) => e === fileName)) status = 'modified';
if (fileName.startsWith(VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER)) {
type = 'workflow';
if (status === 'deleted') {
id = fileName
.replace(VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER, '')
.replace(/[\/,\\]/, '')
.replace('.json', '');
if (location === 'remote') {
const existingWorkflow = await Db.collections.Workflow.find({
where: { id },
});
if (existingWorkflow?.length > 0) {
name = existingWorkflow[0].name;
}
} else {
name = '(deleted)';
}
} else {
const workflow = await this.versionControlExportService.getWorkflowFromFile(fileName);
if (!workflow?.id) {
if (location === 'local') {
return;
}
id = fileName
.replace(VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER + '/', '')
.replace('.json', '');
status = 'created';
} else {
id = workflow.id;
name = workflow.name;
}
}
}
if (fileName.startsWith(VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER)) {
type = 'credential';
if (status === 'deleted') {
id = fileName
.replace(VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER, '')
.replace(/[\/,\\]/, '')
.replace('.json', '');
if (location === 'remote') {
const existingCredential = await Db.collections.Credentials.find({
where: { id },
});
if (existingCredential?.length > 0) {
name = existingCredential[0].name;
}
} else {
name = '(deleted)';
}
} else {
const credential = await this.versionControlExportService.getCredentialFromFile(fileName);
if (!credential?.id) {
if (location === 'local') {
return;
}
id = fileName
.replace(VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER + '/', '')
.replace('.json', '');
status = 'created';
} else {
id = credential.id;
name = credential.name;
}
}
}
if (fileName.startsWith(VERSION_CONTROL_VARIABLES_EXPORT_FILE)) {
id = 'variables';
name = 'variables';
type = 'variables';
}
if (fileName.startsWith(VERSION_CONTROL_TAGS_EXPORT_FILE)) {
id = 'tags';
name = 'tags';
type = 'tags';
}
if (!id) return;
return {
file: fileName,
id,
name,
type,
status,
location,
conflict,
};
}
async getStatus(): Promise<VersionControlledFile[]> {
await this.export();
await this.stage({});
await this.gitService.fetch();
const versionControlledFiles: VersionControlledFile[] = [];
const diffRemote = await this.gitService.diffRemote();
const diffLocal = await this.gitService.diffLocal();
const status = await this.gitService.status();
await Promise.all([
...(diffRemote?.files.map(async (e) => {
const resolvedFile = await this.fileNameToVersionControlledFile(e.file, 'remote', status);
if (resolvedFile) {
versionControlledFiles.push(resolvedFile);
}
}) ?? []),
...(diffLocal?.files.map(async (e) => {
const resolvedFile = await this.fileNameToVersionControlledFile(e.file, 'local', status);
if (resolvedFile) {
versionControlledFiles.push(resolvedFile);
}
}) ?? []),
]);
versionControlledFiles.forEach((e, index, array) => {
const similarItems = array.filter(
(f) => f.type === e.type && (f.file === e.file || f.id === e.id),
);
if (similarItems.length > 1) {
similarItems.forEach((item) => {
item.conflict = true;
});
}
});
return versionControlledFiles;
}
// #region Version Control Test Functions
//TODO: SEPARATE FUNCTIONS FOR DEVELOPMENT ONLY
//TODO: REMOVE THESE FUNCTIONS AFTER TESTING
async commit(message?: string): Promise<CommitResult> {
return this.gitService.commit(message ?? 'Updated Workfolder');
}
async fetch(): Promise<FetchResult> {
return this.gitService.fetch();
}
async diff(): Promise<DiffResult> {
return this.gitService.diff();
}
async pull(): Promise<PullResult> {
return this.gitService.pull();
}
async push(force = false): Promise<PushResult> {
return this.gitService.push({
branch: this.versionControlPreferencesService.getBranchName(),
force,
});
}
// #endregion
}