From 1b321416c0ba5371e0016398ae660ce298b8cdd6 Mon Sep 17 00:00:00 2001 From: Csaba Tuncsik Date: Wed, 31 May 2023 15:01:57 +0200 Subject: [PATCH] feat: Version control mvp (#6271) * implement basic git service * cleanup connected prop * add skeleton of git functions * initial import/export setup * split out export service * refactor and improve export * begin import * more commands and basic import * clean up imports with transactions * work folder import functions * reintroduce versionid * add missing import to pull workfolder * add get-status endpoint * add cleanup to disconnect * add initRepo options * add more checks and cleanup * minor cleanup * refactor prefs * fix server.ts * fix sending deleted files * rename files to ee * add variable override and fix critical cred import bug * fix mkdir race condition * make initRepo default to true * fix front back integration * improve connect flow * add comment to generated ssh key * fix(editor): use useToast composable * fix buttons size * commenting out repo init for now * fix(editor): update UI logic * fix(editor): remove console.log * fix(editor): remove unused ref * adjust endpoints for improved UI * fix(editor): add push and pull buttons * keep or not ssh key * switching file name to id * fix(editor): add success messages, fix save button * fixed faulty diff preventing pull * fix build * fix(editor): adding loader to VC components * removing duplicate exports * improve conflict finding on push pull * manage pull conflict * alternate push pull * fix pull confirmation * fix rm and credential export/import * switch to alternative pull implementation * fix initial commit * fix(editor): subscribing to VC store action to refresh lists * fix(editor): wrap VC store actions with try * feat: add fine-grained file selection for push action * fix: close modal after successful push * fix(editor): VC preferences validation * fix confirm * fix: update endpoint to /get-status * feat: update pull modal override changes message * fix missing wf error * undo * removing connect endpoint * fix(editor): add button titles * fix(editor): cleaning up store action * add version-control/set-read-only protection * fix(editor): adding set branch readonly * fix(editor): remove Push button if branch set to readonly * fix(editor): fix some styles * fix(editor): remove duplicate and delete actions in WF list when branch is readonly * fix: load status before opening selective push modal * fix(editor): extend readonly logic * add cleanup after failed initRepo * fix deleted files crashing get-status * fix n8n-checkbox in staging dialog * fix(editor): fix loading * fix(editor): resize buttons * fix(editor): fix translation * fix(editor): fix copy text size * fix(editor): fix copy text size * fix(editor): add disconnection confirmation * fix(editor): add disconnection confirmation * fix(editor): set large buttons * add public api Pull endpoint * feat: add refresh ssh key * return prefs when new keys are generated * fix(editor): adding readOnly mode to main header * fix(editor): adding readOnly mode to workflow settings * improve credential owner import * add middleware to endpoints * improve public api error/doc * do not create branch if one already exists * update wordings for connect toasts * fix(editor): updating and separating readonly modes * fix(editor): fix readonly mode in WF list * fix(editor): disable elements dragging on canvas in readonly mode (WIP: not working when NodeView page is loaded first) * fix(editor): fix canvas draggables in readonly env * fix(editor): remove unused variables * fix(editor): hide actions in node connections when readonly * fix(editor): hide actions in node connections when readonly * fix(editor): disable Save button when readonly * fix(editor): disable Save settings if no branch is selected * fix(editor): lint fix * fix(editor): update snapshots * fix(editor): replace Loading... text * fix(editor): reset Loading... text * reset branchname on disconnect * fix(editor): adding some translations * fix(editor): fix unit test * fix(editor): fix loading * fix(editor): set settings saved message * fix(editor): update connection flag * fix branchName not returning after connect * temporary (but still breaking) fix for postgres * fix(editor): adding tooltip to Push/Pull buttons when they're collapsed * fix(editor): enabled activator in readonly mode * fix test * fix(editor): disabling new item addition for workflows in readonly mode * fix(editor): modify Pull/Push button tooltips * do not commit empty variables file --------- Co-authored-by: Michael Auerswald Co-authored-by: Romain Minaud Co-authored-by: Alex Grozav --- packages/cli/package.json | 5 + packages/cli/src/PublicApi/types.d.ts | 10 + .../spec/paths/versionControl.yml | 25 + .../spec/schemas/importResult.yml | 55 ++ .../versionControl/spec/schemas/pull.yml | 8 + .../versionControl/versionControl.handler.ts | 47 ++ packages/cli/src/PublicApi/v1/openapi.yml | 32 +- .../v1/shared/spec/responses/_index.yml | 8 +- .../v1/shared/spec/responses/conflict.yml | 1 + .../v1/shared/spec/schemas/_index.yml | 26 +- packages/cli/src/Server.ts | 6 +- .../environments/versionControl/constants.ts | 14 + ... => versionControlEnabledMiddleware.ee.ts} | 10 +- .../versionControl/types/exportResult.ts | 9 + .../types/exportableCredential.ts | 8 + .../types/exportableWorkflow.ts | 13 + .../versionControl/types/importResult.ts | 13 + .../versionControl/types/requests.ts | 16 + .../types/versionControlCommit.ts | 6 + .../types/versionControlDisconnect.ts | 7 + .../types/versionControlPreferences.ts | 27 +- .../types/versionControlPullWorkFolder.ts | 29 + .../types/versionControlPush.ts | 7 + .../types/versionControlPushWorkFolder.ts | 27 + .../types/versionControlSetBranch.ts | 6 + .../types/versionControlSetReadOnly.ts | 6 + .../types/versionControlStage.ts | 15 + .../types/versionControlledFile.ts | 19 + .../versionControl.controller.ee.ts | 315 +++++++- .../versionControl.service.ee.ts | 563 ++++++++++++--- .../versionControlExport.service.ee.ts | 674 ++++++++++++++++++ .../versionControlGit.service.ee.ts | 330 +++++++++ ...olHelper.ts => versionControlHelper.ee.ts} | 29 +- .../versionControlPreferences.service.ee.ts | 191 +++++ .../environments/VersionControl.test.ts | 12 +- packages/cli/test/integration/shared/utils.ts | 4 +- packages/cli/test/unit/VersionControl.test.ts | 2 +- .../src/components/N8nButton/Button.vue | 8 +- .../__snapshots__/Button.spec.ts.snap | 12 +- packages/editor-ui/src/App.vue | 34 +- packages/editor-ui/src/Interface.ts | 33 +- packages/editor-ui/src/api/versionControl.ts | 68 +- .../editor-ui/src/components/CopyInput.vue | 2 +- .../src/components/InlineTextEdit.vue | 30 +- .../src/components/MainHeader/MainHeader.vue | 13 +- .../components/MainHeader/WorkflowDetails.vue | 101 +-- .../editor-ui/src/components/MainSidebar.vue | 78 +- .../components/MainSidebarVersionControl.vue | 154 ++++ packages/editor-ui/src/components/Modals.vue | 10 + packages/editor-ui/src/components/Node.vue | 4 +- .../components/VersionControlPushModal.ee.vue | 254 +++++++ .../editor-ui/src/components/WorkflowCard.vue | 31 +- .../src/components/WorkflowSettings.vue | 30 +- packages/editor-ui/src/composables/index.ts | 1 + .../src/composables/useLoadingService.ts | 45 ++ packages/editor-ui/src/constants.ts | 2 + .../editor-ui/src/mixins/genericHelpers.ts | 4 + .../src/plugins/i18n/locales/en.json | 48 +- packages/editor-ui/src/plugins/icons/index.ts | 4 + packages/editor-ui/src/stores/canvas.store.ts | 22 +- packages/editor-ui/src/stores/index.ts | 2 + packages/editor-ui/src/stores/ui.store.ts | 4 + .../src/stores/versionControl.store.ts | 90 ++- .../src/utils/__tests__/userUtils.test.ts | 3 +- .../__tests__/versionControlUtils.test.ts | 56 ++ packages/editor-ui/src/utils/index.ts | 1 + .../src/utils/versionControlUtils.ts | 27 + .../editor-ui/src/views/CredentialsView.vue | 24 +- packages/editor-ui/src/views/NodeView.vue | 60 +- .../src/views/SettingsVersionControl.vue | 275 +++++-- .../editor-ui/src/views/VariablesView.vue | 26 +- .../editor-ui/src/views/WorkflowsView.vue | 21 +- .../__tests__/SettingsVersionControl.test.ts | 2 +- packages/workflow/src/Interfaces.ts | 1 + pnpm-lock.yaml | 25 + 75 files changed, 3720 insertions(+), 460 deletions(-) create mode 100644 packages/cli/src/PublicApi/v1/handlers/versionControl/spec/paths/versionControl.yml create mode 100644 packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/importResult.yml create mode 100644 packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/pull.yml create mode 100644 packages/cli/src/PublicApi/v1/handlers/versionControl/versionControl.handler.ts create mode 100644 packages/cli/src/PublicApi/v1/shared/spec/responses/conflict.yml rename packages/cli/src/environments/versionControl/middleware/{versionControlEnabledMiddleware.ts => versionControlEnabledMiddleware.ee.ts} (55%) create mode 100644 packages/cli/src/environments/versionControl/types/exportResult.ts create mode 100644 packages/cli/src/environments/versionControl/types/exportableCredential.ts create mode 100644 packages/cli/src/environments/versionControl/types/exportableWorkflow.ts create mode 100644 packages/cli/src/environments/versionControl/types/importResult.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlCommit.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlDisconnect.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlPullWorkFolder.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlPush.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlPushWorkFolder.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlSetBranch.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlSetReadOnly.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlStage.ts create mode 100644 packages/cli/src/environments/versionControl/types/versionControlledFile.ts create mode 100644 packages/cli/src/environments/versionControl/versionControlExport.service.ee.ts create mode 100644 packages/cli/src/environments/versionControl/versionControlGit.service.ee.ts rename packages/cli/src/environments/versionControl/{versionControlHelper.ts => versionControlHelper.ee.ts} (67%) create mode 100644 packages/cli/src/environments/versionControl/versionControlPreferences.service.ee.ts create mode 100644 packages/editor-ui/src/components/MainSidebarVersionControl.vue create mode 100644 packages/editor-ui/src/components/VersionControlPushModal.ee.vue create mode 100644 packages/editor-ui/src/composables/useLoadingService.ts create mode 100644 packages/editor-ui/src/utils/__tests__/versionControlUtils.test.ts create mode 100644 packages/editor-ui/src/utils/versionControlUtils.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 7125f86a85..02817de2e3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -78,6 +78,7 @@ "@types/jsonwebtoken": "^9.0.1", "@types/localtunnel": "^1.9.0", "@types/lodash.debounce": "^4.0.7", + "@types/lodash.difference": "^4", "@types/lodash.get": "^4.4.6", "@types/lodash.intersection": "^4.4.7", "@types/lodash.iteratee": "^4.7.7", @@ -91,6 +92,7 @@ "@types/lodash.uniq": "^4.5.7", "@types/lodash.uniqby": "^4.7.7", "@types/lodash.unset": "^4.5.7", + "@types/lodash.without": "^4.4.7", "@types/parseurl": "^1.3.1", "@types/passport-jwt": "^3.0.6", "@types/psl": "^1.1.0", @@ -159,6 +161,7 @@ "jwks-rsa": "^3.0.1", "ldapts": "^4.2.6", "localtunnel": "^2.0.0", + "lodash.difference": "^4", "lodash.get": "^4.4.2", "lodash.intersection": "^4.4.0", "lodash.iteratee": "^4.7.0", @@ -172,6 +175,7 @@ "lodash.uniq": "^4.5.0", "lodash.uniqby": "^4.7.0", "lodash.unset": "^4.5.2", + "lodash.without": "^4.4.0", "luxon": "^3.3.0", "mysql2": "~2.3.3", "n8n-core": "workspace:*", @@ -197,6 +201,7 @@ "samlify": "^2.8.9", "semver": "^7.3.8", "shelljs": "^0.8.5", + "simple-git": "^3.17.0", "source-map-support": "^0.5.21", "sqlite3": "^5.1.6", "sse-channel": "^4.0.0", diff --git a/packages/cli/src/PublicApi/types.d.ts b/packages/cli/src/PublicApi/types.d.ts index b1bd59d6dc..f5795b877b 100644 --- a/packages/cli/src/PublicApi/types.d.ts +++ b/packages/cli/src/PublicApi/types.d.ts @@ -163,6 +163,16 @@ export interface IJsonSchema { required: string[]; } +export class VersionControlPull { + force?: boolean; + + variables?: { [key: string]: string }; +} + +export declare namespace PublicVersionControlRequest { + type Pull = AuthenticatedRequest<{}, {}, VersionControlPull, {}>; +} + // ---------------------------------- // /audit // ---------------------------------- diff --git a/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/paths/versionControl.yml b/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/paths/versionControl.yml new file mode 100644 index 0000000000..0d0c8d78d2 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/paths/versionControl.yml @@ -0,0 +1,25 @@ +post: + x-eov-operation-id: pull + x-eov-operation-handler: v1/handlers/versionControl/versionControl.handler + tags: + - VersionControl + summary: Pull changes from the remote repository + description: Requires the Version Control feature to be licensed and connected to a repository. + requestBody: + description: Pull options + required: true + content: + application/json: + schema: + $ref: "../schemas/pull.yml" + responses: + "200": + description: Import result + content: + application/json: + schema: + $ref: "../schemas/importResult.yml" + "400": + $ref: "../../../../shared/spec/responses/badRequest.yml" + "409": + $ref: "../../../../shared/spec/responses/conflict.yml" diff --git a/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/importResult.yml b/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/importResult.yml new file mode 100644 index 0000000000..ea726f4249 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/importResult.yml @@ -0,0 +1,55 @@ +type: object +additionalProperties: true +properties: + variables: + type: object + properties: + added: + type: array + items: + type: string + changed: + type: array + items: + type: string + credentials: + type: array + items: + type: object + properties: + id: + type: string + name: + type: string + type: + type: string + workflows: + type: array + items: + type: object + properties: + id: + type: string + name: + type: string + tags: + type: object + properties: + tags: + type: array + items: + type: object + properties: + id: + type: string + name: + type: string + mappings: + type: array + items: + type: object + properties: + workflowId: + type: string + tagId: + type: string diff --git a/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/pull.yml b/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/pull.yml new file mode 100644 index 0000000000..0b94cb4646 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/versionControl/spec/schemas/pull.yml @@ -0,0 +1,8 @@ +type: object +properties: + force: + type: boolean + example: true + variables: + type: object + example: { "foo": "bar" } diff --git a/packages/cli/src/PublicApi/v1/handlers/versionControl/versionControl.handler.ts b/packages/cli/src/PublicApi/v1/handlers/versionControl/versionControl.handler.ts new file mode 100644 index 0000000000..90cead0fe9 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/handlers/versionControl/versionControl.handler.ts @@ -0,0 +1,47 @@ +import type express from 'express'; +import type { StatusResult } from 'simple-git'; +import type { PublicVersionControlRequest } from '../../../types'; +import { authorize } from '../../shared/middlewares/global.middleware'; +import type { ImportResult } from '@/environments/versionControl/types/importResult'; +import Container from 'typedi'; +import { VersionControlService } from '@/environments/versionControl/versionControl.service.ee'; +import { VersionControlPreferencesService } from '@/environments/versionControl/versionControlPreferences.service.ee'; +import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper.ee'; + +export = { + pull: [ + authorize(['owner', 'member']), + async ( + req: PublicVersionControlRequest.Pull, + res: express.Response, + ): Promise> => { + const versionControlPreferencesService = Container.get(VersionControlPreferencesService); + if (!isVersionControlLicensed()) { + return res + .status(401) + .json({ status: 'Error', message: 'Version Control feature is not licensed' }); + } + if (!versionControlPreferencesService.isVersionControlConnected()) { + return res + .status(400) + .json({ status: 'Error', message: 'Version Control is not connected to a repository' }); + } + try { + const versionControlService = Container.get(VersionControlService); + const result = await versionControlService.pullWorkfolder({ + force: req.body.force, + variables: req.body.variables, + userId: req.user.id, + importAfterPull: true, + }); + if ((result as ImportResult)?.workflows) { + return res.status(200).send(result as ImportResult); + } else { + return res.status(409).send(result); + } + } catch (error) { + return res.status(400).send((error as { message: string }).message); + } + }, + ], +}; diff --git a/packages/cli/src/PublicApi/v1/openapi.yml b/packages/cli/src/PublicApi/v1/openapi.yml index 216ae07521..fcb94b4092 100644 --- a/packages/cli/src/PublicApi/v1/openapi.yml +++ b/packages/cli/src/PublicApi/v1/openapi.yml @@ -1,7 +1,7 @@ --- openapi: 3.0.0 info: - title: n8n Public API + title: n8n Public API11 description: n8n Public API termsOfService: https://n8n.io/legal/terms contact: @@ -24,35 +24,39 @@ tags: description: Operations about workflows - name: Credential description: Operations about credentials + - name: VersionControl + description: Operations about version control paths: /audit: - $ref: './handlers/audit/spec/paths/audit.yml' + $ref: "./handlers/audit/spec/paths/audit.yml" /credentials: - $ref: './handlers/credentials/spec/paths/credentials.yml' + $ref: "./handlers/credentials/spec/paths/credentials.yml" /credentials/{id}: - $ref: './handlers/credentials/spec/paths/credentials.id.yml' + $ref: "./handlers/credentials/spec/paths/credentials.id.yml" /credentials/schema/{credentialTypeName}: - $ref: './handlers/credentials/spec/paths/credentials.schema.id.yml' + $ref: "./handlers/credentials/spec/paths/credentials.schema.id.yml" /executions: - $ref: './handlers/executions/spec/paths/executions.yml' + $ref: "./handlers/executions/spec/paths/executions.yml" /executions/{id}: - $ref: './handlers/executions/spec/paths/executions.id.yml' + $ref: "./handlers/executions/spec/paths/executions.id.yml" /workflows: - $ref: './handlers/workflows/spec/paths/workflows.yml' + $ref: "./handlers/workflows/spec/paths/workflows.yml" /workflows/{id}: - $ref: './handlers/workflows/spec/paths/workflows.id.yml' + $ref: "./handlers/workflows/spec/paths/workflows.id.yml" /workflows/{id}/activate: - $ref: './handlers/workflows/spec/paths/workflows.id.activate.yml' + $ref: "./handlers/workflows/spec/paths/workflows.id.activate.yml" /workflows/{id}/deactivate: - $ref: './handlers/workflows/spec/paths/workflows.id.deactivate.yml' + $ref: "./handlers/workflows/spec/paths/workflows.id.deactivate.yml" + /version-control/pull: + $ref: "./handlers/versionControl/spec/paths/versionControl.yml" components: schemas: - $ref: './shared/spec/schemas/_index.yml' + $ref: "./shared/spec/schemas/_index.yml" responses: - $ref: './shared/spec/responses/_index.yml' + $ref: "./shared/spec/responses/_index.yml" parameters: - $ref: './shared/spec/parameters/_index.yml' + $ref: "./shared/spec/parameters/_index.yml" securitySchemes: ApiKeyAuth: type: apiKey diff --git a/packages/cli/src/PublicApi/v1/shared/spec/responses/_index.yml b/packages/cli/src/PublicApi/v1/shared/spec/responses/_index.yml index 1d02bb8156..274c824335 100644 --- a/packages/cli/src/PublicApi/v1/shared/spec/responses/_index.yml +++ b/packages/cli/src/PublicApi/v1/shared/spec/responses/_index.yml @@ -1,6 +1,8 @@ NotFound: - $ref: './notFound.yml' + $ref: "./notFound.yml" Unauthorized: - $ref: './unauthorized.yml' + $ref: "./unauthorized.yml" BadRequest: - $ref: './badRequest.yml' + $ref: "./badRequest.yml" +Conflict: + $ref: "./conflict.yml" diff --git a/packages/cli/src/PublicApi/v1/shared/spec/responses/conflict.yml b/packages/cli/src/PublicApi/v1/shared/spec/responses/conflict.yml new file mode 100644 index 0000000000..3399c0afe5 --- /dev/null +++ b/packages/cli/src/PublicApi/v1/shared/spec/responses/conflict.yml @@ -0,0 +1 @@ +description: Conflict diff --git a/packages/cli/src/PublicApi/v1/shared/spec/schemas/_index.yml b/packages/cli/src/PublicApi/v1/shared/spec/schemas/_index.yml index d3c3d128dc..686abd2384 100644 --- a/packages/cli/src/PublicApi/v1/shared/spec/schemas/_index.yml +++ b/packages/cli/src/PublicApi/v1/shared/spec/schemas/_index.yml @@ -1,22 +1,26 @@ Error: - $ref: './error.yml' + $ref: "./error.yml" Execution: - $ref: './../../../handlers/executions/spec/schemas/execution.yml' + $ref: "./../../../handlers/executions/spec/schemas/execution.yml" Node: - $ref: './../../../handlers/workflows/spec/schemas/node.yml' + $ref: "./../../../handlers/workflows/spec/schemas/node.yml" Tag: - $ref: './../../../handlers/workflows/spec/schemas/tag.yml' + $ref: "./../../../handlers/workflows/spec/schemas/tag.yml" Workflow: - $ref: './../../../handlers/workflows/spec/schemas/workflow.yml' + $ref: "./../../../handlers/workflows/spec/schemas/workflow.yml" WorkflowSettings: - $ref: './../../../handlers/workflows/spec/schemas/workflowSettings.yml' + $ref: "./../../../handlers/workflows/spec/schemas/workflowSettings.yml" ExecutionList: - $ref: './../../../handlers/executions/spec/schemas/executionList.yml' + $ref: "./../../../handlers/executions/spec/schemas/executionList.yml" WorkflowList: - $ref: './../../../handlers/workflows/spec/schemas/workflowList.yml' + $ref: "./../../../handlers/workflows/spec/schemas/workflowList.yml" Credential: - $ref: './../../../handlers/credentials/spec/schemas/credential.yml' + $ref: "./../../../handlers/credentials/spec/schemas/credential.yml" CredentialType: - $ref: './../../../handlers/credentials/spec/schemas/credentialType.yml' + $ref: "./../../../handlers/credentials/spec/schemas/credentialType.yml" Audit: - $ref: './../../../handlers/audit/spec/schemas/audit.yml' + $ref: "./../../../handlers/audit/spec/schemas/audit.yml" +Pull: + $ref: "./../../../handlers/versionControl/spec/schemas/pull.yml" +ImportResult: + $ref: "./../../../handlers/versionControl/spec/schemas/importResult.yml" diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 8cb6f97f84..a0caf908ab 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -167,9 +167,10 @@ import { isLdapCurrentAuthenticationMethod, isSamlCurrentAuthenticationMethod, } from './sso/ssoHelpers'; -import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper'; +import { isVersionControlLicensed } from '@/environments/versionControl/versionControlHelper.ee'; import { VersionControlService } from '@/environments/versionControl/versionControl.service.ee'; import { VersionControlController } from '@/environments/versionControl/versionControl.controller.ee'; +import { VersionControlPreferencesService } from './environments/versionControl/versionControlPreferences.service.ee'; const exec = promisify(callbackExec); @@ -468,6 +469,7 @@ export class Server extends AbstractServer { const postHog = this.postHog; const samlService = Container.get(SamlService); const versionControlService = Container.get(VersionControlService); + const versionControlPreferencesService = Container.get(VersionControlPreferencesService); const controllers: object[] = [ new EventBusController(), @@ -496,7 +498,7 @@ export class Server extends AbstractServer { postHog, }), new SamlController(samlService), - new VersionControlController(versionControlService), + new VersionControlController(versionControlService, versionControlPreferencesService), ]; if (isLdapEnabled()) { diff --git a/packages/cli/src/environments/versionControl/constants.ts b/packages/cli/src/environments/versionControl/constants.ts index dc415672be..19eb138238 100644 --- a/packages/cli/src/environments/versionControl/constants.ts +++ b/packages/cli/src/environments/versionControl/constants.ts @@ -1 +1,15 @@ export const VERSION_CONTROL_PREFERENCES_DB_KEY = 'features.versionControl'; +export const VERSION_CONTROL_GIT_FOLDER = 'git'; +export const VERSION_CONTROL_GIT_KEY_COMMENT = 'n8n deploy key'; +export const VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER = 'workflows'; +export const VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER = 'credentials'; +export const VERSION_CONTROL_VARIABLES_EXPORT_FILE = 'variables.json'; +export const VERSION_CONTROL_TAGS_EXPORT_FILE = 'tags.json'; +export const VERSION_CONTROL_SSH_FOLDER = 'ssh'; +export const VERSION_CONTROL_SSH_KEY_NAME = 'key'; +export const VERSION_CONTROL_DEFAULT_BRANCH = 'main'; +export const VERSION_CONTROL_ORIGIN = 'origin'; +export const VERSION_CONTROL_API_ROOT = 'version-control'; +export const VERSION_CONTROL_README = ` +# n8n Version Control +`; diff --git a/packages/cli/src/environments/versionControl/middleware/versionControlEnabledMiddleware.ts b/packages/cli/src/environments/versionControl/middleware/versionControlEnabledMiddleware.ee.ts similarity index 55% rename from packages/cli/src/environments/versionControl/middleware/versionControlEnabledMiddleware.ts rename to packages/cli/src/environments/versionControl/middleware/versionControlEnabledMiddleware.ee.ts index 5ed3a1293b..8a296d3e17 100644 --- a/packages/cli/src/environments/versionControl/middleware/versionControlEnabledMiddleware.ts +++ b/packages/cli/src/environments/versionControl/middleware/versionControlEnabledMiddleware.ee.ts @@ -1,11 +1,11 @@ import type { RequestHandler } from 'express'; -import { - isVersionControlLicensed, - isVersionControlLicensedAndEnabled, -} from '../versionControlHelper'; +import { isVersionControlLicensed } from '../versionControlHelper.ee'; +import Container from 'typedi'; +import { VersionControlPreferencesService } from '../versionControlPreferences.service.ee'; export const versionControlLicensedAndEnabledMiddleware: RequestHandler = (req, res, next) => { - if (isVersionControlLicensedAndEnabled()) { + const versionControlPreferencesService = Container.get(VersionControlPreferencesService); + if (versionControlPreferencesService.isVersionControlLicensedAndEnabled()) { next(); } else { res.status(401).json({ status: 'error', message: 'Unauthorized' }); diff --git a/packages/cli/src/environments/versionControl/types/exportResult.ts b/packages/cli/src/environments/versionControl/types/exportResult.ts new file mode 100644 index 0000000000..b7c13f3f56 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/exportResult.ts @@ -0,0 +1,9 @@ +export interface ExportResult { + count: number; + folder: string; + files: Array<{ + id: string; + name: string; + }>; + removedFiles?: string[]; +} diff --git a/packages/cli/src/environments/versionControl/types/exportableCredential.ts b/packages/cli/src/environments/versionControl/types/exportableCredential.ts new file mode 100644 index 0000000000..b9d33e0520 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/exportableCredential.ts @@ -0,0 +1,8 @@ +import type { ICredentialDataDecryptedObject } from 'n8n-workflow'; + +export interface ExportableCredential { + id: string; + name: string; + type: string; + data: ICredentialDataDecryptedObject; +} diff --git a/packages/cli/src/environments/versionControl/types/exportableWorkflow.ts b/packages/cli/src/environments/versionControl/types/exportableWorkflow.ts new file mode 100644 index 0000000000..ca0f7087f9 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/exportableWorkflow.ts @@ -0,0 +1,13 @@ +import type { INode, IConnections, IWorkflowSettings } from 'n8n-workflow'; + +export interface ExportableWorkflow { + active: boolean; + id: string; + name: string; + nodes: INode[]; + connections: IConnections; + settings?: IWorkflowSettings; + triggerCount: number; + owner: string; + versionId: string; +} diff --git a/packages/cli/src/environments/versionControl/types/importResult.ts b/packages/cli/src/environments/versionControl/types/importResult.ts new file mode 100644 index 0000000000..6be9885869 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/importResult.ts @@ -0,0 +1,13 @@ +import type { TagEntity } from '@/databases/entities/TagEntity'; +import type { WorkflowTagMapping } from '@/databases/entities/WorkflowTagMapping'; + +export interface ImportResult { + workflows: Array<{ + id: string; + name: string; + }>; + credentials: Array<{ id: string; name: string; type: string }>; + variables: { added: string[]; changed: string[] }; + tags: { tags: TagEntity[]; mappings: WorkflowTagMapping[] }; + removedFiles?: string[]; +} diff --git a/packages/cli/src/environments/versionControl/types/requests.ts b/packages/cli/src/environments/versionControl/types/requests.ts index 0782873ba5..d985e36b0b 100644 --- a/packages/cli/src/environments/versionControl/types/requests.ts +++ b/packages/cli/src/environments/versionControl/types/requests.ts @@ -1,6 +1,22 @@ import type { AuthenticatedRequest } from '@/requests'; import type { VersionControlPreferences } from './versionControlPreferences'; +import type { VersionControlSetBranch } from './versionControlSetBranch'; +import type { VersionControlCommit } from './versionControlCommit'; +import type { VersionControlStage } from './versionControlStage'; +import type { VersionControlPush } from './versionControlPush'; +import type { VersionControlPushWorkFolder } from './versionControlPushWorkFolder'; +import type { VersionControlPullWorkFolder } from './versionControlPullWorkFolder'; +import type { VersionControlDisconnect } from './versionControlDisconnect'; +import type { VersionControlSetReadOnly } from './versionControlSetReadOnly'; export declare namespace VersionControlRequest { type UpdatePreferences = AuthenticatedRequest<{}, {}, Partial, {}>; + type SetReadOnly = AuthenticatedRequest<{}, {}, VersionControlSetReadOnly, {}>; + type SetBranch = AuthenticatedRequest<{}, {}, VersionControlSetBranch, {}>; + type Commit = AuthenticatedRequest<{}, {}, VersionControlCommit, {}>; + type Stage = AuthenticatedRequest<{}, {}, VersionControlStage, {}>; + type Push = AuthenticatedRequest<{}, {}, VersionControlPush, {}>; + type Disconnect = AuthenticatedRequest<{}, {}, VersionControlDisconnect, {}>; + type PushWorkFolder = AuthenticatedRequest<{}, {}, VersionControlPushWorkFolder, {}>; + type PullWorkFolder = AuthenticatedRequest<{}, {}, VersionControlPullWorkFolder, {}>; } diff --git a/packages/cli/src/environments/versionControl/types/versionControlCommit.ts b/packages/cli/src/environments/versionControl/types/versionControlCommit.ts new file mode 100644 index 0000000000..0264f4cb32 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlCommit.ts @@ -0,0 +1,6 @@ +import { IsString } from 'class-validator'; + +export class VersionControlCommit { + @IsString() + message: string; +} diff --git a/packages/cli/src/environments/versionControl/types/versionControlDisconnect.ts b/packages/cli/src/environments/versionControl/types/versionControlDisconnect.ts new file mode 100644 index 0000000000..2c4f8be008 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlDisconnect.ts @@ -0,0 +1,7 @@ +import { IsBoolean, IsOptional } from 'class-validator'; + +export class VersionControlDisconnect { + @IsBoolean() + @IsOptional() + keepKeyPair?: boolean; +} diff --git a/packages/cli/src/environments/versionControl/types/versionControlPreferences.ts b/packages/cli/src/environments/versionControl/types/versionControlPreferences.ts index 9481270ec5..c17e334a7c 100644 --- a/packages/cli/src/environments/versionControl/types/versionControlPreferences.ts +++ b/packages/cli/src/environments/versionControl/types/versionControlPreferences.ts @@ -18,7 +18,7 @@ export class VersionControlPreferences { authorEmail: string; @IsString() - branchName: string; + branchName = 'main'; @IsBoolean() branchReadOnly: boolean; @@ -28,9 +28,28 @@ export class VersionControlPreferences { @IsOptional() @IsString() - readonly privateKey?: string; + readonly publicKey?: string; @IsOptional() - @IsString() - readonly publicKey?: string; + @IsBoolean() + readonly initRepo?: boolean; + + static fromJSON(json: Partial): VersionControlPreferences { + return new VersionControlPreferences(json); + } + + static merge( + preferences: Partial, + defaultPreferences: Partial, + ): VersionControlPreferences { + return new VersionControlPreferences({ + connected: preferences.connected ?? defaultPreferences.connected, + authorEmail: preferences.authorEmail ?? defaultPreferences.authorEmail, + authorName: preferences.authorName ?? defaultPreferences.authorName, + branchName: preferences.branchName ?? defaultPreferences.branchName, + branchColor: preferences.branchColor ?? defaultPreferences.branchColor, + branchReadOnly: preferences.branchReadOnly ?? defaultPreferences.branchReadOnly, + repositoryUrl: preferences.repositoryUrl ?? defaultPreferences.repositoryUrl, + }); + } } diff --git a/packages/cli/src/environments/versionControl/types/versionControlPullWorkFolder.ts b/packages/cli/src/environments/versionControl/types/versionControlPullWorkFolder.ts new file mode 100644 index 0000000000..937e0758b2 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlPullWorkFolder.ts @@ -0,0 +1,29 @@ +import { IsBoolean, IsObject, IsOptional, IsString } from 'class-validator'; + +export class VersionControlPullWorkFolder { + @IsBoolean() + @IsOptional() + force?: boolean; + + @IsBoolean() + @IsOptional() + importAfterPull?: boolean = true; + + @IsString({ each: true }) + @IsOptional() + files?: Set; + + @IsObject() + @IsOptional() + variables?: { [key: string]: string }; +} + +export class VersionControllPullOptions { + userId: string; + + force?: boolean; + + variables?: { [key: string]: string }; + + importAfterPull?: boolean = true; +} diff --git a/packages/cli/src/environments/versionControl/types/versionControlPush.ts b/packages/cli/src/environments/versionControl/types/versionControlPush.ts new file mode 100644 index 0000000000..a8efa050de --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlPush.ts @@ -0,0 +1,7 @@ +import { IsBoolean, IsOptional } from 'class-validator'; + +export class VersionControlPush { + @IsBoolean() + @IsOptional() + force?: boolean; +} diff --git a/packages/cli/src/environments/versionControl/types/versionControlPushWorkFolder.ts b/packages/cli/src/environments/versionControl/types/versionControlPushWorkFolder.ts new file mode 100644 index 0000000000..ff74af3fff --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlPushWorkFolder.ts @@ -0,0 +1,27 @@ +import { IsBoolean, IsOptional, IsString } from 'class-validator'; + +export class VersionControlPushWorkFolder { + @IsBoolean() + @IsOptional() + force?: boolean; + + @IsString({ each: true }) + @IsOptional() + fileNames?: Set; + + @IsString({ each: true }) + @IsOptional() + workflowIds?: Set; + + @IsString({ each: true }) + @IsOptional() + credentialIds?: Set; + + @IsString() + @IsOptional() + message?: string; + + @IsBoolean() + @IsOptional() + skipDiff?: boolean; +} diff --git a/packages/cli/src/environments/versionControl/types/versionControlSetBranch.ts b/packages/cli/src/environments/versionControl/types/versionControlSetBranch.ts new file mode 100644 index 0000000000..353ca1da3e --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlSetBranch.ts @@ -0,0 +1,6 @@ +import { IsString } from 'class-validator'; + +export class VersionControlSetBranch { + @IsString() + branch: string; +} diff --git a/packages/cli/src/environments/versionControl/types/versionControlSetReadOnly.ts b/packages/cli/src/environments/versionControl/types/versionControlSetReadOnly.ts new file mode 100644 index 0000000000..665c42a8d4 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlSetReadOnly.ts @@ -0,0 +1,6 @@ +import { IsBoolean } from 'class-validator'; + +export class VersionControlSetReadOnly { + @IsBoolean() + branchReadOnly: boolean; +} diff --git a/packages/cli/src/environments/versionControl/types/versionControlStage.ts b/packages/cli/src/environments/versionControl/types/versionControlStage.ts new file mode 100644 index 0000000000..b6184bb8e2 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlStage.ts @@ -0,0 +1,15 @@ +import { IsOptional, IsString } from 'class-validator'; + +export class VersionControlStage { + @IsString({ each: true }) + @IsOptional() + fileNames?: Set; + + @IsString({ each: true }) + @IsOptional() + workflowIds?: Set; + + @IsString({ each: true }) + @IsOptional() + credentialIds?: Set; +} diff --git a/packages/cli/src/environments/versionControl/types/versionControlledFile.ts b/packages/cli/src/environments/versionControl/types/versionControlledFile.ts new file mode 100644 index 0000000000..e8e0e6f773 --- /dev/null +++ b/packages/cli/src/environments/versionControl/types/versionControlledFile.ts @@ -0,0 +1,19 @@ +export type VersionControlledFileStatus = + | 'new' + | 'modified' + | 'deleted' + | 'created' + | 'renamed' + | 'conflicted' + | 'unknown'; +export type VersionControlledFileLocation = 'local' | 'remote'; +export type VersionControlledFileType = 'credential' | 'workflow' | 'tags' | 'variables' | 'file'; +export type VersionControlledFile = { + file: string; + id: string; + name: string; + type: VersionControlledFileType; + status: VersionControlledFileStatus; + location: VersionControlledFileLocation; + conflict: boolean; +}; diff --git a/packages/cli/src/environments/versionControl/versionControl.controller.ee.ts b/packages/cli/src/environments/versionControl/versionControl.controller.ee.ts index 5bbac46f0b..a85daa2ca7 100644 --- a/packages/cli/src/environments/versionControl/versionControl.controller.ee.ts +++ b/packages/cli/src/environments/versionControl/versionControl.controller.ee.ts @@ -1,37 +1,320 @@ import { Authorized, Get, Post, RestController } from '@/decorators'; -import { versionControlLicensedMiddleware } from './middleware/versionControlEnabledMiddleware'; +import { + versionControlLicensedMiddleware, + versionControlLicensedAndEnabledMiddleware, +} from './middleware/versionControlEnabledMiddleware.ee'; import { VersionControlService } from './versionControl.service.ee'; import { VersionControlRequest } from './types/requests'; import type { VersionControlPreferences } from './types/versionControlPreferences'; +import { BadRequestError } from '@/ResponseHelper'; +import type { PullResult, PushResult, StatusResult } from 'simple-git'; +import { AuthenticatedRequest } from '../../requests'; +import express from 'express'; +import type { ImportResult } from './types/importResult'; +import type { VersionControlPushWorkFolder } from './types/versionControlPushWorkFolder'; +import { VersionControlPreferencesService } from './versionControlPreferences.service.ee'; +import type { VersionControlledFile } from './types/versionControlledFile'; +import { VERSION_CONTROL_API_ROOT, VERSION_CONTROL_DEFAULT_BRANCH } from './constants'; -@RestController('/versionControl') +@RestController(`/${VERSION_CONTROL_API_ROOT}`) export class VersionControlController { - constructor(private versionControlService: VersionControlService) {} + constructor( + private versionControlService: VersionControlService, + private versionControlPreferencesService: VersionControlPreferencesService, + ) {} @Authorized('any') @Get('/preferences', { middlewares: [versionControlLicensedMiddleware] }) async getPreferences(): Promise { // returns the settings with the privateKey property redacted - return this.versionControlService.versionControlPreferences; + return this.versionControlPreferencesService.getPreferences(); } @Authorized(['global', 'owner']) @Post('/preferences', { middlewares: [versionControlLicensedMiddleware] }) async setPreferences(req: VersionControlRequest.UpdatePreferences) { - const sanitizedPreferences: Partial = { - ...req.body, - privateKey: undefined, - publicKey: undefined, - }; - await this.versionControlService.validateVersionControlPreferences(sanitizedPreferences); - return this.versionControlService.setPreferences(sanitizedPreferences); + if ( + req.body.branchReadOnly === undefined && + this.versionControlPreferencesService.isVersionControlConnected() + ) { + throw new BadRequestError( + 'Cannot change preferences while connected to a version control provider. Please disconnect first.', + ); + } + try { + const sanitizedPreferences: Partial = { + ...req.body, + initRepo: req.body.initRepo ?? true, // default to true if not specified + connected: undefined, + publicKey: undefined, + }; + await this.versionControlPreferencesService.validateVersionControlPreferences( + sanitizedPreferences, + ); + const updatedPreferences = await this.versionControlPreferencesService.setPreferences( + sanitizedPreferences, + ); + if (sanitizedPreferences.initRepo === true) { + try { + await this.versionControlService.initializeRepository({ + ...updatedPreferences, + branchName: + updatedPreferences.branchName === '' + ? VERSION_CONTROL_DEFAULT_BRANCH + : updatedPreferences.branchName, + initRepo: true, + }); + if (this.versionControlPreferencesService.getPreferences().branchName !== '') { + await this.versionControlPreferencesService.setPreferences({ + connected: true, + }); + } + } catch (error) { + // if initialization fails, run cleanup to remove any intermediate state and throw the error + await this.versionControlService.disconnect({ keepKeyPair: true }); + throw error; + } + } + await this.versionControlService.init(); + return this.versionControlPreferencesService.getPreferences(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } } - //TODO: temporary function to generate key and save new pair - // REMOVE THIS FUNCTION AFTER TESTING @Authorized(['global', 'owner']) - @Get('/generateKeyPair', { middlewares: [versionControlLicensedMiddleware] }) - async generateKeyPair() { - return this.versionControlService.generateAndSaveKeyPair(); + @Post('/set-read-only', { middlewares: [versionControlLicensedMiddleware] }) + async setReadOnly(req: VersionControlRequest.SetReadOnly) { + try { + this.versionControlPreferencesService.setBranchReadOnly(req.body.branchReadOnly); + return this.versionControlPreferencesService.getPreferences(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } } + + @Authorized(['global', 'owner']) + @Post('/disconnect', { middlewares: [versionControlLicensedMiddleware] }) + async disconnect(req: VersionControlRequest.Disconnect) { + try { + return await this.versionControlService.disconnect(req.body); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized('any') + @Get('/get-branches', { middlewares: [versionControlLicensedMiddleware] }) + async getBranches() { + try { + return await this.versionControlService.getBranches(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Post('/set-branch', { middlewares: [versionControlLicensedMiddleware] }) + async setBranch(req: VersionControlRequest.SetBranch) { + try { + return await this.versionControlService.setBranch(req.body.branch); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Post('/push-workfolder', { middlewares: [versionControlLicensedAndEnabledMiddleware] }) + async pushWorkfolder( + req: VersionControlRequest.PushWorkFolder, + res: express.Response, + ): Promise { + if (this.versionControlPreferencesService.isBranchReadOnly()) { + throw new BadRequestError('Cannot push onto read-only branch.'); + } + try { + const result = await this.versionControlService.pushWorkfolder(req.body); + if ((result as PushResult).pushed) { + res.statusCode = 200; + } else { + res.statusCode = 409; + } + return result; + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Post('/pull-workfolder', { middlewares: [versionControlLicensedAndEnabledMiddleware] }) + async pullWorkfolder( + req: VersionControlRequest.PullWorkFolder, + res: express.Response, + ): Promise { + try { + const result = await this.versionControlService.pullWorkfolder({ + force: req.body.force, + variables: req.body.variables, + userId: req.user.id, + importAfterPull: req.body.importAfterPull ?? true, + }); + if ((result as ImportResult)?.workflows) { + res.statusCode = 200; + } else { + res.statusCode = 409; + } + return result; + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Get('/reset-workfolder', { middlewares: [versionControlLicensedAndEnabledMiddleware] }) + async resetWorkfolder( + req: VersionControlRequest.PullWorkFolder, + ): Promise { + try { + return await this.versionControlService.resetWorkfolder({ + force: req.body.force, + variables: req.body.variables, + userId: req.user.id, + importAfterPull: req.body.importAfterPull ?? true, + }); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized('any') + @Get('/get-status', { middlewares: [versionControlLicensedAndEnabledMiddleware] }) + async getStatus() { + try { + return await this.versionControlService.getStatus(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized('any') + @Get('/status', { middlewares: [versionControlLicensedMiddleware] }) + async status(): Promise { + try { + return await this.versionControlService.status(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Post('/generate-key-pair', { middlewares: [versionControlLicensedMiddleware] }) + async generateKeyPair(): Promise { + try { + const result = await this.versionControlPreferencesService.generateAndSaveKeyPair(); + return result; + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + // #region Version Control Test Functions + //TODO: SEPARATE FUNCTIONS FOR DEVELOPMENT ONLY + //TODO: REMOVE THESE FUNCTIONS AFTER TESTING + + @Authorized(['global', 'owner']) + @Get('/export', { middlewares: [versionControlLicensedMiddleware] }) + async export() { + try { + return await this.versionControlService.export(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Get('/import', { middlewares: [versionControlLicensedMiddleware] }) + async import(req: AuthenticatedRequest) { + try { + return await this.versionControlService.import({ + userId: req.user.id, + }); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized('any') + @Get('/fetch') + async fetch() { + try { + return await this.versionControlService.fetch(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized('any') + @Get('/diff') + async diff() { + try { + return await this.versionControlService.diff(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Post('/push') + async push(req: VersionControlRequest.Push): Promise { + if (this.versionControlPreferencesService.isBranchReadOnly()) { + throw new BadRequestError('Cannot push onto read-only branch.'); + } + try { + return await this.versionControlService.push(req.body.force); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Post('/commit') + async commit(req: VersionControlRequest.Commit) { + try { + return await this.versionControlService.commit(req.body.message); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Post('/stage') + async stage(req: VersionControlRequest.Stage): Promise<{ staged: string[] } | string> { + try { + return await this.versionControlService.stage(req.body as VersionControlPushWorkFolder); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Post('/unstage') + async unstage(): Promise { + try { + return await this.versionControlService.unstage(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + @Authorized(['global', 'owner']) + @Get('/pull') + async pull(): Promise { + try { + return await this.versionControlService.pull(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + // #endregion } diff --git a/packages/cli/src/environments/versionControl/versionControl.service.ee.ts b/packages/cli/src/environments/versionControl/versionControl.service.ee.ts index d531935f00..7a37cfe357 100644 --- a/packages/cli/src/environments/versionControl/versionControl.service.ee.ts +++ b/packages/cli/src/environments/versionControl/versionControl.service.ee.ts @@ -1,108 +1,491 @@ import { Service } from 'typedi'; -import { generateSshKeyPair } from './versionControlHelper'; -import { VersionControlPreferences } from './types/versionControlPreferences'; -import { VERSION_CONTROL_PREFERENCES_DB_KEY } from './constants'; +import path from 'path'; import * as Db from '@/Db'; -import { jsonParse, LoggerProxy } from 'n8n-workflow'; -import type { ValidationError } from 'class-validator'; -import { validate } from 'class-validator'; - +import { versionControlFoldersExistCheck } from './versionControlHelper.ee'; +import type { VersionControlPreferences } from './types/versionControlPreferences'; +import { + VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER, + VERSION_CONTROL_GIT_FOLDER, + VERSION_CONTROL_README, + VERSION_CONTROL_SSH_FOLDER, + VERSION_CONTROL_SSH_KEY_NAME, + VERSION_CONTROL_TAGS_EXPORT_FILE, + VERSION_CONTROL_VARIABLES_EXPORT_FILE, + VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER, +} from './constants'; +import { LoggerProxy } from 'n8n-workflow'; +import { VersionControlGitService } from './versionControlGit.service.ee'; +import { UserSettings } from 'n8n-core'; +import type { + CommitResult, + DiffResult, + FetchResult, + PullResult, + PushResult, + StatusResult, +} from 'simple-git'; +import type { ExportResult } from './types/exportResult'; +import { VersionControlExportService } from './versionControlExport.service.ee'; +import { BadRequestError } from '../../ResponseHelper'; +import type { ImportResult } from './types/importResult'; +import type { VersionControlPushWorkFolder } from './types/versionControlPushWorkFolder'; +import type { VersionControllPullOptions } from './types/versionControlPullWorkFolder'; +import type { + VersionControlledFileLocation, + VersionControlledFile, + VersionControlledFileStatus, + VersionControlledFileType, +} from './types/versionControlledFile'; +import { VersionControlPreferencesService } from './versionControlPreferences.service.ee'; +import { writeFileSync } from 'fs'; @Service() export class VersionControlService { - private _versionControlPreferences: VersionControlPreferences = new VersionControlPreferences(); + private sshKeyName: string; + + private sshFolder: string; + + private gitFolder: string; + + constructor( + private gitService: VersionControlGitService, + private versionControlPreferencesService: VersionControlPreferencesService, + private versionControlExportService: VersionControlExportService, + ) { + const userFolder = UserSettings.getUserN8nFolderPath(); + this.sshFolder = path.join(userFolder, VERSION_CONTROL_SSH_FOLDER); + this.gitFolder = path.join(userFolder, VERSION_CONTROL_GIT_FOLDER); + this.sshKeyName = path.join(this.sshFolder, VERSION_CONTROL_SSH_KEY_NAME); + } async init(): Promise { - await this.loadFromDbAndApplyVersionControlPreferences(); + this.gitService.resetService(); + versionControlFoldersExistCheck([this.gitFolder, this.sshFolder]); + await this.versionControlPreferencesService.loadFromDbAndApplyVersionControlPreferences(); + await this.gitService.initService({ + versionControlPreferences: this.versionControlPreferencesService.getPreferences(), + gitFolder: this.gitFolder, + sshKeyName: this.sshKeyName, + sshFolder: this.sshFolder, + }); } - public get versionControlPreferences(): VersionControlPreferences { - return { - ...this._versionControlPreferences, - privateKey: '(redacted)', - }; + async disconnect(options: { keepKeyPair?: boolean } = {}) { + try { + await this.versionControlPreferencesService.setPreferences({ + connected: false, + branchName: '', + }); + await this.versionControlExportService.deleteRepositoryFolder(); + if (!options.keepKeyPair) { + await this.versionControlPreferencesService.deleteKeyPairFiles(); + } + this.gitService.resetService(); + return this.versionControlPreferencesService.versionControlPreferences; + } catch (error) { + throw Error(`Failed to disconnect from version control: ${(error as Error).message}`); + } } - public set versionControlPreferences(preferences: Partial) { - this._versionControlPreferences = { - connected: preferences.connected ?? this._versionControlPreferences.connected, - authorEmail: preferences.authorEmail ?? this._versionControlPreferences.authorEmail, - authorName: preferences.authorName ?? this._versionControlPreferences.authorName, - branchName: preferences.branchName ?? this._versionControlPreferences.branchName, - branchColor: preferences.branchColor ?? this._versionControlPreferences.branchColor, - branchReadOnly: preferences.branchReadOnly ?? this._versionControlPreferences.branchReadOnly, - privateKey: preferences.privateKey ?? this._versionControlPreferences.privateKey, - publicKey: preferences.publicKey ?? this._versionControlPreferences.publicKey, - repositoryUrl: preferences.repositoryUrl ?? this._versionControlPreferences.repositoryUrl, - }; - } - - async generateAndSaveKeyPair() { - const keyPair = generateSshKeyPair('ed25519'); - if (keyPair.publicKey && keyPair.privateKey) { - await this.setPreferences({ ...keyPair }); + async initializeRepository(preferences: VersionControlPreferences) { + if (!this.gitService.git) { + await this.init(); + } + LoggerProxy.debug('Initializing repository...'); + await this.gitService.initRepository(preferences); + let getBranchesResult; + try { + getBranchesResult = await this.getBranches(); + } catch (error) { + if ((error as Error).message.includes('Warning: Permanently added')) { + LoggerProxy.debug('Added repository host to the list of known hosts. Retrying...'); + getBranchesResult = await this.getBranches(); + } else { + throw error; + } + } + if (getBranchesResult.branches.includes(preferences.branchName)) { + await this.gitService.setBranch(preferences.branchName); } else { - LoggerProxy.error('Failed to generate key pair'); - } - return keyPair; - } + if (getBranchesResult.branches?.length === 0) { + try { + writeFileSync(path.join(this.gitFolder, '/README.md'), VERSION_CONTROL_README); - async validateVersionControlPreferences( - preferences: Partial, - ): Promise { - const preferencesObject = new VersionControlPreferences(preferences); - const validationResult = await validate(preferencesObject, { - forbidUnknownValues: false, - skipMissingProperties: true, - stopAtFirstError: false, - validationError: { target: false }, - }); - if (validationResult.length > 0) { - throw new Error(`Invalid version control preferences: ${JSON.stringify(validationResult)}`); - } - // TODO: if repositoryUrl is changed, check if it is valid - // TODO: if branchName is changed, check if it is valid - return validationResult; - } - - async setPreferences( - preferences: Partial, - saveToDb = true, - ): Promise { - this.versionControlPreferences = preferences; - if (saveToDb) { - const settingsValue = JSON.stringify(this._versionControlPreferences); - try { - await Db.collections.Settings.save({ - key: VERSION_CONTROL_PREFERENCES_DB_KEY, - value: settingsValue, - loadOnStartup: true, - }); - } catch (error) { - throw new Error(`Failed to save version control preferences: ${(error as Error).message}`); - } - } - return this.versionControlPreferences; - } - - async loadFromDbAndApplyVersionControlPreferences(): Promise< - VersionControlPreferences | undefined - > { - const loadedPreferences = await Db.collections.Settings.findOne({ - where: { key: VERSION_CONTROL_PREFERENCES_DB_KEY }, - }); - if (loadedPreferences) { - try { - const preferences = jsonParse(loadedPreferences.value); - if (preferences) { - await this.setPreferences(preferences, false); - return preferences; + await this.gitService.stage(new Set(['README.md'])); + await this.gitService.commit('Initial commit'); + await this.gitService.push({ + branch: preferences.branchName, + force: true, + }); + getBranchesResult = await this.getBranches(); + } catch (fileError) { + LoggerProxy.error(`Failed to create initial commit: ${(fileError as Error).message}`); } - } catch (error) { - LoggerProxy.warn( - `Could not parse Version Control settings from database: ${(error as Error).message}`, - ); + } else { + await this.versionControlPreferencesService.setPreferences({ + branchName: '', + connected: true, + }); } } + return getBranchesResult; + } + + async export() { + const result: { + tags: ExportResult | undefined; + credentials: ExportResult | undefined; + variables: ExportResult | undefined; + workflows: ExportResult | undefined; + } = { + credentials: undefined, + tags: undefined, + variables: undefined, + workflows: undefined, + }; + try { + // comment next line if needed + await this.versionControlExportService.cleanWorkFolder(); + result.tags = await this.versionControlExportService.exportTagsToWorkFolder(); + result.variables = await this.versionControlExportService.exportVariablesToWorkFolder(); + result.workflows = await this.versionControlExportService.exportWorkflowsToWorkFolder(); + result.credentials = await this.versionControlExportService.exportCredentialsToWorkFolder(); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + return result; + } + + async import(options: VersionControllPullOptions): Promise { + try { + return await this.versionControlExportService.importFromWorkFolder(options); + } catch (error) { + throw new BadRequestError((error as { message: string }).message); + } + } + + async getBranches(): Promise<{ branches: string[]; currentBranch: string }> { + // fetch first to get include remote changes + await this.gitService.fetch(); + return this.gitService.getBranches(); + } + + async setBranch(branch: string): Promise<{ branches: string[]; currentBranch: string }> { + await this.versionControlPreferencesService.setPreferences({ + branchName: branch, + connected: true, + }); + return this.gitService.setBranch(branch); + } + + // will reset the branch to the remote branch and pull + // this will discard all local changes + async resetWorkfolder(options: VersionControllPullOptions): Promise { + const currentBranch = await this.gitService.getCurrentBranch(); + await this.versionControlExportService.cleanWorkFolder(); + await this.gitService.resetBranch({ + hard: true, + target: currentBranch.remote, + }); + await this.gitService.pull(); + if (options.importAfterPull) { + return this.import(options); + } return; } + + async pushWorkfolder( + options: VersionControlPushWorkFolder, + ): Promise { + if (this.versionControlPreferencesService.isBranchReadOnly()) { + throw new BadRequestError('Cannot push onto read-only branch.'); + } + if (!options.skipDiff) { + const diffResult = await this.getStatus(); + const possibleConflicts = diffResult?.filter((file) => file.conflict); + if (possibleConflicts?.length > 0 && options.force !== true) { + await this.unstage(); + return diffResult; + } + } + await this.unstage(); + await this.stage(options); + await this.gitService.commit(options.message ?? 'Updated Workfolder'); + return this.gitService.push({ + branch: this.versionControlPreferencesService.getBranchName(), + force: options.force ?? false, + }); + } + + // async pushWorkfolder( + // options: VersionControlPushWorkFolder, + // ): Promise { + // await this.gitService.fetch(); + // await this.export(); // refresh workfolder + // await this.stage(options); + // await this.gitService.commit(options.message ?? 'Updated Workfolder'); + // return this.gitService.push({ + // branch: this.versionControlPreferencesService.getBranchName(), + // force: options.force ?? false, + // }); + // } + + // TODO: Alternate implementation for pull + // async pullWorkfolder( + // options: VersionControllPullOptions, + // ): Promise { + // const diffResult = await this.getStatus(); + // const possibleConflicts = diffResult?.filter((file) => file.conflict); + // if (possibleConflicts?.length > 0 || options.force === true) { + // await this.unstage(); + // if (options.force === true) { + // return this.resetWorkfolder(options); + // } else { + // return diffResult; + // } + // } + // const pullResult = await this.gitService.pull(); + // if (options.importAfterPull) { + // return this.import(options); + // } + // return pullResult; + // } + + async pullWorkfolder( + options: VersionControllPullOptions, + ): Promise { + await this.resetWorkfolder({ + importAfterPull: false, + userId: options.userId, + force: false, + }); + await this.export(); // refresh workfolder + const status = await this.gitService.status(); + + if (status.modified.length > 0 && options.force !== true) { + return status; + } + await this.resetWorkfolder({ ...options, importAfterPull: false }); + if (options.importAfterPull) { + return this.import(options); + } + return; + } + + async stage( + options: Pick, + ): Promise<{ staged: string[] } | string> { + const { fileNames, credentialIds, workflowIds } = options; + const status = await this.gitService.status(); + let mergedFileNames = new Set(); + fileNames?.forEach((e) => mergedFileNames.add(e)); + credentialIds?.forEach((e) => + mergedFileNames.add(this.versionControlExportService.getCredentialsPath(e)), + ); + workflowIds?.forEach((e) => + mergedFileNames.add(this.versionControlExportService.getWorkflowPath(e)), + ); + if (mergedFileNames.size === 0) { + mergedFileNames = new Set([ + ...status.not_added, + ...status.created, + ...status.modified, + ]); + } + const deletedFiles = new Set(status.deleted); + deletedFiles.forEach((e) => mergedFileNames.delete(e)); + await this.unstage(); + const stageResult = await this.gitService.stage(mergedFileNames, deletedFiles); + if (!stageResult) { + const statusResult = await this.gitService.status(); + return { staged: statusResult.staged }; + } + return stageResult; + } + + async unstage(): Promise { + const stageResult = await this.gitService.resetBranch(); + if (!stageResult) { + return this.gitService.status(); + } + return stageResult; + } + + async status(): Promise { + return this.gitService.status(); + } + + private async fileNameToVersionControlledFile( + fileName: string, + location: VersionControlledFileLocation, + statusResult: StatusResult, + ): Promise { + let id: string | undefined = undefined; + let name = ''; + let conflict = false; + let status: VersionControlledFileStatus = 'unknown'; + let type: VersionControlledFileType = 'file'; + + // initialize status from git status result + if (statusResult.not_added.find((e) => e === fileName)) status = 'new'; + else if (statusResult.conflicted.find((e) => e === fileName)) { + status = 'conflicted'; + conflict = true; + } else if (statusResult.created.find((e) => e === fileName)) status = 'created'; + else if (statusResult.deleted.find((e) => e === fileName)) status = 'deleted'; + else if (statusResult.modified.find((e) => e === fileName)) status = 'modified'; + + if (fileName.startsWith(VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER)) { + type = 'workflow'; + if (status === 'deleted') { + id = fileName + .replace(VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER, '') + .replace(/[\/,\\]/, '') + .replace('.json', ''); + if (location === 'remote') { + const existingWorkflow = await Db.collections.Workflow.find({ + where: { id }, + }); + if (existingWorkflow?.length > 0) { + name = existingWorkflow[0].name; + } + } else { + name = '(deleted)'; + } + } else { + const workflow = await this.versionControlExportService.getWorkflowFromFile(fileName); + if (!workflow?.id) { + if (location === 'local') { + return; + } + id = fileName + .replace(VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER + '/', '') + .replace('.json', ''); + status = 'created'; + } else { + id = workflow.id; + name = workflow.name; + } + } + } + if (fileName.startsWith(VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER)) { + type = 'credential'; + if (status === 'deleted') { + id = fileName + .replace(VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER, '') + .replace(/[\/,\\]/, '') + .replace('.json', ''); + if (location === 'remote') { + const existingCredential = await Db.collections.Credentials.find({ + where: { id }, + }); + if (existingCredential?.length > 0) { + name = existingCredential[0].name; + } + } else { + name = '(deleted)'; + } + } else { + const credential = await this.versionControlExportService.getCredentialFromFile(fileName); + if (!credential?.id) { + if (location === 'local') { + return; + } + id = fileName + .replace(VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER + '/', '') + .replace('.json', ''); + status = 'created'; + } else { + id = credential.id; + name = credential.name; + } + } + } + + if (fileName.startsWith(VERSION_CONTROL_VARIABLES_EXPORT_FILE)) { + id = 'variables'; + name = 'variables'; + type = 'variables'; + } + + if (fileName.startsWith(VERSION_CONTROL_TAGS_EXPORT_FILE)) { + id = 'tags'; + name = 'tags'; + type = 'tags'; + } + + if (!id) return; + + return { + file: fileName, + id, + name, + type, + status, + location, + conflict, + }; + } + + async getStatus(): Promise { + await this.export(); + await this.stage({}); + await this.gitService.fetch(); + const versionControlledFiles: VersionControlledFile[] = []; + const diffRemote = await this.gitService.diffRemote(); + const diffLocal = await this.gitService.diffLocal(); + const status = await this.gitService.status(); + await Promise.all([ + ...(diffRemote?.files.map(async (e) => { + const resolvedFile = await this.fileNameToVersionControlledFile(e.file, 'remote', status); + if (resolvedFile) { + versionControlledFiles.push(resolvedFile); + } + }) ?? []), + ...(diffLocal?.files.map(async (e) => { + const resolvedFile = await this.fileNameToVersionControlledFile(e.file, 'local', status); + if (resolvedFile) { + versionControlledFiles.push(resolvedFile); + } + }) ?? []), + ]); + versionControlledFiles.forEach((e, index, array) => { + const similarItems = array.filter( + (f) => f.type === e.type && (f.file === e.file || f.id === e.id), + ); + if (similarItems.length > 1) { + similarItems.forEach((item) => { + item.conflict = true; + }); + } + }); + return versionControlledFiles; + } + + // #region Version Control Test Functions + //TODO: SEPARATE FUNCTIONS FOR DEVELOPMENT ONLY + //TODO: REMOVE THESE FUNCTIONS AFTER TESTING + + async commit(message?: string): Promise { + return this.gitService.commit(message ?? 'Updated Workfolder'); + } + + async fetch(): Promise { + return this.gitService.fetch(); + } + + async diff(): Promise { + return this.gitService.diff(); + } + + async pull(): Promise { + return this.gitService.pull(); + } + + async push(force = false): Promise { + return this.gitService.push({ + branch: this.versionControlPreferencesService.getBranchName(), + force, + }); + } + // #endregion } diff --git a/packages/cli/src/environments/versionControl/versionControlExport.service.ee.ts b/packages/cli/src/environments/versionControl/versionControlExport.service.ee.ts new file mode 100644 index 0000000000..413c4009e6 --- /dev/null +++ b/packages/cli/src/environments/versionControl/versionControlExport.service.ee.ts @@ -0,0 +1,674 @@ +import Container, { Service } from 'typedi'; +import path from 'path'; +import { + VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER, + VERSION_CONTROL_GIT_FOLDER, + VERSION_CONTROL_TAGS_EXPORT_FILE, + VERSION_CONTROL_VARIABLES_EXPORT_FILE, + VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER, +} from './constants'; +import * as Db from '@/Db'; +import glob from 'fast-glob'; +import type { ICredentialDataDecryptedObject } from 'n8n-workflow'; +import { LoggerProxy, jsonParse } from 'n8n-workflow'; +import { writeFile as fsWriteFile, readFile as fsReadFile, rm as fsRm } from 'fs/promises'; +import { VersionControlGitService } from './versionControlGit.service.ee'; +import { Credentials, UserSettings } from 'n8n-core'; +import type { IWorkflowToImport } from '@/Interfaces'; +import type { ExportableWorkflow } from './types/exportableWorkflow'; +import type { ExportableCredential } from './types/exportableCredential'; +import type { ExportResult } from './types/exportResult'; +import { SharedWorkflow } from '@/databases/entities/SharedWorkflow'; +import { CredentialsEntity } from '@/databases/entities/CredentialsEntity'; +import { Variables } from '@/databases/entities/Variables'; +import type { ImportResult } from './types/importResult'; +import { UM_FIX_INSTRUCTION } from '@/commands/BaseCommand'; +import config from '@/config'; +import { SharedCredentials } from '@/databases/entities/SharedCredentials'; +import { WorkflowEntity } from '@/databases/entities/WorkflowEntity'; +import { WorkflowTagMapping } from '@/databases/entities/WorkflowTagMapping'; +import { TagEntity } from '@/databases/entities/TagEntity'; +import { ActiveWorkflowRunner } from '../../ActiveWorkflowRunner'; +import without from 'lodash.without'; +import type { VersionControllPullOptions } from './types/versionControlPullWorkFolder'; +import { versionControlFoldersExistCheck } from './versionControlHelper.ee'; +import { In } from 'typeorm'; + +@Service() +export class VersionControlExportService { + private gitFolder: string; + + private workflowExportFolder: string; + + private credentialExportFolder: string; + + constructor(private gitService: VersionControlGitService) { + const userFolder = UserSettings.getUserN8nFolderPath(); + this.gitFolder = path.join(userFolder, VERSION_CONTROL_GIT_FOLDER); + this.workflowExportFolder = path.join(this.gitFolder, VERSION_CONTROL_WORKFLOW_EXPORT_FOLDER); + this.credentialExportFolder = path.join( + this.gitFolder, + VERSION_CONTROL_CREDENTIAL_EXPORT_FOLDER, + ); + } + + getWorkflowPath(workflowId: string): string { + return path.join(this.workflowExportFolder, `${workflowId}.json`); + } + + getCredentialsPath(credentialsId: string): string { + return path.join(this.credentialExportFolder, `${credentialsId}.json`); + } + + getTagsPath(): string { + return path.join(this.gitFolder, VERSION_CONTROL_TAGS_EXPORT_FILE); + } + + getVariablesPath(): string { + return path.join(this.gitFolder, VERSION_CONTROL_VARIABLES_EXPORT_FILE); + } + + async getWorkflowFromFile( + filePath: string, + root = this.gitFolder, + ): Promise { + try { + const importedWorkflow = jsonParse( + await fsReadFile(path.join(root, filePath), { encoding: 'utf8' }), + ); + return importedWorkflow; + } catch (error) { + return undefined; + } + } + + async getCredentialFromFile( + filePath: string, + root = this.gitFolder, + ): Promise { + try { + const credential = jsonParse( + await fsReadFile(path.join(root, filePath), { encoding: 'utf8' }), + ); + return credential; + } catch (error) { + return undefined; + } + } + + private async getOwnerGlobalRole() { + const ownerCredentiallRole = await Db.collections.Role.findOne({ + where: { name: 'owner', scope: 'global' }, + }); + + if (!ownerCredentiallRole) { + throw new Error(`Failed to find owner. ${UM_FIX_INSTRUCTION}`); + } + + return ownerCredentiallRole; + } + + private async getOwnerCredentialRole() { + const ownerCredentiallRole = await Db.collections.Role.findOne({ + where: { name: 'owner', scope: 'credential' }, + }); + + if (!ownerCredentiallRole) { + throw new Error(`Failed to find owner. ${UM_FIX_INSTRUCTION}`); + } + + return ownerCredentiallRole; + } + + private async getOwnerWorkflowRole() { + const ownerWorkflowRole = await Db.collections.Role.findOne({ + where: { name: 'owner', scope: 'workflow' }, + }); + + if (!ownerWorkflowRole) { + throw new Error(`Failed to find owner workflow role. ${UM_FIX_INSTRUCTION}`); + } + + return ownerWorkflowRole; + } + + async cleanWorkFolder() { + try { + const workflowFiles = await glob('*.json', { + cwd: this.workflowExportFolder, + absolute: true, + }); + const credentialFiles = await glob('*.json', { + cwd: this.credentialExportFolder, + absolute: true, + }); + const variablesFile = await glob(VERSION_CONTROL_VARIABLES_EXPORT_FILE, { + cwd: this.gitFolder, + absolute: true, + }); + const tagsFile = await glob(VERSION_CONTROL_TAGS_EXPORT_FILE, { + cwd: this.gitFolder, + absolute: true, + }); + await Promise.all(tagsFile.map(async (e) => fsRm(e))); + await Promise.all(variablesFile.map(async (e) => fsRm(e))); + await Promise.all(workflowFiles.map(async (e) => fsRm(e))); + await Promise.all(credentialFiles.map(async (e) => fsRm(e))); + LoggerProxy.debug('Cleaned work folder.'); + } catch (error) { + LoggerProxy.error(`Failed to clean work folder: ${(error as Error).message}`); + } + } + + async deleteRepositoryFolder() { + try { + await fsRm(this.gitFolder, { recursive: true }); + } catch (error) { + LoggerProxy.error(`Failed to delete work folder: ${(error as Error).message}`); + } + } + + private async rmDeletedWorkflowsFromExportFolder( + workflowsToBeExported: SharedWorkflow[], + ): Promise> { + const sharedWorkflowsFileNames = new Set( + workflowsToBeExported.map((e) => this.getWorkflowPath(e?.workflow?.name)), + ); + const existingWorkflowsInFolder = new Set( + await glob('*.json', { + cwd: this.workflowExportFolder, + absolute: true, + }), + ); + const deletedWorkflows = new Set(existingWorkflowsInFolder); + for (const elem of sharedWorkflowsFileNames) { + deletedWorkflows.delete(elem); + } + try { + await Promise.all([...deletedWorkflows].map(async (e) => fsRm(e))); + } catch (error) { + LoggerProxy.error(`Failed to delete workflows from work folder: ${(error as Error).message}`); + } + return deletedWorkflows; + } + + private async writeExportableWorkflowsToExportFolder(workflowsToBeExported: SharedWorkflow[]) { + await Promise.all( + workflowsToBeExported.map(async (e) => { + if (!e.workflow) { + LoggerProxy.debug( + `Found no corresponding workflow ${e.workflowId ?? 'unknown'}, skipping export`, + ); + return; + } + const fileName = this.getWorkflowPath(e.workflow?.id); + const sanitizedWorkflow: ExportableWorkflow = { + active: e.workflow?.active, + id: e.workflow?.id, + name: e.workflow?.name, + nodes: e.workflow?.nodes, + connections: e.workflow?.connections, + settings: e.workflow?.settings, + triggerCount: e.workflow?.triggerCount, + owner: e.user.email, + versionId: e.workflow?.versionId, + }; + LoggerProxy.debug(`Writing workflow ${e.workflowId} to ${fileName}`); + return fsWriteFile(fileName, JSON.stringify(sanitizedWorkflow, null, 2)); + }), + ); + } + + async exportWorkflowsToWorkFolder(): Promise { + try { + versionControlFoldersExistCheck([this.workflowExportFolder]); + const sharedWorkflows = await Db.collections.SharedWorkflow.find({ + relations: ['workflow', 'role', 'user'], + where: { + role: { + name: 'owner', + scope: 'workflow', + }, + }, + }); + + // before exporting, figure out which workflows have been deleted and remove them from the export folder + const removedFiles = await this.rmDeletedWorkflowsFromExportFolder(sharedWorkflows); + // write the workflows to the export folder as json files + await this.writeExportableWorkflowsToExportFolder(sharedWorkflows); + return { + count: sharedWorkflows.length, + folder: this.workflowExportFolder, + files: sharedWorkflows.map((e) => ({ + id: e?.workflow?.id, + name: this.getWorkflowPath(e?.workflow?.name), + })), + removedFiles: [...removedFiles], + }; + } catch (error) { + throw Error(`Failed to export workflows to work folder: ${(error as Error).message}`); + } + } + + async exportVariablesToWorkFolder(): Promise { + try { + versionControlFoldersExistCheck([this.gitFolder]); + const variables = await Db.collections.Variables.find(); + // do not export empty variables + if (variables.length === 0) { + return { + count: 0, + folder: this.gitFolder, + files: [], + }; + } + const fileName = this.getVariablesPath(); + const sanitizedVariables = variables.map((e) => ({ ...e, value: '' })); + await fsWriteFile(fileName, JSON.stringify(sanitizedVariables, null, 2)); + return { + count: sanitizedVariables.length, + folder: this.gitFolder, + files: [ + { + id: '', + name: fileName, + }, + ], + }; + } catch (error) { + throw Error(`Failed to export variables to work folder: ${(error as Error).message}`); + } + } + + async exportTagsToWorkFolder(): Promise { + try { + versionControlFoldersExistCheck([this.gitFolder]); + const tags = await Db.collections.Tag.find(); + const mappings = await Db.collections.WorkflowTagMapping.find(); + const fileName = this.getTagsPath(); + await fsWriteFile( + fileName, + JSON.stringify( + { + tags: tags.map((tag) => ({ id: tag.id, name: tag.name })), + mappings, + }, + null, + 2, + ), + ); + return { + count: tags.length, + folder: this.gitFolder, + files: [ + { + id: '', + name: fileName, + }, + ], + }; + } catch (error) { + throw Error(`Failed to export variables to work folder: ${(error as Error).message}`); + } + } + + private replaceCredentialData = ( + data: ICredentialDataDecryptedObject, + ): ICredentialDataDecryptedObject => { + for (const [key] of Object.entries(data)) { + try { + if (typeof data[key] === 'object') { + data[key] = this.replaceCredentialData(data[key] as ICredentialDataDecryptedObject); + } else if (typeof data[key] === 'string') { + data[key] = (data[key] as string)?.startsWith('={{') ? data[key] : ''; + } else if (typeof data[key] === 'number') { + // TODO: leaving numbers in for now, but maybe we should remove them + // data[key] = 0; + } + } catch (error) { + LoggerProxy.error(`Failed to sanitize credential data: ${(error as Error).message}`); + throw error; + } + } + return data; + }; + + async exportCredentialsToWorkFolder(): Promise { + try { + versionControlFoldersExistCheck([this.credentialExportFolder]); + const sharedCredentials = await Db.collections.SharedCredentials.find({ + relations: ['credentials', 'role', 'user'], + }); + const encryptionKey = await UserSettings.getEncryptionKey(); + await Promise.all( + sharedCredentials.map(async (sharedCredential) => { + const { name, type, nodesAccess, data, id } = sharedCredential.credentials; + const credentialObject = new Credentials({ id, name }, type, nodesAccess, data); + const plainData = credentialObject.getData(encryptionKey); + const sanitizedData = this.replaceCredentialData(plainData); + const fileName = path.join( + this.credentialExportFolder, + `${sharedCredential.credentials.id}.json`, + ); + const sanitizedCredential: ExportableCredential = { + id: sharedCredential.credentials.id, + name: sharedCredential.credentials.name, + type: sharedCredential.credentials.type, + data: sanitizedData, + }; + LoggerProxy.debug(`Writing credential ${sharedCredential.credentials.id} to ${fileName}`); + return fsWriteFile(fileName, JSON.stringify(sanitizedCredential, null, 2)); + }), + ); + return { + count: sharedCredentials.length, + folder: this.credentialExportFolder, + files: sharedCredentials.map((e) => ({ + id: e.credentials.id, + name: path.join(this.credentialExportFolder, `${e.credentials.name}.json`), + })), + }; + } catch (error) { + throw Error(`Failed to export credentials to work folder: ${(error as Error).message}`); + } + } + + private async importCredentialsFromFiles( + userId: string, + ): Promise> { + const credentialFiles = await glob('*.json', { + cwd: this.credentialExportFolder, + absolute: true, + }); + const existingCredentials = await Db.collections.Credentials.find(); + const ownerCredentialRole = await this.getOwnerCredentialRole(); + const ownerGlobalRole = await this.getOwnerGlobalRole(); + const encryptionKey = await UserSettings.getEncryptionKey(); + let importCredentialsResult: Array<{ id: string; name: string; type: string }> = []; + await Db.transaction(async (transactionManager) => { + importCredentialsResult = await Promise.all( + credentialFiles.map(async (file) => { + LoggerProxy.debug(`Importing credentials file ${file}`); + const credential = jsonParse( + await fsReadFile(file, { encoding: 'utf8' }), + ); + const existingCredential = existingCredentials.find( + (e) => e.id === credential.id && e.type === credential.type, + ); + const sharedOwner = await Db.collections.SharedCredentials.findOne({ + select: ['userId'], + where: { + credentialsId: credential.id, + roleId: In([ownerCredentialRole.id, ownerGlobalRole.id]), + }, + }); + + const { name, type, data, id } = credential; + const newCredentialObject = new Credentials({ id, name }, type, []); + if (existingCredential?.data) { + newCredentialObject.data = existingCredential.data; + } else { + newCredentialObject.setData(data, encryptionKey); + } + if (existingCredential?.nodesAccess) { + newCredentialObject.nodesAccess = existingCredential.nodesAccess; + } + + LoggerProxy.debug(`Updating credential id ${newCredentialObject.id as string}`); + await transactionManager.upsert(CredentialsEntity, newCredentialObject, ['id']); + + if (!sharedOwner) { + const newSharedCredential = new SharedCredentials(); + newSharedCredential.credentialsId = newCredentialObject.id as string; + newSharedCredential.userId = userId; + newSharedCredential.roleId = ownerGlobalRole.id; + + await transactionManager.upsert(SharedCredentials, { ...newSharedCredential }, [ + 'credentialsId', + 'userId', + ]); + } + + // TODO: once IDs are unique, remove this + if (config.getEnv('database.type') === 'postgresdb') { + await transactionManager.query( + "SELECT setval('credentials_entity_id_seq', (SELECT MAX(id) from credentials_entity))", + ); + } + return { + id: newCredentialObject.id as string, + name: newCredentialObject.name, + type: newCredentialObject.type, + }; + }), + ); + }); + return importCredentialsResult.filter((e) => e !== undefined); + } + + private async importVariablesFromFile(valueOverrides?: { + [key: string]: string; + }): Promise<{ added: string[]; changed: string[] }> { + const variablesFile = await glob(VERSION_CONTROL_VARIABLES_EXPORT_FILE, { + cwd: this.gitFolder, + absolute: true, + }); + if (variablesFile.length > 0) { + LoggerProxy.debug(`Importing variables from file ${variablesFile[0]}`); + const overriddenKeys = Object.keys(valueOverrides ?? {}); + const importedVariables = jsonParse( + await fsReadFile(variablesFile[0], { encoding: 'utf8' }), + { fallbackValue: [] }, + ); + const importedKeys = importedVariables.map((variable) => variable.key); + const existingVariables = await Db.collections.Variables.find(); + const existingKeys = existingVariables.map((variable) => variable.key); + const addedKeysFromImport = without(importedKeys, ...existingKeys); + const addedKeysFromOverride = without(overriddenKeys, ...existingKeys); + const addedVariables = importedVariables.filter((e) => addedKeysFromImport.includes(e.key)); + addedKeysFromOverride.forEach((key) => { + addedVariables.push({ + key, + value: valueOverrides ? valueOverrides[key] : '', + type: 'string', + } as Variables); + }); + + // first round, add missing variable keys to Db without touching values + await Db.transaction(async (transactionManager) => { + await Promise.all( + addedVariables.map(async (addedVariable) => { + await transactionManager.insert(Variables, { + ...addedVariable, + id: undefined, + }); + }), + ); + }); + + // second round, update values of existing variables if overridden + if (valueOverrides) { + await Db.transaction(async (transactionManager) => { + await Promise.all( + overriddenKeys.map(async (key) => { + await transactionManager.update(Variables, { key }, { value: valueOverrides[key] }); + }), + ); + }); + } + return { + added: [...addedKeysFromImport, ...addedKeysFromOverride], + changed: without(overriddenKeys, ...addedKeysFromOverride), + }; + } + return { added: [], changed: [] }; + } + + private async importTagsFromFile() { + const tagsFile = await glob(VERSION_CONTROL_TAGS_EXPORT_FILE, { + cwd: this.gitFolder, + absolute: true, + }); + if (tagsFile.length > 0) { + LoggerProxy.debug(`Importing tags from file ${tagsFile[0]}`); + const mappedTags = jsonParse<{ tags: TagEntity[]; mappings: WorkflowTagMapping[] }>( + await fsReadFile(tagsFile[0], { encoding: 'utf8' }), + { fallbackValue: { tags: [], mappings: [] } }, + ); + const existingWorkflowIds = new Set( + ( + await Db.collections.Workflow.find({ + select: ['id'], + }) + ).map((e) => e.id), + ); + + await Db.transaction(async (transactionManager) => { + await Promise.all( + mappedTags.tags.map(async (tag) => { + await transactionManager.upsert( + TagEntity, + { + ...tag, + }, + { + skipUpdateIfNoValuesChanged: true, + conflictPaths: { id: true }, + }, + ); + }), + ); + await Promise.all( + mappedTags.mappings.map(async (mapping) => { + if (!existingWorkflowIds.has(String(mapping.workflowId))) return; + await transactionManager.upsert( + WorkflowTagMapping, + { tagId: String(mapping.tagId), workflowId: String(mapping.workflowId) }, + { + skipUpdateIfNoValuesChanged: true, + conflictPaths: { tagId: true, workflowId: true }, + }, + ); + }), + ); + }); + return mappedTags; + } + return { tags: [], mappings: [] }; + } + + private async importWorkflowsFromFiles( + userId: string, + ): Promise> { + const workflowFiles = await glob('*.json', { + cwd: this.workflowExportFolder, + absolute: true, + }); + + const existingWorkflows = await Db.collections.Workflow.find({ + select: ['id', 'name', 'active', 'versionId'], + }); + + const ownerWorkflowRole = await this.getOwnerWorkflowRole(); + const workflowRunner = Container.get(ActiveWorkflowRunner); + + let importWorkflowsResult = new Array<{ id: string; name: string }>(); + // TODO: once IDs are unique and we removed autoincrement, remove this + if (config.getEnv('database.type') === 'postgresdb') { + await Db.transaction(async (transactionManager) => { + await transactionManager.query( + 'ALTER SEQUENCE IF EXISTS "workflow_entity_id_seq" RESTART;', + ); + await transactionManager.query( + "SELECT setval('workflow_entity_id_seq', (SELECT MAX(id) from workflow_entity) );", + // "SELECT setval('workflow_entity_id_seq', (SELECT MAX(v) FROM (VALUES (1), ((SELECT MAX(id) from workflow_entity))) as value(v)));", + ); + }); + } + await Db.transaction(async (transactionManager) => { + importWorkflowsResult = await Promise.all( + workflowFiles.map(async (file) => { + LoggerProxy.debug(`Parsing workflow file ${file}`); + const importedWorkflow = jsonParse( + await fsReadFile(file, { encoding: 'utf8' }), + ); + const existingWorkflow = existingWorkflows.find((e) => e.id === importedWorkflow.id); + if (existingWorkflow?.versionId === importedWorkflow.versionId) { + LoggerProxy.debug( + `Skipping import of workflow ${ + importedWorkflow.id ?? 'n/a' + } - versionId is up to date`, + ); + return { + id: importedWorkflow.id ?? 'n/a', + name: 'skipped', + }; + } + LoggerProxy.debug(`Importing workflow ${importedWorkflow.id ?? 'n/a'}`); + importedWorkflow.active = existingWorkflow?.active ?? false; + LoggerProxy.debug(`Updating workflow id ${importedWorkflow.id ?? 'new'}`); + const upsertResult = await transactionManager.upsert( + WorkflowEntity, + { ...importedWorkflow }, + ['id'], + ); + if (upsertResult?.identifiers?.length !== 1) { + throw new Error(`Failed to upsert workflow ${importedWorkflow.id ?? 'new'}`); + } + // due to sequential Ids, this may have changed during the insert + // TODO: once IDs are unique and we removed autoincrement, remove this + const upsertedWorkflowId = upsertResult.identifiers[0].id as string; + await transactionManager.upsert( + SharedWorkflow, + { + workflowId: upsertedWorkflowId, + userId, + roleId: ownerWorkflowRole.id, + }, + ['workflowId', 'userId'], + ); + + if (existingWorkflow?.active) { + try { + // remove active pre-import workflow + LoggerProxy.debug(`Deactivating workflow id ${existingWorkflow.id}`); + await workflowRunner.remove(existingWorkflow.id); + // try activating the imported workflow + LoggerProxy.debug(`Reactivating workflow id ${existingWorkflow.id}`); + await workflowRunner.add(existingWorkflow.id, 'activate'); + } catch (error) { + LoggerProxy.error( + `Failed to activate workflow ${existingWorkflow.id}`, + error as Error, + ); + } + } + + return { + id: importedWorkflow.id ?? 'unknown', + name: file, + }; + }), + ); + }); + return importWorkflowsResult; + } + + async importFromWorkFolder(options: VersionControllPullOptions): Promise { + try { + const importedVariables = await this.importVariablesFromFile(options.variables); + const importedCredentials = await this.importCredentialsFromFiles(options.userId); + const importWorkflows = await this.importWorkflowsFromFiles(options.userId); + const importTags = await this.importTagsFromFile(); + + return { + variables: importedVariables, + credentials: importedCredentials, + workflows: importWorkflows, + tags: importTags, + }; + } catch (error) { + throw Error(`Failed to import workflows from work folder: ${(error as Error).message}`); + } + } +} diff --git a/packages/cli/src/environments/versionControl/versionControlGit.service.ee.ts b/packages/cli/src/environments/versionControl/versionControlGit.service.ee.ts new file mode 100644 index 0000000000..c038b09371 --- /dev/null +++ b/packages/cli/src/environments/versionControl/versionControlGit.service.ee.ts @@ -0,0 +1,330 @@ +import { Service } from 'typedi'; +import { execSync } from 'child_process'; +import { LoggerProxy } from 'n8n-workflow'; +import path from 'path'; +import type { + CommitResult, + DiffResult, + FetchResult, + PullResult, + PushResult, + SimpleGit, + SimpleGitOptions, + StatusResult, +} from 'simple-git'; +import { simpleGit } from 'simple-git'; +import type { VersionControlPreferences } from './types/versionControlPreferences'; +import { VERSION_CONTROL_DEFAULT_BRANCH, VERSION_CONTROL_ORIGIN } from './constants'; +import { versionControlFoldersExistCheck } from './versionControlHelper.ee'; + +@Service() +export class VersionControlGitService { + git: SimpleGit | null = null; + + private gitOptions: Partial = {}; + + /** + * Run pre-checks before initialising git + * Checks for existence of required binaries (git and ssh) + */ + preInitCheck(): boolean { + LoggerProxy.debug('GitService.preCheck'); + try { + const gitResult = execSync('git --version', { + stdio: ['pipe', 'pipe', 'pipe'], + }); + LoggerProxy.debug(`Git binary found: ${gitResult.toString()}`); + } catch (error) { + throw new Error(`Git binary not found: ${(error as Error).message}`); + } + try { + const sshResult = execSync('ssh -V', { + stdio: ['pipe', 'pipe', 'pipe'], + }); + LoggerProxy.debug(`SSH binary found: ${sshResult.toString()}`); + } catch (error) { + throw new Error(`SSH binary not found: ${(error as Error).message}`); + } + return true; + } + + async initService(options: { + versionControlPreferences: VersionControlPreferences; + gitFolder: string; + sshFolder: string; + sshKeyName: string; + }): Promise { + const { versionControlPreferences, gitFolder, sshKeyName, sshFolder } = options; + LoggerProxy.debug('GitService.init'); + if (this.git !== null) { + return; + } + + this.preInitCheck(); + LoggerProxy.debug('Git pre-check passed'); + + versionControlFoldersExistCheck([gitFolder, sshFolder]); + + const sshKnownHosts = path.join(sshFolder, 'known_hosts'); + const sshCommand = `ssh -o UserKnownHostsFile=${sshKnownHosts} -o StrictHostKeyChecking=no -i ${sshKeyName}`; + + this.gitOptions = { + baseDir: gitFolder, + binary: 'git', + maxConcurrentProcesses: 6, + trimmed: false, + }; + + this.git = simpleGit(this.gitOptions) + // Tell git not to ask for any information via the terminal like for + // example the username. As nobody will be able to answer it would + // n8n keep on waiting forever. + .env('GIT_SSH_COMMAND', sshCommand) + .env('GIT_TERMINAL_PROMPT', '0'); + + if (!(await this.checkRepositorySetup())) { + await this.git.init(); + } + if (!(await this.hasRemote(versionControlPreferences.repositoryUrl))) { + if (versionControlPreferences.connected && versionControlPreferences.repositoryUrl) { + await this.initRepository(versionControlPreferences); + } + } + } + + resetService() { + this.git = null; + } + + resetLocalRepository() { + // TODO: Implement + this.git = null; + } + + async checkRepositorySetup(): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + if (!(await this.git.checkIsRepo())) { + return false; + } + try { + await this.git.status(); + return true; + } catch (error) { + return false; + } + } + + async hasRemote(remote: string): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + try { + const remotes = await this.git.getRemotes(true); + const foundRemote = remotes.find( + (e) => e.name === VERSION_CONTROL_ORIGIN && e.refs.push === remote, + ); + if (foundRemote) { + LoggerProxy.debug(`Git remote found: ${foundRemote.name}: ${foundRemote.refs.push}`); + return true; + } + } catch (error) { + throw new Error(`Git is not initialized ${(error as Error).message}`); + } + LoggerProxy.debug(`Git remote not found: ${remote}`); + return false; + } + + async initRepository( + versionControlPreferences: Pick< + VersionControlPreferences, + 'repositoryUrl' | 'authorEmail' | 'authorName' | 'branchName' | 'initRepo' + >, + ): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + if (versionControlPreferences.initRepo) { + try { + await this.git.init(); + } catch (error) { + LoggerProxy.debug(`Git init: ${(error as Error).message}`); + } + } + try { + await this.git.addRemote(VERSION_CONTROL_ORIGIN, versionControlPreferences.repositoryUrl); + } catch (error) { + if ((error as Error).message.includes('remote origin already exists')) { + LoggerProxy.debug(`Git remote already exists: ${(error as Error).message}`); + } else { + throw error; + } + } + await this.git.addConfig('user.email', versionControlPreferences.authorEmail); + await this.git.addConfig('user.name', versionControlPreferences.authorName); + if (versionControlPreferences.initRepo) { + try { + const branches = await this.getBranches(); + if (branches.branches?.length === 0) { + await this.git.raw(['branch', '-M', versionControlPreferences.branchName]); + } + } catch (error) { + LoggerProxy.debug(`Git init: ${(error as Error).message}`); + } + } + } + + async getBranches(): Promise<{ branches: string[]; currentBranch: string }> { + if (!this.git) { + throw new Error('Git is not initialized'); + } + + try { + // Get remote branches + const { branches } = await this.git.branch(['-r']); + const remoteBranches = Object.keys(branches) + .map((name) => name.split('/')[1]) + .filter((name) => name !== 'HEAD'); + + const { current } = await this.git.branch(); + + return { + branches: remoteBranches, + currentBranch: current, + }; + } catch (error) { + throw new Error(`Could not get remote branches from repository ${(error as Error).message}`); + } + } + + async setBranch(branch: string): Promise<{ branches: string[]; currentBranch: string }> { + if (!this.git) { + throw new Error('Git is not initialized'); + } + await this.git.checkout(branch); + return this.getBranches(); + } + + async fetch(): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + return this.git.fetch(); + } + + async getCurrentBranch(): Promise<{ current: string; remote: string }> { + if (!this.git) { + throw new Error('Git is not initialized'); + } + const currentBranch = (await this.git.branch()).current; + return { + current: currentBranch, + remote: 'origin/' + currentBranch, + }; + } + + async diff(options?: { target?: string; dots?: '..' | '...' }): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + const currentBranch = await this.getCurrentBranch(); + const target = options?.target ?? currentBranch.remote; + const dots = options?.dots ?? '...'; + return this.git.diffSummary([dots + target]); + } + + async diffRemote(): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + const currentBranch = await this.getCurrentBranch(); + if (currentBranch.remote) { + const target = currentBranch.remote; + return this.git.diffSummary(['...' + target]); + } + return; + } + + async diffLocal(): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + const currentBranch = await this.getCurrentBranch(); + if (currentBranch.remote) { + const target = currentBranch.current; + return this.git.diffSummary([target]); + } + return; + } + + async pull(options: { ffOnly: boolean } = { ffOnly: true }): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + if (options.ffOnly) { + // eslint-disable-next-line @typescript-eslint/naming-convention + return this.git.pull(undefined, undefined, { '--ff-only': null }); + } + return this.git.pull(); + } + + async push( + options: { force: boolean; branch: string } = { + force: false, + branch: VERSION_CONTROL_DEFAULT_BRANCH, + }, + ): Promise { + const { force, branch } = options; + if (!this.git) { + throw new Error('Git is not initialized'); + } + if (force) { + return this.git.push(VERSION_CONTROL_ORIGIN, branch, ['-f']); + } + return this.git.push(VERSION_CONTROL_ORIGIN, branch); + } + + async stage(files: Set, deletedFiles?: Set): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + if (deletedFiles?.size) { + try { + await this.git.rm(Array.from(deletedFiles)); + } catch (error) { + LoggerProxy.debug(`Git rm: ${(error as Error).message}`); + } + } + return this.git.add(Array.from(files)); + } + + async resetBranch( + options: { hard?: boolean; target: string } = { hard: false, target: 'HEAD' }, + ): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + if (options?.hard) { + return this.git.raw(['reset', '--hard', options.target]); + } + return this.git.raw(['reset', options.target]); + // built-in reset method does not work + // return this.git.reset(); + } + + async commit(message: string): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + return this.git.commit(message); + } + + async status(): Promise { + if (!this.git) { + throw new Error('Git is not initialized'); + } + const statusResult = await this.git.status(); + return statusResult; + } +} diff --git a/packages/cli/src/environments/versionControl/versionControlHelper.ts b/packages/cli/src/environments/versionControl/versionControlHelper.ee.ts similarity index 67% rename from packages/cli/src/environments/versionControl/versionControlHelper.ts rename to packages/cli/src/environments/versionControl/versionControlHelper.ee.ts index 9810977f91..b27f236035 100644 --- a/packages/cli/src/environments/versionControl/versionControlHelper.ts +++ b/packages/cli/src/environments/versionControl/versionControlHelper.ee.ts @@ -3,21 +3,30 @@ import { License } from '../../License'; import { generateKeyPairSync } from 'crypto'; import sshpk from 'sshpk'; import type { KeyPair } from './types/keyPair'; +import { constants as fsConstants, mkdirSync, accessSync } from 'fs'; +import { LoggerProxy } from 'n8n-workflow'; +import { VERSION_CONTROL_GIT_KEY_COMMENT } from './constants'; + +export function versionControlFoldersExistCheck(folders: string[]) { + // running these file access function synchronously to avoid race conditions + folders.forEach((folder) => { + try { + accessSync(folder, fsConstants.F_OK); + } catch { + try { + mkdirSync(folder); + } catch (error) { + LoggerProxy.error((error as Error).message); + } + } + }); +} export function isVersionControlLicensed() { const license = Container.get(License); return license.isVersionControlLicensed(); } -export function isVersionControlEnabled() { - // TODO: VERSION CONTROL check if enabled - return true; -} - -export function isVersionControlLicensedAndEnabled() { - return isVersionControlLicensed() && isVersionControlEnabled(); -} - export function generateSshKeyPair(keyType: 'ed25519' | 'rsa' = 'ed25519') { const keyPair: KeyPair = { publicKey: '', @@ -46,8 +55,10 @@ export function generateSshKeyPair(keyType: 'ed25519' | 'rsa' = 'ed25519') { break; } const keyPublic = sshpk.parseKey(generatedKeyPair.publicKey, 'pem'); + keyPublic.comment = VERSION_CONTROL_GIT_KEY_COMMENT; keyPair.publicKey = keyPublic.toString('ssh'); const keyPrivate = sshpk.parsePrivateKey(generatedKeyPair.privateKey, 'pem'); + keyPrivate.comment = VERSION_CONTROL_GIT_KEY_COMMENT; keyPair.privateKey = keyPrivate.toString('ssh-private'); return { privateKey: keyPair.privateKey, diff --git a/packages/cli/src/environments/versionControl/versionControlPreferences.service.ee.ts b/packages/cli/src/environments/versionControl/versionControlPreferences.service.ee.ts new file mode 100644 index 0000000000..6233f53f97 --- /dev/null +++ b/packages/cli/src/environments/versionControl/versionControlPreferences.service.ee.ts @@ -0,0 +1,191 @@ +import { Service } from 'typedi'; +import { VersionControlPreferences } from './types/versionControlPreferences'; +import type { ValidationError } from 'class-validator'; +import { validate } from 'class-validator'; +import { readFileSync as fsReadFileSync, existsSync as fsExistsSync } from 'fs'; +import { writeFile as fsWriteFile, rm as fsRm } from 'fs/promises'; +import { + generateSshKeyPair, + isVersionControlLicensed, + versionControlFoldersExistCheck, +} from './versionControlHelper.ee'; +import { UserSettings } from 'n8n-core'; +import { LoggerProxy, jsonParse } from 'n8n-workflow'; +import * as Db from '@/Db'; +import { + VERSION_CONTROL_SSH_FOLDER, + VERSION_CONTROL_GIT_FOLDER, + VERSION_CONTROL_SSH_KEY_NAME, + VERSION_CONTROL_PREFERENCES_DB_KEY, +} from './constants'; +import path from 'path'; + +@Service() +export class VersionControlPreferencesService { + private _versionControlPreferences: VersionControlPreferences = new VersionControlPreferences(); + + private sshKeyName: string; + + private sshFolder: string; + + private gitFolder: string; + + constructor() { + const userFolder = UserSettings.getUserN8nFolderPath(); + this.sshFolder = path.join(userFolder, VERSION_CONTROL_SSH_FOLDER); + this.gitFolder = path.join(userFolder, VERSION_CONTROL_GIT_FOLDER); + this.sshKeyName = path.join(this.sshFolder, VERSION_CONTROL_SSH_KEY_NAME); + } + + public get versionControlPreferences(): VersionControlPreferences { + return { + ...this._versionControlPreferences, + connected: this._versionControlPreferences.connected ?? false, + publicKey: this.getPublicKey(), + }; + } + + public set versionControlPreferences(preferences: Partial) { + this._versionControlPreferences = VersionControlPreferences.merge( + preferences, + this._versionControlPreferences, + ); + } + + getPublicKey(): string { + try { + return fsReadFileSync(this.sshKeyName + '.pub', { encoding: 'utf8' }); + } catch (error) { + LoggerProxy.error(`Failed to read public key: ${(error as Error).message}`); + } + return ''; + } + + hasKeyPairFiles(): boolean { + return fsExistsSync(this.sshKeyName) && fsExistsSync(this.sshKeyName + '.pub'); + } + + async deleteKeyPairFiles(): Promise { + try { + await fsRm(this.sshFolder, { recursive: true }); + } catch (error) { + LoggerProxy.error(`Failed to delete ssh folder: ${(error as Error).message}`); + } + } + + /** + * Will generate an ed25519 key pair and save it to the database and the file system + * Note: this will overwrite any existing key pair + */ + async generateAndSaveKeyPair(): Promise { + versionControlFoldersExistCheck([this.gitFolder, this.sshFolder]); + const keyPair = generateSshKeyPair('ed25519'); + if (keyPair.publicKey && keyPair.privateKey) { + try { + await fsWriteFile(this.sshKeyName + '.pub', keyPair.publicKey, { + encoding: 'utf8', + mode: 0o666, + }); + await fsWriteFile(this.sshKeyName, keyPair.privateKey, { encoding: 'utf8', mode: 0o600 }); + } catch (error) { + throw Error(`Failed to save key pair: ${(error as Error).message}`); + } + } + return this.getPreferences(); + } + + isBranchReadOnly(): boolean { + return this._versionControlPreferences.branchReadOnly; + } + + isVersionControlConnected(): boolean { + return this.versionControlPreferences.connected; + } + + isVersionControlLicensedAndEnabled(): boolean { + return this.isVersionControlConnected() && isVersionControlLicensed(); + } + + getBranchName(): string { + return this.versionControlPreferences.branchName; + } + + getPreferences(): VersionControlPreferences { + return this.versionControlPreferences; + } + + setBranchReadOnly(branchReadOnly: boolean): void { + this._versionControlPreferences.branchReadOnly = branchReadOnly; + } + + async validateVersionControlPreferences( + preferences: Partial, + allowMissingProperties = true, + ): Promise { + if (this.isVersionControlConnected()) { + if (preferences.repositoryUrl !== this._versionControlPreferences.repositoryUrl) { + throw new Error('Cannot change repository while connected'); + } + } + const preferencesObject = new VersionControlPreferences(preferences); + const validationResult = await validate(preferencesObject, { + forbidUnknownValues: false, + skipMissingProperties: allowMissingProperties, + stopAtFirstError: false, + validationError: { target: false }, + }); + if (validationResult.length > 0) { + throw new Error(`Invalid version control preferences: ${JSON.stringify(validationResult)}`); + } + return validationResult; + } + + async setPreferences( + preferences: Partial, + saveToDb = true, + ): Promise { + versionControlFoldersExistCheck([this.gitFolder, this.sshFolder]); + if (!this.hasKeyPairFiles()) { + LoggerProxy.debug('No key pair files found, generating new pair'); + await this.generateAndSaveKeyPair(); + } + this.versionControlPreferences = preferences; + if (saveToDb) { + const settingsValue = JSON.stringify(this._versionControlPreferences); + try { + await Db.collections.Settings.save({ + key: VERSION_CONTROL_PREFERENCES_DB_KEY, + value: settingsValue, + loadOnStartup: true, + }); + } catch (error) { + throw new Error(`Failed to save version control preferences: ${(error as Error).message}`); + } + } + return this.versionControlPreferences; + } + + async loadFromDbAndApplyVersionControlPreferences(): Promise< + VersionControlPreferences | undefined + > { + const loadedPreferences = await Db.collections.Settings.findOne({ + where: { key: VERSION_CONTROL_PREFERENCES_DB_KEY }, + }); + if (loadedPreferences) { + try { + const preferences = jsonParse(loadedPreferences.value); + if (preferences) { + // set local preferences but don't write back to db + await this.setPreferences(preferences, false); + return preferences; + } + } catch (error) { + LoggerProxy.warn( + `Could not parse Version Control settings from database: ${(error as Error).message}`, + ); + } + } + await this.setPreferences(new VersionControlPreferences(), true); + return this.versionControlPreferences; + } +} diff --git a/packages/cli/test/integration/environments/VersionControl.test.ts b/packages/cli/test/integration/environments/VersionControl.test.ts index d438bf906b..87f6afd9e8 100644 --- a/packages/cli/test/integration/environments/VersionControl.test.ts +++ b/packages/cli/test/integration/environments/VersionControl.test.ts @@ -4,7 +4,7 @@ import type { User } from '@db/entities/User'; import { License } from '@/License'; import * as testDb from '../shared/testDb'; import * as utils from '../shared/utils'; -import { VersionControlService } from '../../../src/environments/versionControl/versionControl.service.ee'; +import { VERSION_CONTROL_API_ROOT } from '@/environments/versionControl/constants'; let owner: User; let authOwnerAgent: SuperAgentTest; @@ -22,17 +22,11 @@ afterAll(async () => { describe('GET /versionControl/preferences', () => { test('should return Version Control preferences', async () => { - await Container.get(VersionControlService).generateAndSaveKeyPair(); await authOwnerAgent - .get('/versionControl/preferences') + .get(`/${VERSION_CONTROL_API_ROOT}/preferences`) .expect(200) .expect((res) => { - return ( - 'privateKey' in res.body && - 'publicKey' in res.body && - res.body.publicKey.includes('ssh-ed25519') && - res.body.privateKey.includes('BEGIN OPENSSH PRIVATE KEY') - ); + return 'repositoryUrl' in res.body && 'branchName' in res.body; }); }); }); diff --git a/packages/cli/test/integration/shared/utils.ts b/packages/cli/test/integration/shared/utils.ts index a3bd47691f..19ccee37d4 100644 --- a/packages/cli/test/integration/shared/utils.ts +++ b/packages/cli/test/integration/shared/utils.ts @@ -81,6 +81,7 @@ import { EventBusController } from '@/eventbus/eventBus.controller'; import { License } from '@/License'; import { VersionControlService } from '@/environments/versionControl/versionControl.service.ee'; import { VersionControlController } from '@/environments/versionControl/versionControl.controller.ee'; +import { VersionControlPreferencesService } from '@/environments/versionControl/versionControlPreferences.service.ee'; export const mockInstance = ( ctor: new (...args: any[]) => T, @@ -203,10 +204,11 @@ export async function initTestServer({ break; case 'versionControl': const versionControlService = Container.get(VersionControlService); + const versionControlPreferencesService = Container.get(VersionControlPreferencesService); registerController( testServer.app, config, - new VersionControlController(versionControlService), + new VersionControlController(versionControlService, versionControlPreferencesService), ); break; case 'nodes': diff --git a/packages/cli/test/unit/VersionControl.test.ts b/packages/cli/test/unit/VersionControl.test.ts index 6e880859ff..827f5494ec 100644 --- a/packages/cli/test/unit/VersionControl.test.ts +++ b/packages/cli/test/unit/VersionControl.test.ts @@ -1,4 +1,4 @@ -import { generateSshKeyPair } from '../../src/environments/versionControl/versionControlHelper'; +import { generateSshKeyPair } from '../../src/environments/versionControl/versionControlHelper.ee'; describe('Version Control', () => { it('should generate an SSH key pair', () => { diff --git a/packages/design-system/src/components/N8nButton/Button.vue b/packages/design-system/src/components/N8nButton/Button.vue index a9ec60719d..3250bf9a26 100644 --- a/packages/design-system/src/components/N8nButton/Button.vue +++ b/packages/design-system/src/components/N8nButton/Button.vue @@ -98,7 +98,7 @@ export default defineComponent({ `${this.disabled ? ` ${this.$style.disabled}` : ''}` + `${this.block ? ` ${this.$style.block}` : ''}` + `${this.active ? ` ${this.$style.active}` : ''}` + - `${this.icon || this.loading ? ` ${this.$style.icon}` : ''}` + + `${this.icon || this.loading ? ` ${this.$style.withIcon}` : ''}` + `${this.square ? ` ${this.$style.square}` : ''}` ); }, @@ -464,6 +464,12 @@ $loading-overlay-background-color: rgba(255, 255, 255, 0); --button-active-background-color: transparent; } +.withIcon { + display: inline-flex; + justify-content: center; + align-items: center; +} + .icon { display: inline-flex; justify-content: center; diff --git a/packages/design-system/src/components/N8nButton/__tests__/__snapshots__/Button.spec.ts.snap b/packages/design-system/src/components/N8nButton/__tests__/__snapshots__/Button.spec.ts.snap index fa45534fe4..f03d6421ce 100644 --- a/packages/design-system/src/components/N8nButton/__tests__/__snapshots__/Button.spec.ts.snap +++ b/packages/design-system/src/components/N8nButton/__tests__/__snapshots__/Button.spec.ts.snap @@ -1,16 +1,16 @@ // Vitest Snapshot v1 -exports[`components > N8nButton > overrides > should render as \`secondary\` when \`text\` is given as type 1`] = `""`; +exports[`components > N8nButton > overrides > should render as \`secondary\` when \`text\` is given as type 1`] = `""`; -exports[`components > N8nButton > overrides > should render as \`tertiary\` when \`info\` is given as type 1`] = `""`; +exports[`components > N8nButton > overrides > should render as \`tertiary\` when \`info\` is given as type 1`] = `""`; -exports[`components > N8nButton > overrides > should use default (\`primary\`) type when no type is given 1`] = `""`; +exports[`components > N8nButton > overrides > should use default (\`primary\`) type when no type is given 1`] = `""`; -exports[`components > N8nButton > overrides > should use given (\`secondary\`) type 1`] = `""`; +exports[`components > N8nButton > overrides > should use given (\`secondary\`) type 1`] = `""`; -exports[`components > N8nButton > props > icon > should render icon button 1`] = `""`; +exports[`components > N8nButton > props > icon > should render icon button 1`] = `""`; -exports[`components > N8nButton > props > loading > should render loading spinner 1`] = `""`; +exports[`components > N8nButton > props > loading > should render loading spinner 1`] = `""`; exports[`components > N8nButton > props > square > should render square button 1`] = ` "