admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 01:56:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Switch binary filesystem mode to nested path structure (#7307)

Browse Source 
Depends on #7253 | Story:
[PAY-863](https://linear.app/n8n/issue/PAY-863/switch-binary-filesystem-mode-to-nested-path-structure)

This PR introduces `filesystem-v2` to store binary data in the
filesystem in the same format as `s3`.
This commit is contained in:
Iván Ovejero
2023-10-10 10:06:06 +02:00
committed by GitHub
parent 86e7ec796a
commit 0847623f85
13 changed files with 326 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
packages/cli/src/executionLifecycleHooks/restoreBinaryDataId.ts
View File

@@ -3,23 +3,14 @@ import { BinaryDataService } from 'n8n-core';
import type { IRun } from 'n8n-workflow';
import type { BinaryData } from 'n8n-core';
export function isMissingExecutionId(
fileId: string,
mode: BinaryData.NonDefaultMode,
uuidV4CharLength = 36,
) {
return mode === 'filesystem' ? uuidV4CharLength === fileId.length : fileId.includes('/temp/');
}
/**
* Whenever the execution ID is not available to the binary data service at the
* time of writing a binary data file, its name is missing the execution ID.
*
* This function restores the ID in the file name and run data reference.
*
* ```txt
* filesystem:11869055-83c4-4493-876a-9092c4708b9b ->
* filesystem:39011869055-83c4-4493-876a-9092c4708b9b
* filesystem-v2:workflows/123/executions/temp/binary_data/69055-83c4-4493-876a-9092c4708b9b ->
* filesystem-v2:workflows/123/executions/390/binary_data/69055-83c4-4493-876a-9092c4708b9b
*
* s3:workflows/123/executions/temp/binary_data/69055-83c4-4493-876a-9092c4708b9b ->
* s3:workflows/123/executions/390/binary_data/69055-83c4-4493-876a-9092c4708b9b
@@ -33,12 +24,13 @@ export async function restoreBinaryDataId(run: IRun, executionId: string) {
if (!binaryDataId) return;
const [mode, fileId] = binaryDataId.split(':') as [BinaryData.NonDefaultMode, string];
const [mode, fileId] = binaryDataId.split(':') as [BinaryData.StoredMode, string];
if (!isMissingExecutionId(fileId, mode)) return;
const isMissingExecutionId = fileId.includes('/temp/');
const correctFileId =
mode === 'filesystem' ? `${executionId}${fileId}` : fileId.replace('temp', executionId);
if (!isMissingExecutionId) return;
const correctFileId = fileId.replace('temp', executionId);
await Container.get(BinaryDataService).rename(fileId, correctFileId);

7
packages/cli/src/workflows/workflows.services.ts
View File

@@ -35,6 +35,7 @@ import { OwnershipService } from '@/services/ownership.service';
import { isStringArray, isWorkflowIdValid } from '@/utils';
import { isWorkflowHistoryLicensed } from './workflowHistory/workflowHistoryHelper.ee';
import { WorkflowHistoryService } from './workflowHistory/workflowHistory.service.ee';
import { BinaryDataService } from 'n8n-core';
export class WorkflowsService {
static async getSharing(
@@ -463,7 +464,13 @@ export class WorkflowsService {
await Container.get(ActiveWorkflowRunner).remove(workflowId);
}
const idsForDeletion = await Db.collections.Execution.find({
select: ['id'],
where: { workflowId },
}).then((rows) => rows.map(({ id: executionId }) => ({ workflowId, executionId })));
await Db.collections.Workflow.delete(workflowId);
await Container.get(BinaryDataService).deleteMany(idsForDeletion);
void Container.get(InternalHooks).onWorkflowDeleted(user, workflowId, false);
await Container.get(ExternalHooks).run('workflow.afterDelete', [workflowId]);

179
packages/cli/test/unit/execution.lifecycle.test.ts
View File

@@ -30,148 +30,77 @@ function getDataId(run: IRun, kind: 'binary' | 'json') {
const binaryDataService = mockInstance(BinaryDataService);
describe('on filesystem mode', () => {
describe('restoreBinaryDataId()', () => {
beforeAll(() => {
config.set('binaryDataManager.mode', 'filesystem');
});
afterEach(() => {
jest.clearAllMocks();
});
it('should restore if binary data ID is missing execution ID', async () => {
const executionId = '999';
const incorrectFileId = 'a5c3f1ed-9d59-4155-bc68-9a370b3c51f6';
const run = toIRun({
binary: {
data: { id: `filesystem:${incorrectFileId}` },
},
for (const mode of ['filesystem-v2', 's3'] as const) {
describe(`on ${mode} mode`, () => {
describe('restoreBinaryDataId()', () => {
beforeAll(() => {
config.set('binaryDataManager.mode', mode);
});
await restoreBinaryDataId(run, executionId);
afterEach(() => {
jest.clearAllMocks();
});
const correctFileId = `${executionId}${incorrectFileId}`;
const correctBinaryDataId = `filesystem:${correctFileId}`;
it('should restore if binary data ID is missing execution ID', async () => {
const workflowId = '6HYhhKmJch2cYxGj';
const executionId = 'temp';
const binaryDataFileUuid = 'a5c3f1ed-9d59-4155-bc68-9a370b3c51f6';
expect(binaryDataService.rename).toHaveBeenCalledWith(incorrectFileId, correctFileId);
expect(getDataId(run, 'binary')).toBe(correctBinaryDataId);
});
const incorrectFileId = `workflows/${workflowId}/executions/temp/binary_data/${binaryDataFileUuid}`;
it('should do nothing if binary data ID is not missing execution ID', async () => {
const executionId = '999';
const fileId = `${executionId}a5c3f1ed-9d59-4155-bc68-9a370b3c51f6`;
const binaryDataId = `filesystem:${fileId}`;
const run = toIRun({
binary: {
data: {
id: binaryDataId,
const run = toIRun({
binary: {
data: { id: `s3:${incorrectFileId}` },
},
},
});
await restoreBinaryDataId(run, executionId);
const correctFileId = incorrectFileId.replace('temp', executionId);
const correctBinaryDataId = `s3:${correctFileId}`;
expect(binaryDataService.rename).toHaveBeenCalledWith(incorrectFileId, correctFileId);
expect(getDataId(run, 'binary')).toBe(correctBinaryDataId);
});
await restoreBinaryDataId(run, executionId);
it('should do nothing if binary data ID is not missing execution ID', async () => {
const workflowId = '6HYhhKmJch2cYxGj';
const executionId = '999';
const binaryDataFileUuid = 'a5c3f1ed-9d59-4155-bc68-9a370b3c51f6';
expect(binaryDataService.rename).not.toHaveBeenCalled();
expect(getDataId(run, 'binary')).toBe(binaryDataId);
});
const fileId = `workflows/${workflowId}/executions/${executionId}/binary_data/${binaryDataFileUuid}`;
it('should do nothing if no binary data ID', async () => {
const executionId = '999';
const dataId = '123';
const run = toIRun({
json: {
data: { id: dataId },
},
});
const binaryDataId = `s3:${fileId}`;
await restoreBinaryDataId(run, executionId);
expect(binaryDataService.rename).not.toHaveBeenCalled();
expect(getDataId(run, 'json')).toBe(dataId);
});
});
});
describe('on s3 mode', () => {
describe('restoreBinaryDataId()', () => {
beforeAll(() => {
config.set('binaryDataManager.mode', 's3');
});
afterEach(() => {
jest.clearAllMocks();
});
it('should restore if binary data ID is missing execution ID', async () => {
const workflowId = '6HYhhKmJch2cYxGj';
const executionId = 'temp';
const binaryDataFileUuid = 'a5c3f1ed-9d59-4155-bc68-9a370b3c51f6';
const incorrectFileId = `workflows/${workflowId}/executions/temp/binary_data/${binaryDataFileUuid}`;
const run = toIRun({
binary: {
data: { id: `s3:${incorrectFileId}` },
},
});
await restoreBinaryDataId(run, executionId);
const correctFileId = incorrectFileId.replace('temp', executionId);
const correctBinaryDataId = `s3:${correctFileId}`;
expect(binaryDataService.rename).toHaveBeenCalledWith(incorrectFileId, correctFileId);
expect(getDataId(run, 'binary')).toBe(correctBinaryDataId);
});
it('should do nothing if binary data ID is not missing execution ID', async () => {
const workflowId = '6HYhhKmJch2cYxGj';
const executionId = '999';
const binaryDataFileUuid = 'a5c3f1ed-9d59-4155-bc68-9a370b3c51f6';
const fileId = `workflows/${workflowId}/executions/${executionId}/binary_data/${binaryDataFileUuid}`;
const binaryDataId = `s3:${fileId}`;
const run = toIRun({
binary: {
data: {
id: binaryDataId,
const run = toIRun({
binary: {
data: {
id: binaryDataId,
},
},
},
});
await restoreBinaryDataId(run, executionId);
expect(binaryDataService.rename).not.toHaveBeenCalled();
expect(getDataId(run, 'binary')).toBe(binaryDataId);
});
await restoreBinaryDataId(run, executionId);
it('should do nothing if no binary data ID', async () => {
const executionId = '999';
const dataId = '123';
expect(binaryDataService.rename).not.toHaveBeenCalled();
expect(getDataId(run, 'binary')).toBe(binaryDataId);
});
const run = toIRun({
json: {
data: { id: dataId },
},
});
it('should do nothing if no binary data ID', async () => {
const executionId = '999';
const dataId = '123';
await restoreBinaryDataId(run, executionId);
const run = toIRun({
json: {
data: { id: dataId },
},
expect(binaryDataService.rename).not.toHaveBeenCalled();
expect(getDataId(run, 'json')).toBe(dataId);
});
await restoreBinaryDataId(run, executionId);
expect(binaryDataService.rename).not.toHaveBeenCalled();
expect(getDataId(run, 'json')).toBe(dataId);
});
});
it('should do nothing on itemless case', async () => {
const executionId = '999';
const promise = restoreBinaryDataId(toIRun(), executionId);
await expect(promise).resolves.not.toThrow();
expect(binaryDataService.rename).not.toHaveBeenCalled();
});
});
}