admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-22 12:19:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): Reduce memory consumption on BinaryDataManager.init (#6633)

Browse Source 
fix(core): Reduce memory consumption on BinaryDataManager.init

When there are a few thousand binary data file to delete, the `deleteMarkedFiles` and `deleteMarkedPersistedFiles` methods need a lot of memory to process these files, irrespective of if these files have any data or not.
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™
2023-07-12 10:08:29 +02:00
committed by GitHub
parent 180ab8d7c2
commit 329d22f5d1
5 changed files with 72 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/cli/src/PublicApi/v1/handlers/executions/executions.handler.ts
View File

@@ -37,7 +37,7 @@ export = {
return res.status(404).json({ message: 'Not Found' }); return res.status(404).json({ message: 'Not Found' });
} }
await BinaryDataManager.getInstance().deleteBinaryDataByExecutionId(execution.id!); await BinaryDataManager.getInstance().deleteBinaryDataByExecutionIds([execution.id!]);
await deleteExecution(execution); await deleteExecution(execution);

13
packages/cli/src/databases/repositories/execution.repository.ts
View File

@@ -240,7 +240,7 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
async deleteExecution(executionId: string) { async deleteExecution(executionId: string) {
// TODO: Should this be awaited? Should we add a catch in case it fails? // TODO: Should this be awaited? Should we add a catch in case it fails?
await BinaryDataManager.getInstance().deleteBinaryDataByExecutionId(executionId); await BinaryDataManager.getInstance().deleteBinaryDataByExecutionIds([executionId]);
return this.delete({ id: executionId }); return this.delete({ id: executionId });
} }
@@ -392,17 +392,14 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
return; return;
} }
const idsToDelete = executions.map(({ id }) => id); const executionIds = executions.map(({ id }) => id);
const binaryDataManager = BinaryDataManager.getInstance(); const binaryDataManager = BinaryDataManager.getInstance();
await Promise.all( await binaryDataManager.deleteBinaryDataByExecutionIds(executionIds);
idsToDelete.map(async (id) => binaryDataManager.deleteBinaryDataByExecutionId(id)),
);
do { do {
// Delete in batches to avoid "SQLITE_ERROR: Expression tree is too large (maximum depth 1000)" error // Delete in batches to avoid "SQLITE_ERROR: Expression tree is too large (maximum depth 1000)" error
const batch = idsToDelete.splice(0, 500); const batch = executionIds.splice(0, 500);
await this.delete(batch); await this.delete(batch);
} while (idsToDelete.length > 0); } while (executionIds.length > 0);
} }
} }

112
packages/core/src/BinaryDataManager/FileSystem.ts
View File

@@ -1,3 +1,4 @@
import glob from 'fast-glob';
import { createReadStream } from 'fs'; import { createReadStream } from 'fs';
import fs from 'fs/promises'; import fs from 'fs/promises';
import path from 'path'; import path from 'path';
@@ -12,6 +13,9 @@ import { FileNotFoundError } from '../errors';
const PREFIX_METAFILE = 'binarymeta'; const PREFIX_METAFILE = 'binarymeta';
const PREFIX_PERSISTED_METAFILE = 'persistedmeta'; const PREFIX_PERSISTED_METAFILE = 'persistedmeta';
const executionExtractionRegexp =
/^(\w+)(?:[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})$/;
export class BinaryDataFileSystem implements IBinaryDataManager { export class BinaryDataFileSystem implements IBinaryDataManager {
private storagePath: string; private storagePath: string;
@@ -36,16 +40,12 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
}, this.persistedBinaryDataTTL * 30000); }, this.persistedBinaryDataTTL * 30000);
} }
return fs await this.assertFolder(this.storagePath);
.readdir(this.storagePath) await this.assertFolder(this.getBinaryDataMetaPath());
.catch(async () => fs.mkdir(this.storagePath, { recursive: true })) await this.assertFolder(this.getBinaryDataPersistMetaPath());
.then(async () => fs.readdir(this.getBinaryDataMetaPath()))
.catch(async () => fs.mkdir(this.getBinaryDataMetaPath(), { recursive: true })) await this.deleteMarkedFiles();
.then(async () => fs.readdir(this.getBinaryDataPersistMetaPath())) await this.deleteMarkedPersistedFiles();
.catch(async () => fs.mkdir(this.getBinaryDataPersistMetaPath(), { recursive: true }))
.then(async () => this.deleteMarkedFiles())
.then(async () => this.deleteMarkedPersistedFiles())
.then(() => {});
} }
async getFileSize(identifier: string): Promise<number> { async getFileSize(identifier: string): Promise<number> {
@@ -122,46 +122,37 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
`${PREFIX_PERSISTED_METAFILE}_${executionId}_${timeoutTime}`, `${PREFIX_PERSISTED_METAFILE}_${executionId}_${timeoutTime}`,
); );
return fs try {
.readFile(filePath) await fs.access(filePath);
.catch(async () => fs.writeFile(filePath, identifier)) } catch {
.then(() => {}); await fs.writeFile(filePath, identifier);
}
} }
private async deleteMarkedFilesByMeta(metaPath: string, filePrefix: string): Promise<void> { private async deleteMarkedFilesByMeta(metaPath: string, filePrefix: string): Promise<void> {
const currentTimeValue = new Date().valueOf(); const currentTimeValue = new Date().valueOf();
const metaFileNames = await fs.readdir(metaPath); const metaFileNames = await glob(`${filePrefix}_*`, { cwd: metaPath });
const execsAdded: { [key: string]: number } = {};
const promises = metaFileNames.reduce<Array<Promise<void>>>((prev, curr) => {
const [prefix, executionId, ts] = curr.split('_');
if (prefix !== filePrefix) {
return prev;
}
const executionIds = metaFileNames
.map((f) => f.split('_') as [string, string, string])
.filter(([prefix, , ts]) => {
if (prefix !== filePrefix) return false;
const execTimestamp = parseInt(ts, 10); const execTimestamp = parseInt(ts, 10);
return execTimestamp < currentTimeValue;
})
.map((e) => e[1]);
if (execTimestamp < currentTimeValue) { const filesToDelete = [];
if (execsAdded[executionId]) { const deletedIds = await this.deleteBinaryDataByExecutionIds(executionIds);
// do not delete data, only meta file for (const executionId of deletedIds) {
prev.push(this.deleteMetaFileByPath(path.join(metaPath, curr))); filesToDelete.push(
return prev; ...(await glob(`${filePrefix}_${executionId}_`, {
} absolute: true,
cwd: metaPath,
execsAdded[executionId] = 1; })),
prev.push(
this.deleteBinaryDataByExecutionId(executionId).then(async () =>
this.deleteMetaFileByPath(path.join(metaPath, curr)),
),
); );
} }
await Promise.all(filesToDelete.map(async (file) => fs.rm(file)));
return prev;
}, []);
await Promise.all(promises);
} }
async duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise<string> { async duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise<string> {
@@ -174,18 +165,19 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
return newBinaryDataId; return newBinaryDataId;
} }
async deleteBinaryDataByExecutionId(executionId: string): Promise<void> { async deleteBinaryDataByExecutionIds(executionIds: string[]): Promise<string[]> {
const regex = new RegExp(`${executionId}_*`); const set = new Set(executionIds);
const filenames = await fs.readdir(this.storagePath); const fileNames = await fs.readdir(this.storagePath);
const deletedIds = [];
const promises = filenames.reduce<Array<Promise<void>>>((allProms, filename) => { for (const fileName of fileNames) {
if (regex.test(filename)) { const executionId = fileName.match(executionExtractionRegexp)?.[1];
allProms.push(fs.rm(this.resolveStoragePath(filename))); if (executionId && set.has(executionId)) {
const filePath = this.resolveStoragePath(fileName);
await Promise.all([fs.rm(filePath), fs.rm(`${filePath}.metadata`)]);
deletedIds.push(executionId);
} }
return allProms; }
}, []); return deletedIds;
await Promise.all(promises);
} }
async deleteBinaryDataByIdentifier(identifier: string): Promise<void> { async deleteBinaryDataByIdentifier(identifier: string): Promise<void> {
@@ -193,7 +185,7 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
} }
async persistBinaryDataForExecutionId(executionId: string): Promise<void> { async persistBinaryDataForExecutionId(executionId: string): Promise<void> {
return fs.readdir(this.getBinaryDataPersistMetaPath()).then(async (metaFiles) => { const metaFiles = await fs.readdir(this.getBinaryDataPersistMetaPath());
const promises = metaFiles.reduce<Array<Promise<void>>>((prev, curr) => { const promises = metaFiles.reduce<Array<Promise<void>>>((prev, curr) => {
if (curr.startsWith(`${PREFIX_PERSISTED_METAFILE}_${executionId}_`)) { if (curr.startsWith(`${PREFIX_PERSISTED_METAFILE}_${executionId}_`)) {
prev.push(fs.rm(path.join(this.getBinaryDataPersistMetaPath(), curr))); prev.push(fs.rm(path.join(this.getBinaryDataPersistMetaPath(), curr)));
@@ -202,9 +194,15 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
return prev; return prev;
}, []); }, []);
await Promise.all(promises); await Promise.all(promises);
}); }
private async assertFolder(folder: string): Promise<void> {
try {
await fs.access(folder);
} catch {
await fs.mkdir(folder, { recursive: true });
}
} }
private generateFileName(prefix: string): string { private generateFileName(prefix: string): string {
@@ -219,10 +217,6 @@ export class BinaryDataFileSystem implements IBinaryDataManager {
return path.join(this.storagePath, 'persistMeta'); return path.join(this.storagePath, 'persistMeta');
} }
private async deleteMetaFileByPath(metaFilePath: string): Promise<void> {
return fs.rm(metaFilePath);
}
private async deleteFromLocalStorage(identifier: string) { private async deleteFromLocalStorage(identifier: string) {
return fs.rm(this.getBinaryPath(identifier)); return fs.rm(this.getBinaryPath(identifier));
} }

4
packages/core/src/BinaryDataManager/index.ts
View File

@@ -178,9 +178,9 @@ export class BinaryDataManager {
} }
} }
async deleteBinaryDataByExecutionId(executionId: string): Promise<void> { async deleteBinaryDataByExecutionIds(executionIds: string[]): Promise<void> {
if (this.managers[this.binaryDataMode]) { if (this.managers[this.binaryDataMode]) {
await this.managers[this.binaryDataMode].deleteBinaryDataByExecutionId(executionId); await this.managers[this.binaryDataMode].deleteBinaryDataByExecutionIds(executionIds);
} }
} }

2
packages/core/src/Interfaces.ts
View File

@@ -72,7 +72,7 @@ export interface IBinaryDataManager {
deleteMarkedFiles(): Promise<unknown>; deleteMarkedFiles(): Promise<unknown>;
deleteBinaryDataByIdentifier(identifier: string): Promise<void>; deleteBinaryDataByIdentifier(identifier: string): Promise<void>;
duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise<string>; duplicateBinaryDataByIdentifier(binaryDataId: string, prefix: string): Promise<string>;
deleteBinaryDataByExecutionId(executionId: string): Promise<void>; deleteBinaryDataByExecutionIds(executionIds: string[]): Promise<string[]>;
persistBinaryDataForExecutionId(executionId: string): Promise<void>; persistBinaryDataForExecutionId(executionId: string): Promise<void>;
} }