admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-19 11:01:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Dedupe (#10101)

Browse Source 
Co-authored-by: Jan Oberhauser <jan@n8n.io>
Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
Co-authored-by: Tomi Turtiainen <10324676+tomi@users.noreply.github.com>
Co-authored-by: Elias Meire <elias@meire.dev>
This commit is contained in:
Shireen Missi
2024-10-10 16:12:05 +01:00
committed by GitHub
parent 6823e8f2dd
commit 52dd2c7619
32 changed files with 2298 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
packages/core/src/NodeExecuteFunctions.ts
View File

@@ -102,6 +102,13 @@ import type {
EnsureTypeOptions,
SSHTunnelFunctions,
SchedulingFunctions,
DeduplicationHelperFunctions,
IDeduplicationOutput,
IDeduplicationOutputItems,
ICheckProcessedOptions,
DeduplicationScope,
DeduplicationItemTypes,
ICheckProcessedContextData,
AiEvent,
} from 'n8n-workflow';
import {
@@ -149,6 +156,7 @@ import {
UM_EMAIL_TEMPLATES_PWRESET,
} from './Constants';
import { createNodeAsTool } from './CreateNodeAsTool';
import { DataDeduplicationService } from './data-deduplication-service';
import {
getAllWorkflowExecutionMetadata,
getWorkflowExecutionMetadata,
@@ -1284,6 +1292,72 @@ async function prepareBinaryData(
return await setBinaryDataBuffer(returnData, binaryData, workflowId, executionId);
}
export async function checkProcessedAndRecord(
items: DeduplicationItemTypes[],
scope: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<IDeduplicationOutput> {
return await DataDeduplicationService.getInstance().checkProcessedAndRecord(
items,
scope,
contextData,
options,
);
}
export async function checkProcessedItemsAndRecord(
key: string,
items: IDataObject[],
scope: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<IDeduplicationOutputItems> {
return await DataDeduplicationService.getInstance().checkProcessedItemsAndRecord(
key,
items,
scope,
contextData,
options,
);
}
export async function removeProcessed(
items: DeduplicationItemTypes[],
scope: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<void> {
return await DataDeduplicationService.getInstance().removeProcessed(
items,
scope,
contextData,
options,
);
}
export async function clearAllProcessedItems(
scope: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<void> {
return await DataDeduplicationService.getInstance().clearAllProcessedItems(
scope,
contextData,
options,
);
}
export async function getProcessedDataCount(
scope: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<number> {
return await DataDeduplicationService.getInstance().getProcessedDataCount(
scope,
contextData,
options,
);
}
function applyPaginationRequestData(
requestData: IRequestOptions,
paginationRequestData: PaginationOptions['request'],
@@ -3453,6 +3527,52 @@ const getBinaryHelperFunctions = (
},
});
const getCheckProcessedHelperFunctions = (
workflow: Workflow,
node: INode,
): DeduplicationHelperFunctions => ({
async checkProcessedAndRecord(
items: DeduplicationItemTypes[],
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<IDeduplicationOutput> {
return await checkProcessedAndRecord(items, scope, { node, workflow }, options);
},
async checkProcessedItemsAndRecord(
propertyName: string,
items: IDataObject[],
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<IDeduplicationOutputItems> {
return await checkProcessedItemsAndRecord(
propertyName,
items,
scope,
{ node, workflow },
options,
);
},
async removeProcessed(
items: DeduplicationItemTypes[],
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<void> {
return await removeProcessed(items, scope, { node, workflow }, options);
},
async clearAllProcessedItems(
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<void> {
return await clearAllProcessedItems(scope, { node, workflow }, options);
},
async getProcessedDataCount(
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<number> {
return await getProcessedDataCount(scope, { node, workflow }, options);
},
});
/**
* Returns a copy of the items which only contains the json data and
* of that only the defined properties
@@ -3896,6 +4016,7 @@ export function getExecuteFunctions(
...getSSHTunnelFunctions(),
...getFileSystemHelperFunctions(node),
...getBinaryHelperFunctions(additionalData, workflow.id),
...getCheckProcessedHelperFunctions(workflow, node),
assertBinaryData: (itemIndex, propertyName) =>
assertBinaryData(inputData, node, itemIndex, propertyName, 0),
getBinaryDataBuffer: async (itemIndex, propertyName) =>