admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 10:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Dedupe (#10101)

Browse Source 
Co-authored-by: Jan Oberhauser <jan@n8n.io>
Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
Co-authored-by: Tomi Turtiainen <10324676+tomi@users.noreply.github.com>
Co-authored-by: Elias Meire <elias@meire.dev>
This commit is contained in:
Shireen Missi
2024-10-10 16:12:05 +01:00
committed by GitHub
parent 6823e8f2dd
commit 52dd2c7619
32 changed files with 2298 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
packages/workflow/src/Interfaces.ts
View File

@@ -766,6 +766,48 @@ export interface BinaryHelperFunctions {
}>;
}
export type DeduplicationScope = 'node' | 'workflow';
export type DeduplicationItemTypes = string | number;
export type DeduplicationMode = 'entries' | 'latestIncrementalKey' | 'latestDate';
export interface IDeduplicationOutput {
new: DeduplicationItemTypes[];
processed: DeduplicationItemTypes[];
}
export interface IDeduplicationOutputItems {
new: IDataObject[];
processed: IDataObject[];
}
export interface ICheckProcessedOptions {
mode: DeduplicationMode;
maxEntries?: number;
}
export interface DeduplicationHelperFunctions {
checkProcessedAndRecord(
items: DeduplicationItemTypes[],
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<IDeduplicationOutput>;
checkProcessedItemsAndRecord(
propertyName: string,
items: IDataObject[],
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<IDeduplicationOutputItems>;
removeProcessed(
items: DeduplicationItemTypes[],
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<void>;
clearAllProcessedItems(scope: DeduplicationScope, options: ICheckProcessedOptions): Promise<void>;
getProcessedDataCount(
scope: DeduplicationScope,
options: ICheckProcessedOptions,
): Promise<number>;
}
export interface NodeHelperFunctions {
copyBinaryFile(filePath: string, fileName: string, mimeType?: string): Promise<IBinaryData>;
}
@@ -939,6 +981,7 @@ export type IExecuteFunctions = ExecuteFunctions.GetNodeParameterFn &
helpers: RequestHelperFunctions &
BaseHelperFunctions &
BinaryHelperFunctions &
DeduplicationHelperFunctions &
FileSystemHelperFunctions &
SSHTunnelFunctions &
JsonHelperFunctions & {
@@ -2645,6 +2688,46 @@ export interface IUserSettings {
npsSurvey?: NpsSurveyState;
}
export interface IProcessedDataConfig {
availableModes: string;
mode: string;
}
export interface IDataDeduplicator {
checkProcessedAndRecord(
items: DeduplicationItemTypes[],
context: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<IDeduplicationOutput>;
removeProcessed(
items: DeduplicationItemTypes[],
context: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<void>;
clearAllProcessedItems(
context: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<void>;
getProcessedDataCount(
context: DeduplicationScope,
contextData: ICheckProcessedContextData,
options: ICheckProcessedOptions,
): Promise<number>;
}
export interface ICheckProcessedContextData {
node?: INode;
workflow: {
id: string;
active: boolean;
};
}
export type ExpressionEvaluatorType = 'tmpl' | 'tournament';
export type N8nAIProviderType = 'openai' | 'unknown';