feat(core): Dedupe (#10101)

Co-authored-by: Jan Oberhauser <jan@n8n.io>
Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
Co-authored-by: Tomi Turtiainen <10324676+tomi@users.noreply.github.com>
Co-authored-by: Elias Meire <elias@meire.dev>

packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts

@@ -1,5 +1,11 @@
+import { isEqual, lt, pick } from 'lodash';
 import get from 'lodash/get';
-import { NodeOperationError, type INode, type INodeExecutionData } from 'n8n-workflow';
+import { NodeOperationError } from 'n8n-workflow';
+import type { IExecuteFunctions, INode, INodeExecutionData } from 'n8n-workflow';
+
+import { compareItems, flattenKeys } from '@utils/utilities';
+
+import { prepareFieldsArray } from '../utils/utils';
 
 export const validateInputData = (
 	node: INode,
@@ -39,3 +45,124 @@ export const validateInputData = (
 		}
 	}
 };
+
+export function removeDuplicateInputItems(context: IExecuteFunctions, items: INodeExecutionData[]) {
+	const compare = context.getNodeParameter('compare', 0) as string;
+	const disableDotNotation = context.getNodeParameter(
+		'options.disableDotNotation',
+		0,
+		false,
+	) as boolean;
+	const removeOtherFields = context.getNodeParameter(
+		'options.removeOtherFields',
+		0,
+		false,
+	) as boolean;
+
+	let keys = disableDotNotation
+		? Object.keys(items[0].json)
+		: Object.keys(flattenKeys(items[0].json));
+
+	for (const item of items) {
+		const itemKeys = disableDotNotation
+			? Object.keys(item.json)
+			: Object.keys(flattenKeys(item.json));
+		for (const key of itemKeys) {
+			if (!keys.includes(key)) {
+				keys.push(key);
+			}
+		}
+	}
+
+	if (compare === 'allFieldsExcept') {
+		const fieldsToExclude = prepareFieldsArray(
+			context.getNodeParameter('fieldsToExclude', 0, '') as string,
+			'Fields To Exclude',
+		);
+
+		if (!fieldsToExclude.length) {
+			throw new NodeOperationError(
+				context.getNode(),
+				'No fields specified. Please add a field to exclude from comparison',
+			);
+		}
+		if (!disableDotNotation) {
+			keys = Object.keys(flattenKeys(items[0].json));
+		}
+		keys = keys.filter((key) => !fieldsToExclude.includes(key));
+	}
+	if (compare === 'selectedFields') {
+		const fieldsToCompare = prepareFieldsArray(
+			context.getNodeParameter('fieldsToCompare', 0, '') as string,
+			'Fields To Compare',
+		);
+		if (!fieldsToCompare.length) {
+			throw new NodeOperationError(
+				context.getNode(),
+				'No fields specified. Please add a field to compare on',
+			);
+		}
+		if (!disableDotNotation) {
+			keys = Object.keys(flattenKeys(items[0].json));
+		}
+		keys = fieldsToCompare.map((key) => key.trim());
+	}
+
+	// This solution is O(nlogn)
+	// add original index to the items
+	const newItems = items.map(
+		(item, index) =>
+			({
+				json: { ...item.json, __INDEX: index },
+				pairedItem: { item: index },
+			}) as INodeExecutionData,
+	);
+	//sort items using the compare keys
+	newItems.sort((a, b) => {
+		let result = 0;
+
+		for (const key of keys) {
+			let equal;
+			if (!disableDotNotation) {
+				equal = isEqual(get(a.json, key), get(b.json, key));
+			} else {
+				equal = isEqual(a.json[key], b.json[key]);
+			}
+			if (!equal) {
+				let lessThan;
+				if (!disableDotNotation) {
+					lessThan = lt(get(a.json, key), get(b.json, key));
+				} else {
+					lessThan = lt(a.json[key], b.json[key]);
+				}
+				result = lessThan ? -1 : 1;
+				break;
+			}
+		}
+		return result;
+	});
+
+	validateInputData(context.getNode(), newItems, keys, disableDotNotation);
+
+	// collect the original indexes of items to be removed
+	const removedIndexes: number[] = [];
+	let temp = newItems[0];
+	for (let index = 1; index < newItems.length; index++) {
+		if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
+			removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
+		} else {
+			temp = newItems[index];
+		}
+	}
+	let updatedItems: INodeExecutionData[] = items.filter(
+		(_, index) => !removedIndexes.includes(index),
+	);
+
+	if (removeOtherFields) {
+		updatedItems = updatedItems.map((item, index) => ({
+			json: pick(item.json, ...keys),
+			pairedItem: { item: index },
+		}));
+	}
+	return [updatedItems];
+}