diff --git a/packages/nodes-base/jest.config.js b/packages/nodes-base/jest.config.js index 61e5f7302d..860b43c630 100644 --- a/packages/nodes-base/jest.config.js +++ b/packages/nodes-base/jest.config.js @@ -2,5 +2,9 @@ module.exports = { ...require('../../jest.config'), collectCoverageFrom: ['credentials/**/*.ts', 'nodes/**/*.ts', 'utils/**/*.ts'], - setupFilesAfterEnv: ['jest-expect-message', '/test/setup.ts'], + setupFilesAfterEnv: [ + 'jest-expect-message', + 'n8n-workflow/test/setup.ts', + '/test/setup.ts', + ], }; diff --git a/packages/nodes-base/nodes/Html/Html.node.ts b/packages/nodes-base/nodes/Html/Html.node.ts index 4b2a7d216b..260f7ffaf1 100644 --- a/packages/nodes-base/nodes/Html/Html.node.ts +++ b/packages/nodes-base/nodes/Html/Html.node.ts @@ -12,7 +12,7 @@ import get from 'lodash/get'; import { placeholder } from './placeholder'; import { getValue } from './utils'; import type { IValueData } from './types'; -import { getResolvables, sanitazeDataPathKey } from '@utils/utilities'; +import { getResolvables, sanitizeDataPathKey } from '@utils/utilities'; export const capitalizeHeader = (header: string, capitalize?: boolean) => { if (!capitalize) return header; @@ -516,7 +516,7 @@ export class Html implements INodeType { let htmlArray: string[] | string = []; if (sourceData === 'json') { if (nodeVersion === 1) { - const key = sanitazeDataPathKey(item.json, dataPropertyName); + const key = sanitizeDataPathKey(item.json, dataPropertyName); if (item.json[key] === undefined) { throw new NodeOperationError( this.getNode(), diff --git a/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts b/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts index c2f17001d9..85663ae6a7 100644 --- a/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts +++ b/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts @@ -1,62 +1,22 @@ import type { IDataObject, IExecuteFunctions, - INode, INodeExecutionData, INodeType, INodeTypeBaseDescription, INodeTypeDescription, } from 'n8n-workflow'; -import { NodeOperationError, randomInt } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; import isEqual from 'lodash/isEqual'; -import isObject from 'lodash/isObject'; import lt from 'lodash/lt'; -import merge from 'lodash/merge'; import pick from 'lodash/pick'; -import reduce from 'lodash/reduce'; import set from 'lodash/set'; import unset from 'lodash/unset'; -const compareItems = ( - obj: INodeExecutionData, - obj2: INodeExecutionData, - keys: string[], - disableDotNotation: boolean, - _node: INode, -) => { - let result = true; - for (const key of keys) { - if (!disableDotNotation) { - if (!isEqual(get(obj.json, key), get(obj2.json, key))) { - result = false; - break; - } - } else { - if (!isEqual(obj.json[key], obj2.json[key])) { - result = false; - break; - } - } - } - return result; -}; - -const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { - return !isObject(obj) - ? { [path.join('.')]: obj } - : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore -}; - -const shuffleArray = (array: any[]) => { - for (let i = array.length - 1; i > 0; i--) { - const j = randomInt(i + 1); - [array[i], array[j]] = [array[j], array[i]]; - } -}; - +import { flattenKeys, shuffleArray, compareItems } from '@utils/utilities'; import { sortByCode } from '../V3/helpers/utils'; import * as summarize from './summarize.operation'; @@ -1226,7 +1186,7 @@ return 0;`, const removedIndexes: number[] = []; let temp = newItems[0]; for (let index = 1; index < newItems.length; index++) { - if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + if (compareItems(newItems[index], temp, keys, disableDotNotation)) { removedIndexes.push(newItems[index].json.__INDEX as unknown as number); } else { temp = newItems[index]; diff --git a/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts b/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts index 5a41622055..78511c058c 100644 --- a/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts +++ b/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts @@ -1,63 +1,23 @@ import type { IDataObject, IExecuteFunctions, - INode, INodeExecutionData, INodeType, INodeTypeBaseDescription, INodeTypeDescription, IPairedItemData, } from 'n8n-workflow'; -import { NodeOperationError, deepCopy, randomInt } from 'n8n-workflow'; +import { NodeOperationError, deepCopy } from 'n8n-workflow'; import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; import isEqual from 'lodash/isEqual'; -import isObject from 'lodash/isObject'; import lt from 'lodash/lt'; -import merge from 'lodash/merge'; import pick from 'lodash/pick'; -import reduce from 'lodash/reduce'; import set from 'lodash/set'; import unset from 'lodash/unset'; -const compareItems = ( - obj: INodeExecutionData, - obj2: INodeExecutionData, - keys: string[], - disableDotNotation: boolean, - _node: INode, -) => { - let result = true; - for (const key of keys) { - if (!disableDotNotation) { - if (!isEqual(get(obj.json, key), get(obj2.json, key))) { - result = false; - break; - } - } else { - if (!isEqual(obj.json[key], obj2.json[key])) { - result = false; - break; - } - } - } - return result; -}; - -const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { - return !isObject(obj) - ? { [path.join('.')]: obj } - : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore -}; - -const shuffleArray = (array: any[]) => { - for (let i = array.length - 1; i > 0; i--) { - const j = randomInt(i + 1); - [array[i], array[j]] = [array[j], array[i]]; - } -}; - +import { flattenKeys, shuffleArray, compareItems } from '@utils/utilities'; import { sortByCode } from '../V3/helpers/utils'; import * as summarize from './summarize.operation'; @@ -1273,7 +1233,7 @@ return 0;`, const removedIndexes: number[] = []; let temp = newItems[0]; for (let index = 1; index < newItems.length; index++) { - if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + if (compareItems(newItems[index], temp, keys, disableDotNotation)) { removedIndexes.push(newItems[index].json.__INDEX as unknown as number); } else { temp = newItems[index]; diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts index 8b8c7e6948..8536e742e2 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts @@ -6,9 +6,9 @@ import isEqual from 'lodash/isEqual'; import lt from 'lodash/lt'; import pick from 'lodash/pick'; -import { compareItems, flattenKeys, prepareFieldsArray, typeToNumber } from '../../helpers/utils'; +import { compareItems, flattenKeys, updateDisplayOptions } from '@utils/utilities'; +import { prepareFieldsArray, typeToNumber } from '../../helpers/utils'; import { disableDotNotationBoolean } from '../common.descriptions'; -import { updateDisplayOptions } from '@utils/utilities'; const properties: INodeProperties[] = [ { @@ -229,7 +229,7 @@ export async function execute( const removedIndexes: number[] = []; let temp = newItems[0]; for (let index = 1; index < newItems.length; index++) { - if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + if (compareItems(newItems[index], temp, keys, disableDotNotation)) { removedIndexes.push(newItems[index].json.__INDEX as unknown as number); } else { temp = newItems[index]; diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts index efc979917d..0bd3ec9ec9 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts @@ -11,9 +11,9 @@ import get from 'lodash/get'; import isEqual from 'lodash/isEqual'; import lt from 'lodash/lt'; -import { shuffleArray, sortByCode } from '../../helpers/utils'; +import { sortByCode } from '../../helpers/utils'; import { disableDotNotationBoolean } from '../common.descriptions'; -import { updateDisplayOptions } from '@utils/utilities'; +import { shuffleArray, updateDisplayOptions } from '@utils/utilities'; const properties: INodeProperties[] = [ { diff --git a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts index e98bd773ae..8500322a7c 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts @@ -1,56 +1,11 @@ import { NodeVM } from '@n8n/vm2'; import type { - IDataObject, IExecuteFunctions, IBinaryData, - INode, INodeExecutionData, GenericValue, } from 'n8n-workflow'; -import { ApplicationError, NodeOperationError, randomInt } from 'n8n-workflow'; - -import get from 'lodash/get'; -import isEqual from 'lodash/isEqual'; -import isObject from 'lodash/isObject'; -import merge from 'lodash/merge'; -import reduce from 'lodash/reduce'; - -export const compareItems = ( - obj: INodeExecutionData, - obj2: INodeExecutionData, - keys: string[], - disableDotNotation: boolean, - _node: INode, -) => { - let result = true; - for (const key of keys) { - if (!disableDotNotation) { - if (!isEqual(get(obj.json, key), get(obj2.json, key))) { - result = false; - break; - } - } else { - if (!isEqual(obj.json[key], obj2.json[key])) { - result = false; - break; - } - } - } - return result; -}; - -export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { - return !isObject(obj) - ? { [path.join('.')]: obj } - : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore -}; - -export const shuffleArray = (array: any[]) => { - for (let i = array.length - 1; i > 0; i--) { - const j = randomInt(i + 1); - [array[i], array[j]] = [array[j], array[i]]; - } -}; +import { ApplicationError, NodeOperationError } from 'n8n-workflow'; export const prepareFieldsArray = (fields: string | string[], fieldName = 'Fields') => { if (typeof fields === 'string') { diff --git a/packages/nodes-base/nodes/Set/v2/helpers/utils.ts b/packages/nodes-base/nodes/Set/v2/helpers/utils.ts index a0f0eddb38..46536f312b 100644 --- a/packages/nodes-base/nodes/Set/v2/helpers/utils.ts +++ b/packages/nodes-base/nodes/Set/v2/helpers/utils.ts @@ -18,7 +18,7 @@ import get from 'lodash/get'; import set from 'lodash/set'; import unset from 'lodash/unset'; -import { getResolvables, sanitazeDataPathKey } from '../../../../utils/utilities'; +import { getResolvables, sanitizeDataPathKey } from '../../../../utils/utilities'; import type { SetNodeOptions } from './interfaces'; import { INCLUDE } from './interfaces'; @@ -38,13 +38,13 @@ const configureFieldHelper = (dotNotation?: boolean) => { } else { return { set: (item: IDataObject, key: string, value: IDataObject) => { - item[sanitazeDataPathKey(item, key)] = value; + item[sanitizeDataPathKey(item, key)] = value; }, get: (item: IDataObject, key: string) => { - return item[sanitazeDataPathKey(item, key)]; + return item[sanitizeDataPathKey(item, key)]; }, unset: (item: IDataObject, key: string) => { - delete item[sanitazeDataPathKey(item, key)]; + delete item[sanitizeDataPathKey(item, key)]; }, }; } diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts index 26f37210dc..33260922d7 100644 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts @@ -10,7 +10,8 @@ import { type INodeTypeDescription, } from 'n8n-workflow'; import { prepareFieldsArray } from '../utils/utils'; -import { compareItems, flattenKeys, validateInputData } from './utils'; +import { validateInputData } from './utils'; +import { compareItems, flattenKeys } from '@utils/utilities'; export class RemoveDuplicates implements INodeType { description: INodeTypeDescription = { @@ -211,7 +212,7 @@ export class RemoveDuplicates implements INodeType { const removedIndexes: number[] = []; let temp = newItems[0]; for (let index = 1; index < newItems.length; index++) { - if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + if (compareItems(newItems[index], temp, keys, disableDotNotation)) { removedIndexes.push(newItems[index].json.__INDEX as unknown as number); } else { temp = newItems[index]; diff --git a/packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts b/packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts index 9370a60178..532ff4ec3a 100644 --- a/packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts +++ b/packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts @@ -1,44 +1,5 @@ import get from 'lodash/get'; -import isEqual from 'lodash/isEqual'; -import isObject from 'lodash/isObject'; -import merge from 'lodash/merge'; -import reduce from 'lodash/reduce'; -import { - NodeOperationError, - type IDataObject, - type INode, - type INodeExecutionData, -} from 'n8n-workflow'; - -export const compareItems = ( - obj: INodeExecutionData, - obj2: INodeExecutionData, - keys: string[], - disableDotNotation: boolean, - _node: INode, -) => { - let result = true; - for (const key of keys) { - if (!disableDotNotation) { - if (!isEqual(get(obj.json, key), get(obj2.json, key))) { - result = false; - break; - } - } else { - if (!isEqual(obj.json[key], obj2.json[key])) { - result = false; - break; - } - } - } - return result; -}; - -export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { - return !isObject(obj) - ? { [path.join('.')]: obj } - : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore -}; +import { NodeOperationError, type INode, type INodeExecutionData } from 'n8n-workflow'; export const validateInputData = ( node: INode, diff --git a/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts b/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts index 52ac687132..43b5fa85c0 100644 --- a/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts +++ b/packages/nodes-base/nodes/Transform/Sort/Sort.node.ts @@ -9,7 +9,8 @@ import { type INodeType, type INodeTypeDescription, } from 'n8n-workflow'; -import { shuffleArray, sortByCode } from './utils'; +import { sortByCode } from './utils'; +import { shuffleArray } from '@utils/utilities'; export class Sort implements INodeType { description: INodeTypeDescription = { diff --git a/packages/nodes-base/nodes/Transform/Sort/utils.ts b/packages/nodes-base/nodes/Transform/Sort/utils.ts index 00b7b6798b..4e23ae8ab0 100644 --- a/packages/nodes-base/nodes/Transform/Sort/utils.ts +++ b/packages/nodes-base/nodes/Transform/Sort/utils.ts @@ -1,13 +1,6 @@ import { NodeVM } from '@n8n/vm2'; import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; -import { NodeOperationError, randomInt } from 'n8n-workflow'; - -export const shuffleArray = (array: any[]) => { - for (let i = array.length - 1; i > 0; i--) { - const j = randomInt(i + 1); - [array[i], array[j]] = [array[j], array[i]]; - } -}; +import { NodeOperationError } from 'n8n-workflow'; const returnRegExp = /\breturn\b/g; export function sortByCode( diff --git a/packages/nodes-base/test/utils/utilities.test.ts b/packages/nodes-base/test/utils/utilities.test.ts index 88edcedc27..ff7ee8f4ed 100644 --- a/packages/nodes-base/test/utils/utilities.test.ts +++ b/packages/nodes-base/test/utils/utilities.test.ts @@ -1,4 +1,12 @@ -import { fuzzyCompare, getResolvables, keysToLowercase, wrapData } from '@utils/utilities'; +import { + compareItems, + flattenKeys, + fuzzyCompare, + getResolvables, + keysToLowercase, + shuffleArray, + wrapData, +} from '@utils/utilities'; //most test cases for fuzzyCompare are done in Compare Datasets node tests describe('Test fuzzyCompare', () => { @@ -137,3 +145,110 @@ describe('Test getResolvables', () => { ]); }); }); + +describe('shuffleArray', () => { + it('should shuffle array', () => { + const array = [1, 2, 3, 4, 5]; + const toShuffle = [...array]; + shuffleArray(toShuffle); + expect(toShuffle).not.toEqual(array); + expect(toShuffle).toHaveLength(array.length); + expect(toShuffle).toEqual(expect.arrayContaining(array)); + }); +}); + +describe('flattenKeys', () => { + const name = 'Lisa'; + const city1 = 'Berlin'; + const city2 = 'Schoenwald'; + const withNestedObject = { + name, + address: { city: city1 }, + }; + + const withNestedArrays = { + name, + addresses: [{ city: city1 }, { city: city2 }], + }; + + it('should handle empty object', () => { + const flattenedObj = flattenKeys({}); + expect(flattenedObj).toEqual({}); + }); + + it('should flatten object with nested object', () => { + const flattenedObj = flattenKeys(withNestedObject); + expect(flattenedObj).toEqual({ + name, + 'address.city': city1, + }); + }); + + it('should handle object with nested arrays', () => { + const flattenedObj = flattenKeys(withNestedArrays); + expect(flattenedObj).toEqual({ + name, + 'addresses.0.city': city1, + 'addresses.1.city': city2, + }); + }); + + it('should flatten object with nested object and specified prefix', () => { + const flattenedObj = flattenKeys(withNestedObject, ['test']); + expect(flattenedObj).toEqual({ + 'test.name': name, + 'test.address.city': city1, + }); + }); + + it('should handle object with nested arrays and specified prefix', () => { + const flattenedObj = flattenKeys(withNestedArrays, ['test']); + expect(flattenedObj).toEqual({ + 'test.name': name, + 'test.addresses.0.city': city1, + 'test.addresses.1.city': city2, + }); + }); +}); + +describe('compareItems', () => { + it('should return true if all values of specified keys are equal', () => { + const obj1 = { json: { a: 1, b: 2, c: 3 } }; + const obj2 = { json: { a: 1, b: 2, c: 3 } }; + const keys = ['a', 'b', 'c']; + const result = compareItems(obj1, obj2, keys); + expect(result).toBe(true); + }); + + it('should return false if any values of specified keys are not equal', () => { + const obj1 = { json: { a: 1, b: 2, c: 3 } }; + const obj2 = { json: { a: 1, b: 2, c: 4 } }; + const keys = ['a', 'b', 'c']; + const result = compareItems(obj1, obj2, keys); + expect(result).toBe(false); + }); + + it('should return true if all values of specified keys are equal using dot notation', () => { + const obj1 = { json: { a: { b: { c: 1 } } } }; + const obj2 = { json: { a: { b: { c: 1 } } } }; + const keys = ['a.b.c']; + const result = compareItems(obj1, obj2, keys); + expect(result).toBe(true); + }); + + it('should return false if any values of specified keys are not equal using dot notation', () => { + const obj1 = { json: { a: { b: { c: 1 } } } }; + const obj2 = { json: { a: { b: { c: 2 } } } }; + const keys = ['a.b.c']; + const result = compareItems(obj1, obj2, keys); + expect(result).toBe(false); + }); + + it('should return true if all values of specified keys are equal using bracket notation', () => { + const obj1 = { json: { 'a.b': { 'c.d': 1 } } }; + const obj2 = { json: { 'a.b': { 'c.d': 1 } } }; + const keys = ['a.b.c.d']; + const result = compareItems(obj1, obj2, keys, true); + expect(result).toBe(true); + }); +}); diff --git a/packages/nodes-base/utils/utilities.ts b/packages/nodes-base/utils/utilities.ts index 6783a5d8b6..54949d87c3 100644 --- a/packages/nodes-base/utils/utilities.ts +++ b/packages/nodes-base/utils/utilities.ts @@ -6,9 +6,9 @@ import type { IPairedItemData, } from 'n8n-workflow'; -import { ApplicationError, jsonParse } from 'n8n-workflow'; +import { ApplicationError, jsonParse, randomInt } from 'n8n-workflow'; -import { isEqual, isNull, merge } from 'lodash'; +import { isEqual, isNull, merge, isObject, reduce, get } from 'lodash'; /** * Creates an array of elements split into groups the length of `size`. @@ -41,6 +41,32 @@ export function chunk(array: T[], size = 1) { return result as T[][]; } +/** + * Shuffles an array in place using the Fisher-Yates shuffle algorithm + * @param {Array} array The array to shuffle. + */ +export const shuffleArray = (array: T[]): void => { + for (let i = array.length - 1; i > 0; i--) { + const j = randomInt(i + 1); + [array[i], array[j]] = [array[j], array[i]]; + } +}; + +/** + * Flattens an object with deep data + * @param {IDataObject} data The object to flatten + * @param {string[]} prefix The prefix to add to each key in the returned flat object + */ +export const flattenKeys = (obj: IDataObject, prefix: string[] = []): IDataObject => { + return !isObject(obj) + ? { [prefix.join('.')]: obj } + : reduce( + obj, + (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...prefix, key])), + {}, + ); +}; + /** * Takes a multidimensional array and converts it to a one-dimensional array. * @@ -70,6 +96,38 @@ export function flatten(nestedArray: T[][]) { return result as any; } +/** + * Compares the values of specified keys in two objects. + * + * @param {T} obj1 - The first object to compare. + * @param {T} obj2 - The second object to compare. + * @param {string[]} keys - An array of keys to compare. + * @param {boolean} disableDotNotation - Whether to use dot notation to access nested properties. + * @returns {boolean} - Whether the values of the specified keys are equal in both objects. + */ +export const compareItems = }>( + obj1: T, + obj2: T, + keys: string[], + disableDotNotation: boolean = false, +): boolean => { + let result = true; + for (const key of keys) { + if (!disableDotNotation) { + if (!isEqual(get(obj1.json, key), get(obj2.json, key))) { + result = false; + break; + } + } else { + if (!isEqual(obj1.json[key], obj2.json[key])) { + result = false; + break; + } + } + } + return result; +}; + export function updateDisplayOptions( displayOptions: IDisplayOptions, properties: INodeProperties[], @@ -330,7 +388,7 @@ export function preparePairedItemDataArray( return [pairedItem]; } -export const sanitazeDataPathKey = (item: IDataObject, key: string) => { +export const sanitizeDataPathKey = (item: IDataObject, key: string) => { if (item[key] !== undefined) { return key; }