fix(core): Fix data transformation function that are reported not to work properly (#5338)

* 🔥 Remove test extensions * 🚧 Add test description * 📘 Expand types * ⚡ Export extensions * ⚡ Export collection * ⚡ Mark all proxies * ✏️ Rename for clarity * ⚡ Export from barrel * ✨ Create datatype completions * ⚡ Mount datatype completions * 🧪 Adjust tests * ⚡ Add `path` prop * 🔥 Remove `()` from completion labels * ⚡ Filter out completions for pseudo-proxies * 🐛 Fix method error * ⚡ Add metrics * ✏️ Improve naming * ✨ Start completion on empty resolvable * ✨ Implement completion previews * ⚡ Break out completion manager * ⚡ Implement in expression editor modal * ✏️ Improve naming * ⚡ Filter out irrelevant completions * ✨ Add preview hint * ✏️ Improve comments * 🎨 Style preview hint * ⚡ Expand `hasNoParams` * ⚡ Add spacing for readability * ⚡ Add error codes * ✏️ Add comment * 🐛 Fix Esc behavior * ⚡ Parse Unicode * ⚡ Throw on invalid `DateTime` * ⚡ Fix second root completion detection * ⚡ Switch message at completable prefix position * 🐛 Fix function names for non-dev build * 🐛 Fix `json` handling * 🔥 Comment out previews * ♻️ Apply feedback * 🔥 Remove extensions * 🚚 Rename extensions * ⚡ Adjust some implementations * 🔥 Remove dummy extensions * 🐛 Fix object regex * ♻️ Apply feedback * ✏️ Fix typos * ✏️ Add `fn is not a function` message * 🔥 Remove check * ✨ Add `isNotEmpty` for objects * 🚚 Rename `global` to `alpha` * 🔥 Remove `encrypt` * ⚡ Restore `is not a function` error * ⚡ Support `week` on `extract()` * 🧪 Fix tests * ⚡ Add validation to some string extensions * ⚡ Validate number arrays in some extensions * 🧪 Fix tests * ✏️ Improve error message * ⏪ Revert extensions framework changes * 🧹 Previews cleanup * ⚡ Condense blank completions * ⚡ Refactor dollar completions * ⚡ Refactor non-dollar completions * ⚡ Refactor Luxon completions * ⚡ Refactor datatype completions * ⚡ Use `DATETIMEUNIT_MAP` * ✏️ Update test description * ⏪ Revert "Use `DATETIMEUNIT_MAP`" This reverts commit 472a77df5cd789905d162f3c3db02ac767b89b4e. * 🧪 Add tests * ♻️ Restore generic extensions * 🔥 Remove logs * 🧪 Expand tests * ✨ Add `Math` completions * ✏️ List breaking change * ⚡ Add doc tooltips * 🐛 Fix node selector regex * 🐛 Fix `context` resolution * 🐛 Allow dollar completions in args * ⚡ Make numeric array methods context-dependent * 📝 Adjust docs * 🐛 Fix selector ref * ⚡ Surface error for valid URL * 🐛 Disallow whitespace in `isEmail` check * 🧪 Fix test for `isUrl` * ⚡ Add comma validator in `toFloat` * ⚡ Add validation to `$jmespath()` * ⏪ Revert valid URL error * ⚡ Adjust `$jmespath()` validation * 🧪 Adjust `isUrl` test * ⚡ Remove `{}` and `[]` from compact * ✏️ Update docs * 🚚 Rename `stripTags` to `removeTags` * ⚡ Do not inject whitespace inside resolvable * ⚡ Make completions aware of `()` * ✏️ Add note * ⚡ Update sorting * ⚡ Hide active node name from node selector * 🔥 Remove `length()` and its aliases * ⚡ Validate non-zero for `chunk` * ✏️ Reword all error messages * 🐛 Fix `$now` and `$today` * ⚡ Simplify with `stripExcessParens` * ⚡ Fold luxon into datatype * 🧪 Clean up tests * 🔥 Remove tests for removed methods * 👕 Fix type * ⬆️ Upgrade lang pack * ⏪ Undo change to `vitest` command * 🔥 Remove unused method * ⚡ Separate `return` line * ✏️ Improve description * 🧪 Expand tests for initial-only completions * 🧪 Add bracket-aware completions * ⚡ Make check for `all()` stricter * ✏️ Adjust explanatory comments * 🔥 Remove unneded copy * 🔥 Remove outdated comment * ⚡ Make naming consistent * ✏️ Update comments * ⚡ Improve URL scheme check * ✏️ Add comment * 🚚 Move extension * ✏️ Update `BREAKING-CHANGES.md` * ✏️ Update upcoming version * ✏️ Fix grammar * ✏️ Shorten message * 🐛 Fix `Esc` behavior * 🐛 Fix `isNumeric` * ✨ Using UTC to handle-dates on back-end * ✅ Added more unit tests for date extensions * ⚡ Not using `JSON.stringify` to render dates * ⚡ Using `deep-equal` library instead of our `deepCompare` function * ✅ Adding more tests to array extensions * ⚡ Fixing `inBetween` extension function * ✅ Added tests for `.inBetween()` * ⚡ Updating `isEven` and `isOdd` to throw for floats * ⚡ Updating `Array.merge()` so it works without arguments * 🔀 Fixing leftover merge confilct * ⚡ Updating `removeFieldsContaining` and `keepFieldsContaining` to throw on empty strings * ⚡ Fixing `pluck()` so it returns only plucked values * ⬆️ Updating pnpm lockfile * 👕 Fixing lint errors * ⚡ Using workflow timezone to display dates * ✔️ Updating tests with workflow timezone * ⚡ Not using system timezone when creating Luxon dates * ⚡ Updating `merge()` and `pluck()` array functions * 🔀 Sync with `master`: Removing code that was preserved during merge * ⚡ Updating `.pluck()` to return full array if no arguments are passed * ⚡ Updating `keepFieldsContaining` and `merge` object functions * ⚡ Using week as default for `date.extract()` * ✅ Adding more test cases for DT functions * ⚡ Removing `Object.merge` extension function. Adding missing `deep-equal` dependency * ⚡ Handling `toDate` case when time component is not specified * ⚡ Using workflow's timezone to render dates in output panel, updated unit tests after removing `Object.merge` function * ⚡ Not parsing numbers as dates * 👕 Fixing lint errors * ⚡ Fixing a typo * ⚡ Making date detection more strict so only stringified dates are getting converted * 👌 Addressing PR feedback * 🔥 Removing leftover comment --------- Co-authored-by: Iván Ovejero <ivov.src@gmail.com>
2025-12-20 11:22:15 +00:00 · 2023-02-15 10:50:16 +01:00
parent 83505cb0d4
commit 0cf45bc4c8
17 changed files with 372 additions and 224 deletions
--- a/packages/workflow/src/Extensions/ArrayExtensions.ts
+++ b/packages/workflow/src/Extensions/ArrayExtensions.ts
@@ -1,87 +1,7 @@
 import { ExpressionError, ExpressionExtensionError } from '../ExpressionError';
 import type { ExtensionMap } from './Extensions';
-import { compact as oCompact, merge as oMerge } from './ObjectExtensions';
-
-function deepCompare(left: unknown, right: unknown): boolean {
-	if (left === right) {
-		return true;
-	}
-
-	// Check to see if they're the basic type
-	if (typeof left !== typeof right) {
-		return false;
-	}
-
-	if (typeof left === 'number' && isNaN(left) && isNaN(right as number)) {
-		return true;
-	}
-
-	// Explicitly return false if certain primitives don't equal each other
-	if (['number', 'string', 'bigint', 'boolean', 'symbol'].includes(typeof left) && left !== right) {
-		return false;
-	}
-
-	// Quickly check how many properties each has to avoid checking obviously mismatching
-	// objects
-	if (Object.keys(left as object).length !== Object.keys(right as object).length) {
-		return false;
-	}
-
-	// Quickly check if they're arrays
-	if (Array.isArray(left) !== Array.isArray(right)) {
-		return false;
-	}
-
-	// Check if arrays are equal, ordering is important
-	if (Array.isArray(left)) {
-		if (left.length !== (right as unknown[]).length) {
-			return false;
-		}
-		return left.every((v, i) => deepCompare(v, (right as object[])[i]));
-	}
-
-	// Check right first quickly. This is to see if we have mismatched properties.
-	// We'll check the left more indepth later to cover all our bases.
-	for (const key in right as object) {
-		if ((left as object).hasOwnProperty(key) !== (right as object).hasOwnProperty(key)) {
-			return false;
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-		} else if (typeof (left as any)[key] !== typeof (right as any)[key]) {
-			return false;
-		}
-	}
-
-	// Check left more in depth
-	for (const key in left as object) {
-		if ((left as object).hasOwnProperty(key) !== (right as object).hasOwnProperty(key)) {
-			return false;
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-		} else if (typeof (left as any)[key] !== typeof (right as any)[key]) {
-			return false;
-		}
-
-		if (
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-			typeof (left as any)[key] === 'object'
-		) {
-			if (
-				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-				(left as any)[key] !== (right as any)[key] &&
-				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-				!deepCompare((left as any)[key], (right as any)[key])
-			) {
-				return false;
-			}
-		} else {
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-			if ((left as any)[key] !== (right as any)[key]) {
-				return false;
-			}
-		}
-	}
-
-	return true;
-}
+import { compact as oCompact } from './ObjectExtensions';
+import deepEqual from 'deep-equal';

 function first(value: unknown[]): unknown {
 	return value[0];
@@ -103,18 +23,26 @@ function pluck(value: unknown[], extraArgs: unknown[]): unknown[] {
 	if (!Array.isArray(extraArgs)) {
 		throw new ExpressionError('arguments must be passed to pluck');
 	}
-	const fieldsToPluck = extraArgs;
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any
-	return (value as any[]).map((element: object) => {
-		const entries = Object.entries(element);
-		return entries.reduce((p, c) => {
-			const [key, val] = c as [string, Date | string | number];
-			if (fieldsToPluck.includes(key)) {
-				Object.assign(p, { [key]: val });
+	if (!extraArgs || extraArgs.length === 0) {
+		return value;
+	}
+	const plucked = value.reduce<unknown[]>((pluckedFromObject, current) => {
+		if (current && typeof current === 'object') {
+			const p: unknown[] = [];
+			Object.keys(current).forEach((k) => {
+				extraArgs.forEach((field: string) => {
+					if (current && field === k) {
+						p.push((current as { [key: string]: unknown })[k]);
+					}
+				});
+			});
+			if (p.length > 0) {
+				pluckedFromObject.push(p.length === 1 ? p[0] : p);
 			}
-			return p;
-		}, {});
-	}) as unknown[];
+		}
+		return pluckedFromObject;
+	}, new Array<unknown>());
+	return plucked;
 }

 function randomItem(value: unknown[]): unknown {
@@ -127,8 +55,13 @@ function unique(value: unknown[], extraArgs: string[]): unknown[] {
 		return value.reduce<unknown[]>((l, v) => {
 			if (typeof v === 'object' && v !== null && extraArgs.every((i) => i in v)) {
 				const alreadySeen = l.find((i) =>
-					// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
-					extraArgs.every((j) => deepCompare((i as any)[j], (v as any)[j])),
+					extraArgs.every((j) =>
+						deepEqual(
+							(i as Record<string, unknown>)[j],
+							(v as Record<string, unknown>, { strict: true })[j],
+							{ strict: true },
+						),
+					),
 				);
 				if (!alreadySeen) {
 					l.push(v);
@@ -138,7 +71,7 @@ function unique(value: unknown[], extraArgs: string[]): unknown[] {
 		}, []);
 	}
 	return value.reduce<unknown[]>((l, v) => {
-		if (l.findIndex((i) => deepCompare(i, v)) === -1) {
+		if (l.findIndex((i) => deepEqual(i, v, { strict: true })) === -1) {
 			l.push(v);
 		}
 		return l;
@@ -281,28 +214,58 @@ function renameKeys(value: unknown[], extraArgs: string[]): unknown[] {
 	});
 }

-function merge(value: unknown[], extraArgs: unknown[][]): unknown[] {
+function mergeObjects(value: Record<string, unknown>, extraArgs: unknown[]): unknown {
+	const [other] = extraArgs;
+
+	if (!other) {
+		return value;
+	}
+
+	if (typeof other !== 'object') {
+		throw new ExpressionExtensionError('merge(): expected object arg');
+	}
+
+	const newObject = { ...value };
+	for (const [key, val] of Object.entries(other)) {
+		if (!(key in newObject)) {
+			newObject[key] = val;
+		}
+	}
+	return newObject;
+}
+
+function merge(value: unknown[], extraArgs: unknown[][]): unknown {
 	const [others] = extraArgs;
+
+	if (others === undefined) {
+		// If there are no arguments passed, merge all objects within the array
+		const merged = value.reduce((combined, current) => {
+			if (current !== null && typeof current === 'object' && !Array.isArray(current)) {
+				combined = mergeObjects(combined as Record<string, unknown>, [current]);
+			}
+			return combined;
+		}, {});
+		return merged;
+	}
+
 	if (!Array.isArray(others)) {
 		throw new ExpressionExtensionError(
 			'merge(): expected array arg, e.g. .merge([{ id: 1, otherValue: 3 }])',
 		);
 	}
 	const listLength = value.length > others.length ? value.length : others.length;
-	const newList = new Array(listLength);
+	let merged = {};
 	for (let i = 0; i < listLength; i++) {
 		if (value[i] !== undefined) {
 			if (typeof value[i] === 'object' && typeof others[i] === 'object') {
-				newList[i] = oMerge(value[i] as object, [others[i]]);
-			} else {
-				newList[i] = value[i];
+				merged = Object.assign(
+					merged,
+					mergeObjects(value[i] as Record<string, unknown>, [others[i]]),
+				);
 			}
-		} else {
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-			newList[i] = others[i];
 		}
 	}
-	return newList;
+	return merged;
 }

 function union(value: unknown[], extraArgs: unknown[][]): unknown[] {
@@ -312,7 +275,8 @@ function union(value: unknown[], extraArgs: unknown[][]): unknown[] {
 	}
 	const newArr: unknown[] = Array.from(value);
 	for (const v of others) {
-		if (newArr.findIndex((w) => deepCompare(w, v)) === -1) {
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-call
+		if (newArr.findIndex((w) => deepEqual(w, v, { strict: true })) === -1) {
 			newArr.push(v);
 		}
 	}
@@ -328,7 +292,7 @@ function difference(value: unknown[], extraArgs: unknown[][]): unknown[] {
 	}
 	const newArr: unknown[] = [];
 	for (const v of value) {
-		if (others.findIndex((w) => deepCompare(w, v)) === -1) {
+		if (others.findIndex((w) => deepEqual(w, v, { strict: true })) === -1) {
 			newArr.push(v);
 		}
 	}
@@ -344,12 +308,13 @@ function intersection(value: unknown[], extraArgs: unknown[][]): unknown[] {
 	}
 	const newArr: unknown[] = [];
 	for (const v of value) {
-		if (others.findIndex((w) => deepCompare(w, v)) !== -1) {
+		if (others.findIndex((w) => deepEqual(w, v, { strict: true })) !== -1) {
 			newArr.push(v);
 		}
 	}
 	for (const v of others) {
-		if (value.findIndex((w) => deepCompare(w, v)) !== -1) {
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-call
+		if (value.findIndex((w) => deepEqual(w, v, { strict: true })) !== -1) {
 			newArr.push(v);
 		}
 	}
--- a/packages/workflow/src/Extensions/DateExtensions.ts
+++ b/packages/workflow/src/Extensions/DateExtensions.ts
@@ -1,8 +1,10 @@
+import { ExpressionExtensionError } from './../ExpressionError';
 /* eslint-disable @typescript-eslint/unbound-method */
 /* eslint-disable @typescript-eslint/explicit-member-accessibility */
 import { DateTime } from 'luxon';
 import type { DateTimeUnit, DurationLike, DurationObjectUnits, LocaleOptions } from 'luxon';
 import type { ExtensionMap } from './Extensions';
+import { convertToDateTime } from './utils';

 type DurationUnit =
 	| 'milliseconds'
@@ -82,11 +84,8 @@ function beginningOf(date: Date | DateTime, extraArgs: DurationUnit[]): Date {
 	if (isDateTime(date)) {
 		return date.startOf(DATETIMEUNIT_MAP[unit] || unit).toJSDate();
 	}
-	let datetime = DateTime.fromJSDate(date);
-	if (date.getTimezoneOffset() === 0) {
-		datetime = datetime.setZone('UTC');
-	}
-	return datetime.startOf(DATETIMEUNIT_MAP[unit] || unit).toJSDate();
+	const dateTime = DateTime.fromJSDate(date);
+	return dateTime.startOf(DATETIMEUNIT_MAP[unit] || unit).toJSDate();
 }

 function endOfMonth(date: Date | DateTime): Date {
@@ -97,7 +96,7 @@ function endOfMonth(date: Date | DateTime): Date {
 }

 function extract(inputDate: Date | DateTime, extraArgs: DatePart[]): number | Date {
-	let [part] = extraArgs;
+	let [part = 'week'] = extraArgs;
 	let date = inputDate;
 	if (isDateTime(date)) {
 		date = date.toJSDate();
@@ -126,10 +125,22 @@ function format(date: Date | DateTime, extraArgs: unknown[]): string {
 	return DateTime.fromJSDate(date).toFormat(dateFormat, { ...localeOpts });
 }

-function isBetween(date: Date | DateTime, extraArgs: unknown[]): boolean {
-	const [first, second] = extraArgs as string[];
-	const firstDate = new Date(first);
-	const secondDate = new Date(second);
+function isBetween(
+	date: Date | DateTime,
+	extraArgs: Array<string | Date | DateTime>,
+): boolean | undefined {
+	if (extraArgs.length !== 2) {
+		throw new ExpressionExtensionError('isBetween(): expected exactly two args');
+	}
+
+	const [first, second] = extraArgs;
+
+	const firstDate = convertToDateTime(first);
+	const secondDate = convertToDateTime(second);
+
+	if (!firstDate || !secondDate) {
+		return;
+	}

 	if (firstDate > secondDate) {
 		return secondDate < date && date < firstDate;
--- a/packages/workflow/src/Extensions/NumberExtensions.ts
+++ b/packages/workflow/src/Extensions/NumberExtensions.ts
@@ -1,7 +1,7 @@
 /**
 * @jest-environment jsdom
 */
-
+import { ExpressionExtensionError } from './../ExpressionError';
 import type { ExtensionMap } from './Extensions';

 function format(value: number, extraArgs: unknown[]): string {
@@ -14,10 +14,16 @@ function format(value: number, extraArgs: unknown[]): string {
 }

 function isEven(value: number) {
+	if (!Number.isInteger(value)) {
+		throw new ExpressionExtensionError('isEven() is only callable on integers');
+	}
 	return value % 2 === 0;
 }

 function isOdd(value: number) {
+	if (!Number.isInteger(value)) {
+		throw new ExpressionExtensionError('isOdd() is only callable on integers');
+	}
 	return Math.abs(value) % 2 === 1;
 }

--- a/packages/workflow/src/Extensions/ObjectExtensions.ts
+++ b/packages/workflow/src/Extensions/ObjectExtensions.ts
@@ -1,22 +1,6 @@
 import { ExpressionExtensionError } from '../ExpressionError';
 import type { ExtensionMap } from './Extensions';

-export function merge(value: object, extraArgs: unknown[]): unknown {
-	const [other] = extraArgs;
-	if (typeof other !== 'object' || !other) {
-		throw new ExpressionExtensionError('merge(): expected object arg');
-	}
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any
-	const newObject: any = { ...value };
-	for (const [key, val] of Object.entries(other)) {
-		if (!(key in newObject)) {
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
-			newObject[key] = val;
-		}
-	}
-	return newObject;
-}
-
 function isEmpty(value: object): boolean {
 	return Object.keys(value).length === 0;
 }
@@ -43,8 +27,8 @@ function removeField(value: object, extraArgs: string[]): object {

 function removeFieldsContaining(value: object, extraArgs: string[]): object {
 	const [match] = extraArgs;
-	if (typeof match !== 'string') {
-		throw new ExpressionExtensionError('removeFieldsContaining(): expected string arg');
+	if (typeof match !== 'string' || match === '') {
+		throw new ExpressionExtensionError('removeFieldsContaining(): expected non-empty string arg');
 	}
 	const newObject = { ...value };
 	for (const [key, val] of Object.entries(value)) {
@@ -58,12 +42,14 @@ function removeFieldsContaining(value: object, extraArgs: string[]): object {

 function keepFieldsContaining(value: object, extraArgs: string[]): object {
 	const [match] = extraArgs;
-	if (typeof match !== 'string') {
-		throw new ExpressionExtensionError('argument of keepFieldsContaining must be a string');
+	if (typeof match !== 'string' || match === '') {
+		throw new ExpressionExtensionError(
+			'argument of keepFieldsContaining must be a non-empty string',
+		);
 	}
 	const newObject = { ...value };
 	for (const [key, val] of Object.entries(value)) {
-		if (typeof val === 'string' && !val.includes(match)) {
+		if (typeof val !== 'string' || (typeof val === 'string' && !val.includes(match))) {
 			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
 			delete (newObject as any)[key];
 		}
@@ -120,11 +106,6 @@ urlEncode.doc = {

 // @TODO_NEXT_PHASE: Surface extensions below which take args

-merge.doc = {
-	name: 'merge',
-	returnType: 'object',
-};
-
 hasField.doc = {
 	name: 'hasField',
 	returnType: 'boolean',
@@ -150,7 +131,6 @@ export const objectExtensions: ExtensionMap = {
 	functions: {
 		isEmpty,
 		isNotEmpty,
-		merge,
 		hasField,
 		removeField,
 		removeFieldsContaining,
--- a/packages/workflow/src/Extensions/StringExtensions.ts
+++ b/packages/workflow/src/Extensions/StringExtensions.ts
@@ -113,12 +113,15 @@ function removeTags(value: string): string {
 }

 function toDate(value: string): Date {
-	const date = new Date(value.toString());
+	const date = new Date(Date.parse(value));

 	if (date.toString() === 'Invalid Date') {
 		throw new ExpressionError.ExpressionExtensionError('cannot convert to date');
 	}
-
+	// If time component is not specified, force 00:00h
+	if (!/:/.test(value)) {
+		date.setHours(0, 0, 0);
+	}
 	return date;
 }

--- a/packages/workflow/src/Extensions/utils.ts
+++ b/packages/workflow/src/Extensions/utils.ts
@@ -0,0 +1,19 @@
+import { DateTime } from 'luxon';
+
+// Utility functions and type guards for expression extensions
+
+export const convertToDateTime = (value: string | Date | DateTime): DateTime | undefined => {
+	let converted: DateTime | undefined;
+
+	if (typeof value === 'string') {
+		converted = DateTime.fromJSDate(new Date(value));
+		if (converted.invalidReason !== null) {
+			return;
+		}
+	} else if (value instanceof Date) {
+		converted = DateTime.fromJSDate(value);
+	} else if (DateTime.isDateTime(value)) {
+		converted = value;
+	}
+	return converted;
+};