feat(Google BigQuery Node): Node improvements (#4877)

* ⚡ setup * ⚡ finished v2 setup * ⚡ fix return all, fix simplify with nested schema * ⚡ fix for external tables, updated scopes * ⚡ query operation * ⚡ linter fixes * ⚡ fixed not processed errors when inserting, move main loop to execute function to allow bulk request * ⚡ customizible batch size when inserting, improoved errors * ⚡ options for mapping input * ⚡ fix for inserting RECORD type * ⚡ updated simplify logic * ⚡ fix for return with selected fields * ⚡ option to return table schema * ⚡ linter fixes * ⚡ fix imports * ⚡ query resource and fixes, rlc for projects * ⚡ removed simplify, added raw output option * ⚡ rlc for tables and datasets, no urls option * ⚡ updated hints and description of query parameter, fix getMany VIEW, multioptions fo fields * ⚡ added case when rows are empty * ⚡ linter fixes * ⚡ UI update, one resource * ⚡ fix for output with field named json * ⚡ using jobs instead queries * ⚡ added error message * ⚡ search for RLCs, fixes * ⚡ json processing * ⚡ removed getAll operation * ⚡ executeQuery update * ⚡ unit test * ⚡ tests setup, fixes * ⚡ tests * Remove script for checking unused loadOptions --------- Co-authored-by: agobrech <ael.gobrecht@gmail.com>
2025-12-17 10:02:05 +00:00 · 2023-04-19 15:55:01 +03:00
parent c291ef5dae
commit 9817a15da4
27 changed files with 2521 additions and 287 deletions
--- a/packages/nodes-base/nodes/Google/BigQuery/v2/helpers/utils.ts
+++ b/packages/nodes-base/nodes/Google/BigQuery/v2/helpers/utils.ts
@@ -0,0 +1,135 @@
+import type { IExecuteFunctions } from 'n8n-core';
+import { constructExecutionMetaData } from 'n8n-core';
+import type { IDataObject, INodeExecutionData } from 'n8n-workflow';
+import { jsonParse, NodeOperationError } from 'n8n-workflow';
+import type { SchemaField, TableRawData, TableSchema } from './interfaces';
+
+function getFieldValue(schemaField: SchemaField, field: IDataObject) {
+	if (schemaField.type === 'RECORD') {
+		// eslint-disable-next-line @typescript-eslint/no-use-before-define
+		return simplify([field.v as TableRawData], schemaField.fields as unknown as SchemaField[]);
+	} else {
+		let value = field.v;
+		if (schemaField.type === 'JSON') {
+			try {
+				value = jsonParse(value as string);
+			} catch (error) {}
+		}
+		return value;
+	}
+}
+
+export function wrapData(data: IDataObject | IDataObject[]): INodeExecutionData[] {
+	if (!Array.isArray(data)) {
+		return [{ json: data }];
+	}
+	return data.map((item) => ({
+		json: item,
+	}));
+}
+
+export function simplify(data: TableRawData[], schema: SchemaField[], includeSchema = false) {
+	const returnData: IDataObject[] = [];
+	for (const entry of data) {
+		const record: IDataObject = {};
+
+		for (const [index, field] of entry.f.entries()) {
+			if (schema[index].mode !== 'REPEATED') {
+				record[schema[index].name] = getFieldValue(schema[index], field);
+			} else {
+				record[schema[index].name] = (field.v as unknown as IDataObject[]).flatMap(
+					(repeatedField) => {
+						return getFieldValue(schema[index], repeatedField as unknown as IDataObject);
+					},
+				);
+			}
+		}
+
+		if (includeSchema) {
+			record._schema = schema;
+		}
+
+		returnData.push(record);
+	}
+
+	return returnData;
+}
+
+export function prepareOutput(
+	response: IDataObject,
+	itemIndex: number,
+	rawOutput: boolean,
+	includeSchema = false,
+) {
+	let responseData;
+
+	if (response === undefined) return [];
+
+	if (rawOutput) {
+		responseData = response;
+	} else {
+		const { rows, schema } = response;
+
+		if (rows !== undefined && schema !== undefined) {
+			const fields = (schema as TableSchema).fields;
+			responseData = rows;
+
+			responseData = simplify(responseData as TableRawData[], fields, includeSchema);
+		} else if (schema && includeSchema) {
+			responseData = { success: true, _schema: schema };
+		} else {
+			responseData = { success: true };
+		}
+	}
+
+	const executionData = constructExecutionMetaData(wrapData(responseData as IDataObject[]), {
+		itemData: { item: itemIndex },
+	});
+
+	return executionData;
+}
+
+export function checkSchema(
+	this: IExecuteFunctions,
+	schema: TableSchema,
+	record: IDataObject,
+	i: number,
+) {
+	const returnData = { ...record };
+
+	schema.fields.forEach(({ name, mode, type, fields }) => {
+		if (mode === 'REQUIRED' && returnData[name] === undefined) {
+			throw new NodeOperationError(
+				this.getNode(),
+				`The property '${name}' is required, please define it in the 'Fields to Send'`,
+				{ itemIndex: i },
+			);
+		}
+		if (type !== 'STRING' && returnData[name] === '') {
+			returnData[name] = null;
+		}
+		if (type === 'JSON') {
+			let value = returnData[name];
+			if (typeof value === 'object') {
+				value = JSON.stringify(value);
+			}
+			returnData[name] = value;
+		}
+		if (type === 'RECORD' && typeof returnData[name] !== 'object') {
+			let parsedField;
+			try {
+				parsedField = jsonParse(returnData[name] as string);
+			} catch (error) {
+				const recordField = fields ? `Field Schema:\n ${JSON.stringify(fields)}` : '';
+				throw new NodeOperationError(
+					this.getNode(),
+					`The property '${name}' is a RECORD type, but the value is nor an object nor a valid JSON string`,
+					{ itemIndex: i, description: recordField },
+				);
+			}
+			returnData[name] = parsedField as IDataObject;
+		}
+	});
+
+	return returnData;
+}