feat(Google BigQuery Node): Node improvements (#4877)

* ⚡ setup * ⚡ finished v2 setup * ⚡ fix return all, fix simplify with nested schema * ⚡ fix for external tables, updated scopes * ⚡ query operation * ⚡ linter fixes * ⚡ fixed not processed errors when inserting, move main loop to execute function to allow bulk request * ⚡ customizible batch size when inserting, improoved errors * ⚡ options for mapping input * ⚡ fix for inserting RECORD type * ⚡ updated simplify logic * ⚡ fix for return with selected fields * ⚡ option to return table schema * ⚡ linter fixes * ⚡ fix imports * ⚡ query resource and fixes, rlc for projects * ⚡ removed simplify, added raw output option * ⚡ rlc for tables and datasets, no urls option * ⚡ updated hints and description of query parameter, fix getMany VIEW, multioptions fo fields * ⚡ added case when rows are empty * ⚡ linter fixes * ⚡ UI update, one resource * ⚡ fix for output with field named json * ⚡ using jobs instead queries * ⚡ added error message * ⚡ search for RLCs, fixes * ⚡ json processing * ⚡ removed getAll operation * ⚡ executeQuery update * ⚡ unit test * ⚡ tests setup, fixes * ⚡ tests * Remove script for checking unused loadOptions --------- Co-authored-by: agobrech <ael.gobrecht@gmail.com>
2025-12-17 18:12:04 +00:00 · 2023-04-19 15:55:01 +03:00
parent c291ef5dae
commit 9817a15da4
27 changed files with 2521 additions and 287 deletions
--- a/packages/nodes-base/nodes/Google/BigQuery/v1/GenericFunctions.ts
+++ b/packages/nodes-base/nodes/Google/BigQuery/v1/GenericFunctions.ts
@@ -0,0 +1,154 @@
+import type { OptionsWithUri } from 'request';
+
+import type {
+	IExecuteFunctions,
+	IExecuteSingleFunctions,
+	ILoadOptionsFunctions,
+	IDataObject,
+	JsonObject,
+} from 'n8n-workflow';
+import { NodeApiError, NodeOperationError } from 'n8n-workflow';
+
+import moment from 'moment-timezone';
+
+import * as jwt from 'jsonwebtoken';
+
+async function getAccessToken(
+	this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions,
+	credentials: IDataObject,
+): Promise<IDataObject> {
+	//https://developers.google.com/identity/protocols/oauth2/service-account#httprest
+
+	const privateKey = (credentials.privateKey as string).replace(/\\n/g, '\n').trim();
+
+	const scopes = ['https://www.googleapis.com/auth/bigquery'];
+
+	const now = moment().unix();
+
+	const signature = jwt.sign(
+		{
+			iss: credentials.email as string,
+			sub: credentials.delegatedEmail || (credentials.email as string),
+			scope: scopes.join(' '),
+			aud: 'https://oauth2.googleapis.com/token',
+			iat: now,
+			exp: now + 3600,
+		},
+		privateKey,
+		{
+			algorithm: 'RS256',
+			header: {
+				kid: privateKey,
+				typ: 'JWT',
+				alg: 'RS256',
+			},
+		},
+	);
+
+	const options: OptionsWithUri = {
+		headers: {
+			'Content-Type': 'application/x-www-form-urlencoded',
+		},
+		method: 'POST',
+		form: {
+			grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+			assertion: signature,
+		},
+		uri: 'https://oauth2.googleapis.com/token',
+		json: true,
+	};
+
+	return this.helpers.request(options);
+}
+
+export async function googleApiRequest(
+	this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions,
+	method: string,
+	resource: string,
+
+	body: any = {},
+	qs: IDataObject = {},
+	uri?: string,
+	headers: IDataObject = {},
+): Promise<any> {
+	const authenticationMethod = this.getNodeParameter(
+		'authentication',
+		0,
+		'serviceAccount',
+	) as string;
+
+	const options: OptionsWithUri = {
+		headers: {
+			'Content-Type': 'application/json',
+		},
+		method,
+		body,
+		qs,
+		uri: uri || `https://bigquery.googleapis.com/bigquery${resource}`,
+		json: true,
+	};
+	try {
+		if (Object.keys(headers).length !== 0) {
+			options.headers = Object.assign({}, options.headers, headers);
+		}
+		if (Object.keys(body as IDataObject).length === 0) {
+			delete options.body;
+		}
+
+		if (authenticationMethod === 'serviceAccount') {
+			const credentials = await this.getCredentials('googleApi');
+
+			if (credentials === undefined) {
+				throw new NodeOperationError(this.getNode(), 'No credentials got returned!');
+			}
+
+			const { access_token } = await getAccessToken.call(this, credentials as IDataObject);
+
+			options.headers!.Authorization = `Bearer ${access_token}`;
+			return await this.helpers.request(options);
+		} else {
+			return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
+		}
+	} catch (error) {
+		if (error.code === 'ERR_OSSL_PEM_NO_START_LINE') {
+			error.statusCode = '401';
+		}
+
+		throw new NodeApiError(this.getNode(), error as JsonObject);
+	}
+}
+
+export async function googleApiRequestAllItems(
+	this: IExecuteFunctions | ILoadOptionsFunctions,
+	propertyName: string,
+	method: string,
+	endpoint: string,
+
+	body: any = {},
+	query: IDataObject = {},
+): Promise<any> {
+	const returnData: IDataObject[] = [];
+
+	let responseData;
+	query.maxResults = 100;
+
+	do {
+		responseData = await googleApiRequest.call(this, method, endpoint, body, query);
+		query.pageToken = responseData.pageToken;
+		returnData.push.apply(returnData, responseData[propertyName] as IDataObject[]);
+	} while (responseData.pageToken !== undefined && responseData.pageToken !== '');
+
+	return returnData;
+}
+
+export function simplify(rows: IDataObject[], fields: string[]) {
+	const results = [];
+	for (const row of rows) {
+		const record: IDataObject = {};
+		for (const [index, field] of fields.entries()) {
+			record[field] = (row.f as IDataObject[])[index].v;
+		}
+		results.push(record);
+	}
+	return results;
+}