admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 10:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(Google BigQuery Node): Send timeoutMs in query, pagination support (#10205)

Browse Source
This commit is contained in:
Michael Kret
2024-07-29 21:28:16 +03:00
committed by GitHub
parent cf70b06545
commit f5722e8823
8 changed files with 80 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
packages/nodes-base/nodes/Google/BigQuery/v2/actions/database/executeQuery.operation.ts
View File

@@ -9,7 +9,7 @@ import { ApplicationError, NodeOperationError, sleep } from 'n8n-workflow';
import type { ResponseWithJobReference } from '../../helpers/interfaces';
import { prepareOutput } from '../../helpers/utils';
import { googleApiRequest } from '../../transport';
import { googleBigQueryApiRequestAllItems, googleBigQueryApiRequest } from '../../transport';
import { getResolvables, updateDisplayOptions } from '@utils/utilities';
const properties: INodeProperties[] = [
@@ -108,18 +108,21 @@ const properties: INodeProperties[] = [
'Limits the bytes billed for this query. Queries with bytes billed above this limit will fail (without incurring a charge). String in <a href="https://developers.google.com/discovery/v1/type-format?utm_source=cloud.google.com&utm_medium=referral" target="_blank">Int64Value</a> format',
},
{
displayName: 'Max Results',
displayName: 'Max Results Per Page',
name: 'maxResults',
type: 'number',
default: 1000,
description: 'The maximum number of rows of data to return',
description:
'Maximum number of results to return per page of results. This is particularly useful when dealing with large datasets. It will not affect the total number of results returned, e.g. rows in a table. You can use LIMIT in your SQL query to limit the number of rows returned.',
},
{
displayName: 'Timeout',
name: 'timeoutMs',
type: 'number',
default: 10000,
description: 'How long to wait for the query to complete, in milliseconds',
hint: 'How long to wait for the query to complete, in milliseconds',
description:
'Specifies the maximum amount of time, in milliseconds, that the client is willing to wait for the query to complete. Be aware that the call is not guaranteed to wait for the specified timeout; it typically returns after around 200 seconds (200,000 milliseconds), even if the query is not complete.',
},
{
displayName: 'Raw Output',
@@ -154,19 +157,31 @@ const displayOptions = {
export const description = updateDisplayOptions(displayOptions, properties);
export async function execute(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
// https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query
const items = this.getInputData();
const length = items.length;
const returnData: INodeExecutionData[] = [];
let jobs = [];
let maxResults = 1000;
let timeoutMs = 10000;
for (let i = 0; i < length; i++) {
try {
let sqlQuery = this.getNodeParameter('sqlQuery', i) as string;
const options = this.getNodeParameter('options', i);
const options = this.getNodeParameter('options', i) as {
defaultDataset?: string;
dryRun?: boolean;
includeSchema?: boolean;
location?: string;
maximumBytesBilled?: string;
maxResults?: number;
timeoutMs?: number;
rawOutput?: boolean;
useLegacySql?: boolean;
};
const projectId = this.getNodeParameter('projectId', i, undefined, {
extractValue: true,
});
@@ -179,15 +194,25 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
let includeSchema = false;
if (options.rawOutput !== undefined) {
rawOutput = options.rawOutput as boolean;
rawOutput = options.rawOutput;
delete options.rawOutput;
}
if (options.includeSchema !== undefined) {
includeSchema = options.includeSchema as boolean;
includeSchema = options.includeSchema;
delete options.includeSchema;
}
if (options.maxResults) {
maxResults = options.maxResults;
delete options.maxResults;
}
if (options.timeoutMs) {
timeoutMs = options.timeoutMs;
delete options.timeoutMs;
}
const body: IDataObject = { ...options };
body.query = sqlQuery;
@@ -203,7 +228,8 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
body.useLegacySql = false;
}
const response: ResponseWithJobReference = await googleApiRequest.call(
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/insert
const response: ResponseWithJobReference = await googleBigQueryApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/jobs`,
@@ -222,13 +248,14 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
}
const jobId = response?.jobReference?.jobId;
const raw = rawOutput || (options.dryRun as boolean) || false;
const raw = rawOutput || options.dryRun || false;
const location = options.location || response.jobReference.location;
if (response.status?.state === 'DONE') {
const qs = { location };
const qs = { location, maxResults, timeoutMs };
const queryResponse: IDataObject = await googleApiRequest.call(
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
const queryResponse: IDataObject = await googleBigQueryApiRequestAllItems.call(
this,
'GET',
`/v2/projects/${projectId}/queries/${jobId}`,
@@ -272,9 +299,13 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
for (const job of jobs) {
try {
const qs = job.location ? { location: job.location } : {};
const qs: IDataObject = job.location ? { location: job.location } : {};
const response: IDataObject = await googleApiRequest.call(
qs.maxResults = maxResults;
qs.timeoutMs = timeoutMs;
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
const response: IDataObject = await googleBigQueryApiRequestAllItems.call(
this,
'GET',
`/v2/projects/${job.projectId}/queries/${job.jobId}`,