mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-17 10:02:05 +00:00
feat(Google BigQuery Node): Node improvements (#4877)
* ⚡ setup * ⚡ finished v2 setup * ⚡ fix return all, fix simplify with nested schema * ⚡ fix for external tables, updated scopes * ⚡ query operation * ⚡ linter fixes * ⚡ fixed not processed errors when inserting, move main loop to execute function to allow bulk request * ⚡ customizible batch size when inserting, improoved errors * ⚡ options for mapping input * ⚡ fix for inserting RECORD type * ⚡ updated simplify logic * ⚡ fix for return with selected fields * ⚡ option to return table schema * ⚡ linter fixes * ⚡ fix imports * ⚡ query resource and fixes, rlc for projects * ⚡ removed simplify, added raw output option * ⚡ rlc for tables and datasets, no urls option * ⚡ updated hints and description of query parameter, fix getMany VIEW, multioptions fo fields * ⚡ added case when rows are empty * ⚡ linter fixes * ⚡ UI update, one resource * ⚡ fix for output with field named json * ⚡ using jobs instead queries * ⚡ added error message * ⚡ search for RLCs, fixes * ⚡ json processing * ⚡ removed getAll operation * ⚡ executeQuery update * ⚡ unit test * ⚡ tests setup, fixes * ⚡ tests * Remove script for checking unused loadOptions --------- Co-authored-by: agobrech <ael.gobrecht@gmail.com>
This commit is contained in:
Michael Kret
@@ -0,0 +1,285 @@
|
||||
import type { IExecuteFunctions } from 'n8n-core';
|
||||
import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow';
|
||||
import { NodeOperationError } from 'n8n-workflow';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import { updateDisplayOptions } from '../../../../../../utils/utilities';
|
||||
import type { TableSchema } from '../../helpers/interfaces';
|
||||
import { checkSchema, wrapData } from '../../helpers/utils';
|
||||
import { googleApiRequest } from '../../transport';
|
||||
|
||||
const properties: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Data Mode',
|
||||
name: 'dataMode',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Auto-Map Input Data',
|
||||
value: 'autoMap',
|
||||
description: 'Use when node input properties match destination field names',
|
||||
},
|
||||
{
|
||||
name: 'Map Each Field Below',
|
||||
value: 'define',
|
||||
description: 'Set the value for each destination field',
|
||||
},
|
||||
],
|
||||
default: 'autoMap',
|
||||
description: 'Whether to insert the input data this node receives in the new row',
|
||||
},
|
||||
{
|
||||
displayName:
|
||||
"In this mode, make sure the incoming data fields are named the same as the columns in BigQuery. (Use a 'set' node before this node to change them if required.)",
|
||||
name: 'info',
|
||||
type: 'notice',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['autoMap'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields to Send',
|
||||
name: 'fieldsUi',
|
||||
placeholder: 'Add Field',
|
||||
type: 'fixedCollection',
|
||||
typeOptions: {
|
||||
multipleValueButtonText: 'Add Field',
|
||||
multipleValues: true,
|
||||
},
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Field',
|
||||
name: 'values',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Field Name or ID',
|
||||
name: 'fieldId',
|
||||
type: 'options',
|
||||
description:
|
||||
'Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code-examples/expressions/">expression</a>',
|
||||
typeOptions: {
|
||||
loadOptionsDependsOn: ['projectId.value', 'datasetId.value', 'tableId.value'],
|
||||
loadOptionsMethod: 'getSchema',
|
||||
},
|
||||
default: '',
|
||||
},
|
||||
{
|
||||
displayName: 'Field Value',
|
||||
name: 'fieldValue',
|
||||
type: 'string',
|
||||
default: '',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['define'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Options',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Batch Size',
|
||||
name: 'batchSize',
|
||||
type: 'number',
|
||||
default: 100,
|
||||
typeOptions: {
|
||||
minValue: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Ignore Unknown Values',
|
||||
name: 'ignoreUnknownValues',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether to gnore row values that do not match the schema',
|
||||
},
|
||||
{
|
||||
displayName: 'Skip Invalid Rows',
|
||||
name: 'skipInvalidRows',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether to skip rows with values that do not match the schema',
|
||||
},
|
||||
{
|
||||
displayName: 'Template Suffix',
|
||||
name: 'templateSuffix',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>',
|
||||
},
|
||||
{
|
||||
displayName: 'Trace ID',
|
||||
name: 'traceId',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const displayOptions = {
|
||||
show: {
|
||||
resource: ['database'],
|
||||
operation: ['insert'],
|
||||
},
|
||||
};
|
||||
|
||||
export const description = updateDisplayOptions(displayOptions, properties);
|
||||
|
||||
export async function execute(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
|
||||
// https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
|
||||
const projectId = this.getNodeParameter('projectId', 0, undefined, {
|
||||
extractValue: true,
|
||||
});
|
||||
const datasetId = this.getNodeParameter('datasetId', 0, undefined, {
|
||||
extractValue: true,
|
||||
});
|
||||
const tableId = this.getNodeParameter('tableId', 0, undefined, {
|
||||
extractValue: true,
|
||||
});
|
||||
|
||||
const options = this.getNodeParameter('options', 0);
|
||||
const dataMode = this.getNodeParameter('dataMode', 0) as string;
|
||||
|
||||
let batchSize = 100;
|
||||
if (options.batchSize) {
|
||||
batchSize = options.batchSize as number;
|
||||
delete options.batchSize;
|
||||
}
|
||||
|
||||
const items = this.getInputData();
|
||||
const length = items.length;
|
||||
|
||||
const returnData: INodeExecutionData[] = [];
|
||||
const rows: IDataObject[] = [];
|
||||
const body: IDataObject = {};
|
||||
|
||||
Object.assign(body, options);
|
||||
if (body.traceId === undefined) {
|
||||
body.traceId = uuid();
|
||||
}
|
||||
|
||||
const schema = (
|
||||
await googleApiRequest.call(
|
||||
this,
|
||||
'GET',
|
||||
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
|
||||
{},
|
||||
)
|
||||
).schema as TableSchema;
|
||||
|
||||
if (schema === undefined) {
|
||||
throw new NodeOperationError(this.getNode(), 'The destination table has no defined schema');
|
||||
}
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
try {
|
||||
const record: IDataObject = {};
|
||||
|
||||
if (dataMode === 'autoMap') {
|
||||
schema.fields.forEach(({ name }) => {
|
||||
record[name] = items[i].json[name];
|
||||
});
|
||||
}
|
||||
|
||||
if (dataMode === 'define') {
|
||||
const fields = this.getNodeParameter('fieldsUi.values', i, []) as IDataObject[];
|
||||
|
||||
fields.forEach(({ fieldId, fieldValue }) => {
|
||||
record[`${fieldId}`] = fieldValue;
|
||||
});
|
||||
}
|
||||
|
||||
rows.push({ json: checkSchema.call(this, schema, record, i) });
|
||||
} catch (error) {
|
||||
if (this.continueOnFail()) {
|
||||
const executionErrorData = this.helpers.constructExecutionMetaData(
|
||||
this.helpers.returnJsonArray({ error: error.message }),
|
||||
{ itemData: { item: i } },
|
||||
);
|
||||
returnData.push(...executionErrorData);
|
||||
continue;
|
||||
}
|
||||
throw new NodeOperationError(this.getNode(), error.message as string, {
|
||||
itemIndex: i,
|
||||
description: error?.description,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < rows.length; i += batchSize) {
|
||||
const batch = rows.slice(i, i + batchSize);
|
||||
body.rows = batch;
|
||||
|
||||
const responseData = await googleApiRequest.call(
|
||||
this,
|
||||
'POST',
|
||||
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
|
||||
body,
|
||||
);
|
||||
|
||||
if (responseData?.insertErrors && !options.skipInvalidRows) {
|
||||
const errors: string[] = [];
|
||||
const failedRows: number[] = [];
|
||||
const stopedRows: number[] = [];
|
||||
|
||||
(responseData.insertErrors as IDataObject[]).forEach((entry) => {
|
||||
const invalidRows = (entry.errors as IDataObject[]).filter(
|
||||
(error) => error.reason !== 'stopped',
|
||||
);
|
||||
if (invalidRows.length) {
|
||||
const entryIndex = (entry.index as number) + i;
|
||||
errors.push(
|
||||
`Row ${entryIndex} failed with error: ${invalidRows
|
||||
.map((error) => error.message)
|
||||
.join(', ')}`,
|
||||
);
|
||||
failedRows.push(entryIndex);
|
||||
} else {
|
||||
const entryIndex = (entry.index as number) + i;
|
||||
stopedRows.push(entryIndex);
|
||||
}
|
||||
});
|
||||
|
||||
if (this.continueOnFail()) {
|
||||
const executionErrorData = this.helpers.constructExecutionMetaData(
|
||||
this.helpers.returnJsonArray({ error: errors.join('\n, ') }),
|
||||
{ itemData: { item: i } },
|
||||
);
|
||||
returnData.push(...executionErrorData);
|
||||
continue;
|
||||
}
|
||||
|
||||
const failedMessage = `Problem inserting item(s) [${failedRows.join(', ')}]`;
|
||||
const stoppedMessage = stopedRows.length
|
||||
? `, nothing was inserted item(s) [${stopedRows.join(', ')}]`
|
||||
: '';
|
||||
throw new NodeOperationError(this.getNode(), `${failedMessage}${stoppedMessage}`, {
|
||||
description: errors.join('\n, '),
|
||||
});
|
||||
}
|
||||
|
||||
const executionData = this.helpers.constructExecutionMetaData(
|
||||
wrapData(responseData as IDataObject[]),
|
||||
{ itemData: { item: 0 } },
|
||||
);
|
||||
|
||||
returnData.push(...executionData);
|
||||
}
|
||||
|
||||
return returnData;
|
||||
}
|
||||
Reference in New Issue
Block a user