mirror of
https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git
synced 2025-12-17 18:12:04 +00:00
feat(editor): Implement Resource Mapper component (#6207)
* ⚡ scaffolding * ⚡ finished scaffolding * ⚡ renamed types * ⚡ updated subtitle * ⚡ renamed functions file, UI updates * ⚡ query parameters fixes, ui updates, refactoring * ⚡ fixes for credentials test, setup for error parsing * ⚡ rlc for schema and table, error handling tweaks * ⚡ delete operation, new options * ⚡ columns loader * ⚡ linter fixes * ⚡ where clauses setup * ⚡ logic for processing where clauses * ⚡ select operation * ⚡ refactoring * ⚡ data mode for insert and update, wip * ⚡ data mapping, insert update, skip on conflict option * ⚡ select columns with spaces fix * ⚡ update operation update, wip * ⚡ finished update operation * ⚡ upsert operation * ⚡ ui fixes * Copy updates. * Copy updates. * ⚡ option to convert empty strings to nulls, schema checks * ⚡ UI requested updates * ⚡ ssh setup WIP * ⚡ fixes, ssh WIP * ⚡ ssh fixes, credentials * ⚡ credentials testing update * ⚡ uncaught error fix * ⚡ clean up * ⚡ address in use fix * ⚡ improved error message * ⚡ tests setup * ⚡ unit tests wip * ⚡ config files clean up * ⚡ utils unit tests * ⚡ refactoring * ⚡ setup for testing operations, tests for deleteTable operation * ⚡ executeQuery and insert operations tests * ⚡ select, update, upsert operations tests * ⚡ runQueries tests setup * ⚡ hint to query * Copy updates. * ⚡ ui fixes * ⚡ clean up * ⚡ error message update * ⚡ ui update * Minor tweaks to query params decription. * feat(Google Sheets Node): Implement Resource mapper in Google Sheets node (#5752) * ✨ Added initial resource mapping support in google sheets node * ✨ Wired mapping API endpoint with node-specific logic for fetching mapping fields * ✨ Implementing mapping fields logic for google sheets * ✨ Updating Google Sheets execute methods to support resource mapper fields * 🚧 Added initial version of `ResourceLocator` component * 👌 Added `update` mode to resource mapper modes * 👌 Addressing PR feedback * 👌 Removing leftover const reference * 👕 Fixing lint errors * ⚡ singlton for conections * ⚡ credentials test fix, clean up * feat(Postgres Node): Add resource mapper to new version of Postgres node (#5814) * ⚡ scaffolding * ⚡ finished scaffolding * ⚡ renamed types * ⚡ updated subtitle * ⚡ renamed functions file, UI updates * ⚡ query parameters fixes, ui updates, refactoring * ⚡ fixes for credentials test, setup for error parsing * ⚡ rlc for schema and table, error handling tweaks * ⚡ delete operation, new options * ⚡ columns loader * ⚡ linter fixes * ⚡ where clauses setup * ⚡ logic for processing where clauses * ⚡ select operation * ⚡ refactoring * ⚡ data mode for insert and update, wip * ⚡ data mapping, insert update, skip on conflict option * ⚡ select columns with spaces fix * ⚡ update operation update, wip * ⚡ finished update operation * ⚡ upsert operation * ⚡ ui fixes * Copy updates. * Copy updates. * ⚡ option to convert empty strings to nulls, schema checks * ⚡ UI requested updates * ⚡ ssh setup WIP * ⚡ fixes, ssh WIP * ⚡ ssh fixes, credentials * ⚡ credentials testing update * ⚡ uncaught error fix * ⚡ clean up * ⚡ address in use fix * ⚡ improved error message * ⚡ tests setup * ⚡ unit tests wip * ⚡ config files clean up * ⚡ utils unit tests * ⚡ refactoring * ⚡ setup for testing operations, tests for deleteTable operation * ⚡ executeQuery and insert operations tests * ⚡ select, update, upsert operations tests * ⚡ runQueries tests setup * ⚡ hint to query * Copy updates. * ⚡ ui fixes * ⚡ clean up * ⚡ error message update * ⚡ ui update * Minor tweaks to query params decription. * ✨ Updated Postgres node to use resource mapper component * ✨ Implemented postgres <-> resource mapper type mapping * ✨ Updated Postgres node execution to use resource mapper fields in v3 * 🔥 Removing unused import --------- Co-authored-by: Michael Kret <michael.k@radency.com> Co-authored-by: Giulio Andreini <g.andreini@gmail.com> * feat(core): Resource editor componend P0 (#5970) * ✨ Added inital value of mapping mode dropdown * ✨ Finished mapping mode selector * ✨ Finished implementing mapping mode selector * ✨ Implemented 'Columns to match on' dropdown * ✨ Implemented `loadOptionsDependOn` support in resource mapper * ✨ Implemented initial version of mapping fields * ✨ Implementing dependant fields watcher in new component setup * ✨ Generating correct resource mapper field types. Added `supportAutoMap` to node specification and UI. Not showing fields with `display=false`. Pre-selecting matching columns if it's the only one * ✨ Handling matching columns correctly in UI * ✨ Saving and loading resourceMapper values in component * ✨ Implemented proper data saving and loading * ✨ ResourceMapper component refactor, fixing value save/load * ✨ Refactoring MatchingColumnSelect component. Updating Sheets node to use single key match and Postgres to use multi key * ✨ Updated Google Sheets node to work with the new UI * ✨ Updating Postgres Node to work with new UI * ✨ Additional loading indicator that shown if there is no mapping mode selector * ✨ Removing hard-coded values, fixing matching columns ordering, refactoring * ✨ Updating field names in nodes * ✨ Fixing minor UI issues * ✨ Implemented matching fields filter logic * ✨ Moving loading label outside of fields list * ✅ Added initial unit tests for resource mapper * ✅ Finished default rendering test * ✅ Test refactoring * ✅ Finished unit tests * 🔨 Updating the way i18n is used in resource mapper components * ✔️ Fixing value to match on logic for postgres node * ✨ Hiding mapping fields when auto-map mode is selected * ✨ Syncing selected mapping mode between components * ✨ Fixing dateTime input rendering and adding update check to Postgres node * ✨ Properly handling database connections. Sending null for empty string values. * 💄 Updated wording in the error message for non-existing rows * ✨ Fixing issues with selected matching values * ✔️ Updating unit tests after matching logic update * ✨ Updating matching columns when new fields are loaded * ✨ Defaulting to null for empty parameter values * ✨ Allowing zero as valid value for number imputs * ✨ Updated list of types that use datepicker as widger * ✨ Using text inputs for time types * ✨ Initial mapping field rework * ✨ Added new component for mapping fields, moved bit of logic from root component to matching selector, fixing some lint errors * ✨ Added tooltip for columns that cannot be deleted * ✨ Saving deleted values in parameter value * ✨ Implemented control to add/remove mapping fields * ✨ Syncing field list with add field dropdown when changing dependent values * ✨ Not showing removed fields in matching columns selector. Updating wording in matching columns selector description * ✨ Implementing disabled states for add/remove all fields options * ✨ Saving removed columns separately, updating copy * ✨ Implemented resource mapper values validation * ✨ Updated validation logic and error input styling * ✨ Validating resource mapper fields when new nodes are added * ✨ Using node field words in validation, refactoring resource mapper component * ✨ Implemented schema syncing and add/remove all fields * ✨ Implemented custom parameter actions * ✨ Implemented loading indicator in parameter options * 🔨 Removing unnecessary constants and vue props * ✨ Handling default values properly * ✨ Fixing validation logic * 👕 Fixing lint errors * ⚡ Fixing type issues * ⚡ Not showing fields by default if `addAllFields` is set to `false` * ✨ Implemented field type validation in resource mapper * ✨ Updated casing in copy, removed all/remove all option from bottom menu * ✨ Added auto mapping mode notice * ✨ Added support for more types in validation * ✨ Added support for enumerated values * ✨ Fixing imports after merging * ✨ Not showing removed fields in matching columns selector. Refactoring validation logic. * 👕 Fixing imports * ✔️ Updating unit tests * ✅ Added resource mapper schema tests * ⚡ Removing `match` from resource mapper field definition, fixing matching columns loading * ⚡ Fixed schema merging * ⚡ update operation return data fix * ⚡ review * 🐛 Added missing import * 💄 Updating parameter actions icon based on the ui review * 💄 Updating word capitalisation in tooltips * 💄 Added empty state to mapping fields list * 💄 Removing asterisk from fields, updating tooltips for matching fields * ⚡ Preventing matching fields from being removed by 'Remove All option' * ⚡ Not showing hidden fields in the `Add field` dropdown * ⚡ Added support for custom matching columns labels * ⚡ query optimization * ⚡ fix * ⚡ Optimizing Postgres node enumeration logic * ⚡ Added empty state for matching columns * ⚡ Only fully loading fields if there is no schema fetched * ⚡ Hiding mapping fields if there is no matching columns available in the schema * ✔️ Fixing minor issues * ✨ Implemented runtime type validation * 🔨 Refactoring validation logic * ✨ Implemented required check, added more custom messages * ✨ Skipping boolean type in required check * Type check improvements * ✨ Only reloading fields if dependent values actually change * ✨ Adding item index to validation error title * ✨ Updating Postgres fetching logic, using resource mapper mode to determine if a field can be deleted * ✨ Resetting field values when adding them via the addAll option * ⚡ Using minor version (2.2) for new Postgres node * ⚡ Implemented proper date validation and type casting * 👕 Consolidating typing * ✅ Added unit tests for type validations * 👌 Addressing front-end review comments * ⚡ More refactoring to address review changes * ⚡ Updating leftover props * ⚡ Added fallback for ISO dates with invalid timezones * Added timestamp to datetime test cases * ⚡ Reseting matching columns if operation changes * ⚡ Not forcing auto-increment fields to be filled in in Postgres node. Handling null values * 💄 Added a custom message for invalid dates * ⚡ Better handling of JSON values * ⚡ Updating codemirror readonly stauts based on component property, handling objects in json validation * Deleting leftover console.log * ⚡ Better time validation * ⚡ Fixing build error after merging * 👕 Fixing lint error * ⚡ Updating node configuration values * ⚡ Handling postgres arrays better * ⚡ Handling SQL array syntax * ⚡ Updating time validation rules to include timezone * ⚡ Sending expressions that resolve to `null` or `undefined` by the resource mapper to delete cell content in Google Sheets * ⚡ Allowing removed fields to be selected for match * ⚡ Updated the query for fetching unique columns and primary keys * ⚡ Optimizing the unique query * ⚡ Setting timezone to all parsed dates * ⚡ Addressing PR review feedback * ⚡ Configuring Sheets node for production, minor vue component update * New cases added to the TypeValidation test. * ✅ Tweaking validation rules for arrays/objects and updating test cases --------- Co-authored-by: Michael Kret <michael.k@radency.com> Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
This commit is contained in:
committed by
GitHub
parent
5ae1124106
commit
04cfa548af
@@ -11,7 +11,7 @@ export class Postgres extends VersionedNodeType {
|
||||
name: 'postgres',
|
||||
icon: 'file:postgres.svg',
|
||||
group: ['input'],
|
||||
defaultVersion: 2.1,
|
||||
defaultVersion: 2.2,
|
||||
description: 'Get, add and update data in Postgres',
|
||||
};
|
||||
|
||||
@@ -19,6 +19,7 @@ export class Postgres extends VersionedNodeType {
|
||||
1: new PostgresV1(baseDescription),
|
||||
2: new PostgresV2(baseDescription),
|
||||
2.1: new PostgresV2(baseDescription),
|
||||
2.2: new PostgresV2(baseDescription),
|
||||
};
|
||||
|
||||
super(nodeVersions, baseDescription);
|
||||
|
||||
@@ -9,7 +9,7 @@ import type {
|
||||
import { router } from './actions/router';
|
||||
|
||||
import { versionDescription } from './actions/versionDescription';
|
||||
import { credentialTest, listSearch, loadOptions } from './methods';
|
||||
import { credentialTest, listSearch, loadOptions, resourceMapping } from './methods';
|
||||
|
||||
export class PostgresV2 implements INodeType {
|
||||
description: INodeTypeDescription;
|
||||
@@ -21,7 +21,7 @@ export class PostgresV2 implements INodeType {
|
||||
};
|
||||
}
|
||||
|
||||
methods = { credentialTest, listSearch, loadOptions };
|
||||
methods = { credentialTest, listSearch, loadOptions, resourceMapping };
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
return router.call(this);
|
||||
|
||||
@@ -40,6 +40,11 @@ const properties: INodeProperties[] = [
|
||||
default: 'autoMapInputData',
|
||||
description:
|
||||
'Whether to map node input properties and the table data automatically or manually',
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: `
|
||||
@@ -51,6 +56,7 @@ const properties: INodeProperties[] = [
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['autoMapInputData'],
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -66,6 +72,7 @@ const properties: INodeProperties[] = [
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['defineBelow'],
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
default: {},
|
||||
@@ -97,6 +104,35 @@ const properties: INodeProperties[] = [
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
displayName: 'Columns',
|
||||
name: 'columns',
|
||||
type: 'resourceMapper',
|
||||
default: {
|
||||
mappingMode: 'defineBelow',
|
||||
value: null,
|
||||
},
|
||||
noDataExpression: true,
|
||||
required: true,
|
||||
typeOptions: {
|
||||
loadOptionsDependsOn: ['table.value', 'operation'],
|
||||
resourceMapper: {
|
||||
resourceMapperMethod: 'getMappingColumns',
|
||||
mode: 'add',
|
||||
fieldWords: {
|
||||
singular: 'column',
|
||||
plural: 'columns',
|
||||
},
|
||||
addAllFields: true,
|
||||
multiKeyMatch: true,
|
||||
},
|
||||
},
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [2.2],
|
||||
},
|
||||
},
|
||||
},
|
||||
optionsCollection,
|
||||
];
|
||||
|
||||
@@ -142,7 +178,11 @@ export async function execute(
|
||||
let query = `INSERT INTO $1:name.$2:name($3:name) VALUES($3:csv)${onConflict}`;
|
||||
let values: QueryValues = [schema, table];
|
||||
|
||||
const dataMode = this.getNodeParameter('dataMode', i) as string;
|
||||
const nodeVersion = this.getNode().typeVersion;
|
||||
const dataMode =
|
||||
nodeVersion < 2.2
|
||||
? (this.getNodeParameter('dataMode', i) as string)
|
||||
: (this.getNodeParameter('columns.mappingMode', i) as string);
|
||||
|
||||
let item: IDataObject = {};
|
||||
|
||||
@@ -151,10 +191,17 @@ export async function execute(
|
||||
}
|
||||
|
||||
if (dataMode === 'defineBelow') {
|
||||
const valuesToSend = (this.getNodeParameter('valuesToSend', i, []) as IDataObject)
|
||||
.values as IDataObject[];
|
||||
const valuesToSend =
|
||||
nodeVersion < 2.2
|
||||
? ((this.getNodeParameter('valuesToSend', i, []) as IDataObject).values as IDataObject[])
|
||||
: ((this.getNodeParameter('columns.values', i, []) as IDataObject)
|
||||
.values as IDataObject[]);
|
||||
|
||||
item = prepareItem(valuesToSend);
|
||||
if (nodeVersion < 2.2) {
|
||||
item = prepareItem(valuesToSend);
|
||||
} else {
|
||||
item = this.getNodeParameter('columns.value', i) as IDataObject;
|
||||
}
|
||||
}
|
||||
|
||||
const tableSchema = await getTableSchema(db, schema, table);
|
||||
|
||||
@@ -14,6 +14,7 @@ import type {
|
||||
import {
|
||||
addReturning,
|
||||
checkItemAgainstSchema,
|
||||
doesRowExist,
|
||||
getTableSchema,
|
||||
prepareItem,
|
||||
replaceEmptyStringsByNulls,
|
||||
@@ -41,6 +42,11 @@ const properties: INodeProperties[] = [
|
||||
default: 'autoMapInputData',
|
||||
description:
|
||||
'Whether to map node input properties and the table data automatically or manually',
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: `
|
||||
@@ -52,6 +58,7 @@ const properties: INodeProperties[] = [
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['autoMapInputData'],
|
||||
'@version': [2],
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -69,6 +76,11 @@ const properties: INodeProperties[] = [
|
||||
},
|
||||
default: '',
|
||||
hint: 'The column that identifies the row(s) to modify',
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Value of Column to Match On',
|
||||
@@ -80,6 +92,7 @@ const properties: INodeProperties[] = [
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['defineBelow'],
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -95,6 +108,7 @@ const properties: INodeProperties[] = [
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['defineBelow'],
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
default: {},
|
||||
@@ -126,6 +140,35 @@ const properties: INodeProperties[] = [
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
displayName: 'Columns',
|
||||
name: 'columns',
|
||||
type: 'resourceMapper',
|
||||
noDataExpression: true,
|
||||
default: {
|
||||
mappingMode: 'defineBelow',
|
||||
value: null,
|
||||
},
|
||||
required: true,
|
||||
typeOptions: {
|
||||
loadOptionsDependsOn: ['table.value', 'operation'],
|
||||
resourceMapper: {
|
||||
resourceMapperMethod: 'getMappingColumns',
|
||||
mode: 'update',
|
||||
fieldWords: {
|
||||
singular: 'column',
|
||||
plural: 'columns',
|
||||
},
|
||||
addAllFields: true,
|
||||
multiKeyMatch: true,
|
||||
},
|
||||
},
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [2.2],
|
||||
},
|
||||
},
|
||||
},
|
||||
optionsCollection,
|
||||
];
|
||||
|
||||
@@ -161,32 +204,80 @@ export async function execute(
|
||||
extractValue: true,
|
||||
}) as string;
|
||||
|
||||
const columnToMatchOn = this.getNodeParameter('columnToMatchOn', i) as string;
|
||||
const nodeVersion = this.getNode().typeVersion;
|
||||
const columnsToMatchOn: string[] =
|
||||
nodeVersion < 2.2
|
||||
? [this.getNodeParameter('columnToMatchOn', i) as string]
|
||||
: (this.getNodeParameter('columns.matchingColumns', i) as string[]);
|
||||
|
||||
const dataMode = this.getNodeParameter('dataMode', i) as string;
|
||||
const dataMode =
|
||||
nodeVersion < 2.2
|
||||
? (this.getNodeParameter('dataMode', i) as string)
|
||||
: (this.getNodeParameter('columns.mappingMode', i) as string);
|
||||
|
||||
let item: IDataObject = {};
|
||||
let valueToMatchOn: string | IDataObject = '';
|
||||
|
||||
if (dataMode === 'autoMapInputData') {
|
||||
item = items[i].json;
|
||||
valueToMatchOn = item[columnToMatchOn] as string;
|
||||
}
|
||||
|
||||
if (dataMode === 'defineBelow') {
|
||||
const valuesToSend = (this.getNodeParameter('valuesToSend', i, []) as IDataObject)
|
||||
.values as IDataObject[];
|
||||
|
||||
item = prepareItem(valuesToSend);
|
||||
|
||||
if (nodeVersion < 2.2) {
|
||||
valueToMatchOn = this.getNodeParameter('valueToMatchOn', i) as string;
|
||||
}
|
||||
|
||||
if (!item[columnToMatchOn] && dataMode === 'autoMapInputData') {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
"Column to match on not found in input item. Add a column to match on or set the 'Data Mode' to 'Define Below' to define the value to match on.",
|
||||
);
|
||||
if (dataMode === 'autoMapInputData') {
|
||||
item = items[i].json;
|
||||
if (nodeVersion < 2.2) {
|
||||
valueToMatchOn = item[columnsToMatchOn[0]] as string;
|
||||
}
|
||||
}
|
||||
|
||||
if (dataMode === 'defineBelow') {
|
||||
const valuesToSend =
|
||||
nodeVersion < 2.2
|
||||
? ((this.getNodeParameter('valuesToSend', i, []) as IDataObject).values as IDataObject[])
|
||||
: ((this.getNodeParameter('columns.values', i, []) as IDataObject)
|
||||
.values as IDataObject[]);
|
||||
|
||||
if (nodeVersion < 2.2) {
|
||||
item = prepareItem(valuesToSend);
|
||||
item[columnsToMatchOn[0]] = this.getNodeParameter('valueToMatchOn', i) as string;
|
||||
} else {
|
||||
item = this.getNodeParameter('columns.value', i) as IDataObject;
|
||||
}
|
||||
}
|
||||
|
||||
const matchValues: string[] = [];
|
||||
if (nodeVersion < 2.2) {
|
||||
if (!item[columnsToMatchOn[0]] && dataMode === 'autoMapInputData') {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
"Column to match on not found in input item. Add a column to match on or set the 'Data Mode' to 'Define Below' to define the value to match on.",
|
||||
);
|
||||
}
|
||||
matchValues.push(valueToMatchOn);
|
||||
matchValues.push(columnsToMatchOn[0]);
|
||||
} else {
|
||||
columnsToMatchOn.forEach((column) => {
|
||||
matchValues.push(column);
|
||||
matchValues.push(item[column] as string);
|
||||
});
|
||||
const rowExists = await doesRowExist(db, schema, table, matchValues);
|
||||
if (!rowExists) {
|
||||
const descriptionValues: string[] = [];
|
||||
matchValues.forEach((val, index) => {
|
||||
if (index % 2 === 0) {
|
||||
descriptionValues.push(`${matchValues[index]}=${matchValues[index + 1]}`);
|
||||
}
|
||||
});
|
||||
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
"The row you are trying to update doesn't exist",
|
||||
{
|
||||
description: `No rows matching the provided values (${descriptionValues.join(
|
||||
', ',
|
||||
)}) were found in the table "${table}".`,
|
||||
itemIndex: i,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const tableSchema = await getTableSchema(db, schema, table);
|
||||
@@ -197,11 +288,22 @@ export async function execute(
|
||||
|
||||
let valuesLength = values.length + 1;
|
||||
|
||||
const condition = `$${valuesLength}:name = $${valuesLength + 1}`;
|
||||
valuesLength = valuesLength + 2;
|
||||
values.push(columnToMatchOn, valueToMatchOn);
|
||||
let condition = '';
|
||||
if (nodeVersion < 2.2) {
|
||||
condition = `$${valuesLength}:name = $${valuesLength + 1}`;
|
||||
valuesLength = valuesLength + 2;
|
||||
values.push(columnsToMatchOn[0], valueToMatchOn);
|
||||
} else {
|
||||
const conditions: string[] = [];
|
||||
for (const column of columnsToMatchOn) {
|
||||
conditions.push(`$${valuesLength}:name = $${valuesLength + 1}`);
|
||||
valuesLength = valuesLength + 2;
|
||||
values.push(column, item[column] as string);
|
||||
}
|
||||
condition = conditions.join(' AND ');
|
||||
}
|
||||
|
||||
const updateColumns = Object.keys(item).filter((column) => column !== columnToMatchOn);
|
||||
const updateColumns = Object.keys(item).filter((column) => !columnsToMatchOn.includes(column));
|
||||
|
||||
if (!Object.keys(updateColumns).length) {
|
||||
throw new NodeOperationError(
|
||||
@@ -227,5 +329,6 @@ export async function execute(
|
||||
queries.push({ query, values });
|
||||
}
|
||||
|
||||
return runQueries(queries, items, nodeOptions);
|
||||
const results = await runQueries(queries, items, nodeOptions);
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -41,6 +41,11 @@ const properties: INodeProperties[] = [
|
||||
default: 'autoMapInputData',
|
||||
description:
|
||||
'Whether to map node input properties and the table data automatically or manually',
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: `
|
||||
@@ -52,12 +57,13 @@ const properties: INodeProperties[] = [
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['autoMapInputData'],
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased, n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options
|
||||
displayName: 'Column to Match On',
|
||||
displayName: 'Unique Column',
|
||||
name: 'columnToMatchOn',
|
||||
type: 'options',
|
||||
required: true,
|
||||
@@ -69,9 +75,14 @@ const properties: INodeProperties[] = [
|
||||
},
|
||||
default: '',
|
||||
hint: "Used to find the correct row(s) to update. Doesn't get changed. Has to be unique.",
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Value of Column to Match On',
|
||||
displayName: 'Value of Unique Column',
|
||||
name: 'valueToMatchOn',
|
||||
type: 'string',
|
||||
default: '',
|
||||
@@ -80,6 +91,7 @@ const properties: INodeProperties[] = [
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['defineBelow'],
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -95,6 +107,7 @@ const properties: INodeProperties[] = [
|
||||
displayOptions: {
|
||||
show: {
|
||||
dataMode: ['defineBelow'],
|
||||
'@version': [2, 2.1],
|
||||
},
|
||||
},
|
||||
default: {},
|
||||
@@ -126,6 +139,35 @@ const properties: INodeProperties[] = [
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
displayName: 'Columns',
|
||||
name: 'columns',
|
||||
type: 'resourceMapper',
|
||||
noDataExpression: true,
|
||||
default: {
|
||||
mappingMode: 'defineBelow',
|
||||
value: null,
|
||||
},
|
||||
required: true,
|
||||
typeOptions: {
|
||||
loadOptionsDependsOn: ['table.value', 'operation'],
|
||||
resourceMapper: {
|
||||
resourceMapperMethod: 'getMappingColumns',
|
||||
mode: 'upsert',
|
||||
fieldWords: {
|
||||
singular: 'column',
|
||||
plural: 'columns',
|
||||
},
|
||||
addAllFields: true,
|
||||
multiKeyMatch: true,
|
||||
},
|
||||
},
|
||||
displayOptions: {
|
||||
show: {
|
||||
'@version': [2.2],
|
||||
},
|
||||
},
|
||||
},
|
||||
optionsCollection,
|
||||
];
|
||||
|
||||
@@ -161,9 +203,16 @@ export async function execute(
|
||||
extractValue: true,
|
||||
}) as string;
|
||||
|
||||
const columnToMatchOn = this.getNodeParameter('columnToMatchOn', i) as string;
|
||||
const nodeVersion = this.getNode().typeVersion;
|
||||
const columnsToMatchOn: string[] =
|
||||
nodeVersion < 2.2
|
||||
? [this.getNodeParameter('columnToMatchOn', i) as string]
|
||||
: (this.getNodeParameter('columns.matchingColumns', i) as string[]);
|
||||
|
||||
const dataMode = this.getNodeParameter('dataMode', i) as string;
|
||||
const dataMode =
|
||||
nodeVersion < 2.2
|
||||
? (this.getNodeParameter('dataMode', i) as string)
|
||||
: (this.getNodeParameter('columns.mappingMode', i) as string);
|
||||
|
||||
let item: IDataObject = {};
|
||||
|
||||
@@ -172,22 +221,28 @@ export async function execute(
|
||||
}
|
||||
|
||||
if (dataMode === 'defineBelow') {
|
||||
const valuesToSend = (this.getNodeParameter('valuesToSend', i, []) as IDataObject)
|
||||
.values as IDataObject[];
|
||||
const valuesToSend =
|
||||
nodeVersion < 2.2
|
||||
? ((this.getNodeParameter('valuesToSend', i, []) as IDataObject).values as IDataObject[])
|
||||
: ((this.getNodeParameter('columns.values', i, []) as IDataObject)
|
||||
.values as IDataObject[]);
|
||||
|
||||
item = prepareItem(valuesToSend);
|
||||
|
||||
item[columnToMatchOn] = this.getNodeParameter('valueToMatchOn', i) as string;
|
||||
if (nodeVersion < 2.2) {
|
||||
item = prepareItem(valuesToSend);
|
||||
item[columnsToMatchOn[0]] = this.getNodeParameter('valueToMatchOn', i) as string;
|
||||
} else {
|
||||
item = this.getNodeParameter('columns.value', i) as IDataObject;
|
||||
}
|
||||
}
|
||||
|
||||
if (!item[columnToMatchOn]) {
|
||||
if (!item[columnsToMatchOn[0]]) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
"Column to match on not found in input item. Add a column to match on or set the 'Data Mode' to 'Define Below' to define the value to match on.",
|
||||
);
|
||||
}
|
||||
|
||||
if (item[columnToMatchOn] && Object.keys(item).length === 1) {
|
||||
if (item[columnsToMatchOn[0]] && Object.keys(item).length === 1) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
"Add values to update or insert to the input item or set the 'Data Mode' to 'Define Below' to define the values to insert or update.",
|
||||
@@ -201,16 +256,19 @@ export async function execute(
|
||||
let values: QueryValues = [schema, table];
|
||||
|
||||
let valuesLength = values.length + 1;
|
||||
const onConflict = ` ON CONFLICT ($${valuesLength}:name) DO UPDATE `;
|
||||
valuesLength = valuesLength + 1;
|
||||
values.push(columnToMatchOn);
|
||||
const conflictColumns: string[] = [];
|
||||
columnsToMatchOn.forEach((column) => {
|
||||
conflictColumns.push(`$${valuesLength}:name`);
|
||||
valuesLength = valuesLength + 1;
|
||||
values.push(column);
|
||||
});
|
||||
const onConflict = ` ON CONFLICT (${conflictColumns.join(',')}) DO UPDATE `;
|
||||
|
||||
const insertQuery = `INSERT INTO $1:name.$2:name($${valuesLength}:name) VALUES($${valuesLength}:csv)${onConflict}`;
|
||||
valuesLength = valuesLength + 1;
|
||||
values.push(item);
|
||||
|
||||
const updateColumns = Object.keys(item).filter((column) => column !== columnToMatchOn);
|
||||
|
||||
const updateColumns = Object.keys(item).filter((column) => !columnsToMatchOn.includes(column));
|
||||
const updates: string[] = [];
|
||||
|
||||
for (const column of updateColumns) {
|
||||
|
||||
@@ -8,7 +8,7 @@ export const versionDescription: INodeTypeDescription = {
|
||||
name: 'postgres',
|
||||
icon: 'file:postgres.svg',
|
||||
group: ['input'],
|
||||
version: [2, 2.1],
|
||||
version: [2, 2.1, 2.2],
|
||||
subtitle: '={{ $parameter["operation"] }}',
|
||||
description: 'Get, add and update data in Postgres',
|
||||
defaults: {
|
||||
|
||||
@@ -16,7 +16,17 @@ export type QueryWithValues = { query: string; values?: QueryValues };
|
||||
|
||||
export type WhereClause = { column: string; condition: string; value: string | number };
|
||||
export type SortRule = { column: string; direction: string };
|
||||
export type ColumnInfo = { column_name: string; data_type: string; is_nullable: string };
|
||||
export type ColumnInfo = {
|
||||
column_name: string;
|
||||
data_type: string;
|
||||
is_nullable: string;
|
||||
udt_name: string;
|
||||
column_default?: string;
|
||||
};
|
||||
export type EnumInfo = {
|
||||
typname: string;
|
||||
enumlabel: string;
|
||||
};
|
||||
|
||||
export type PgpClient = pgPromise.IMain<{}, pg.IClient>;
|
||||
export type PgpDatabase = pgPromise.IDatabase<{}, pg.IClient>;
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { IDataObject, INode, INodeExecutionData } from 'n8n-workflow';
|
||||
import type { IDataObject, INode, INodeExecutionData, INodePropertyOptions } from 'n8n-workflow';
|
||||
import { NodeOperationError } from 'n8n-workflow';
|
||||
|
||||
import type {
|
||||
ColumnInfo,
|
||||
ConstructExecutionMetaData,
|
||||
EnumInfo,
|
||||
PgpClient,
|
||||
PgpDatabase,
|
||||
QueryMode,
|
||||
@@ -13,6 +14,8 @@ import type {
|
||||
WhereClause,
|
||||
} from './interfaces';
|
||||
|
||||
const ENUM_VALUES_REGEX = /\{(.+?)\}/gm;
|
||||
|
||||
export function wrapData(data: IDataObject | IDataObject[]): INodeExecutionData[] {
|
||||
if (!Array.isArray(data)) {
|
||||
return [{ json: data }];
|
||||
@@ -324,13 +327,60 @@ export async function getTableSchema(
|
||||
table: string,
|
||||
): Promise<ColumnInfo[]> {
|
||||
const columns = await db.any(
|
||||
'SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2',
|
||||
'SELECT column_name, data_type, is_nullable, udt_name, column_default FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2',
|
||||
[schema, table],
|
||||
);
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
export async function uniqueColumns(db: PgpDatabase, table: string) {
|
||||
// Using the modified query from https://wiki.postgresql.org/wiki/Retrieve_primary_key_columns
|
||||
const unique = await db.any(
|
||||
`
|
||||
SELECT DISTINCT a.attname
|
||||
FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
|
||||
WHERE i.indrelid = quote_ident($1)::regclass
|
||||
AND (i.indisprimary OR i.indisunique);
|
||||
`,
|
||||
[table],
|
||||
);
|
||||
return unique as IDataObject[];
|
||||
}
|
||||
|
||||
export async function getEnums(db: PgpDatabase): Promise<EnumInfo[]> {
|
||||
const enumsData = await db.any(
|
||||
'SELECT pg_type.typname, pg_enum.enumlabel FROM pg_type JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid;',
|
||||
);
|
||||
return enumsData as EnumInfo[];
|
||||
}
|
||||
|
||||
export function getEnumValues(enumInfo: EnumInfo[], enumName: string): INodePropertyOptions[] {
|
||||
return enumInfo.reduce((acc, current) => {
|
||||
if (current.typname === enumName) {
|
||||
acc.push({ name: current.enumlabel, value: current.enumlabel });
|
||||
}
|
||||
return acc;
|
||||
}, [] as INodePropertyOptions[]);
|
||||
}
|
||||
|
||||
export async function doesRowExist(
|
||||
db: PgpDatabase,
|
||||
schema: string,
|
||||
table: string,
|
||||
values: string[],
|
||||
): Promise<boolean> {
|
||||
const where = [];
|
||||
for (let i = 3; i < 3 + values.length; i += 2) {
|
||||
where.push(`$${i}:name=$${i + 1}`);
|
||||
}
|
||||
const exists = await db.any(
|
||||
`SELECT EXISTS(SELECT 1 FROM $1:name.$2:name WHERE ${where.join(' AND ')})`,
|
||||
[schema, table, ...values],
|
||||
);
|
||||
return exists[0].exists;
|
||||
}
|
||||
|
||||
export function checkItemAgainstSchema(
|
||||
node: INode,
|
||||
item: IDataObject,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export * as credentialTest from './credentialTest';
|
||||
export * as listSearch from './listSearch';
|
||||
export * as loadOptions from './loadOptions';
|
||||
export * as resourceMapping from './resourceMapping';
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import type { ILoadOptionsFunctions, ResourceMapperFields, FieldType } from 'n8n-workflow';
|
||||
import { getEnumValues, getEnums, getTableSchema, uniqueColumns } from '../helpers/utils';
|
||||
import { configurePostgres } from '../transport';
|
||||
|
||||
const fieldTypeMapping: Partial<Record<FieldType, string[]>> = {
|
||||
string: ['text', 'varchar', 'character varying', 'character', 'char'],
|
||||
number: [
|
||||
'integer',
|
||||
'smallint',
|
||||
'bigint',
|
||||
'decimal',
|
||||
'numeric',
|
||||
'real',
|
||||
'double precision',
|
||||
'smallserial',
|
||||
'serial',
|
||||
'bigserial',
|
||||
],
|
||||
boolean: ['boolean'],
|
||||
dateTime: [
|
||||
'timestamp',
|
||||
'date',
|
||||
'timestampz',
|
||||
'timestamp without time zone',
|
||||
'timestamp with time zone',
|
||||
],
|
||||
time: ['time', 'time without time zone', 'time with time zone'],
|
||||
object: ['json', 'jsonb'],
|
||||
options: ['enum', 'USER-DEFINED'],
|
||||
array: ['ARRAY'],
|
||||
};
|
||||
|
||||
function mapPostgresType(postgresType: string): FieldType {
|
||||
let mappedType: FieldType = 'string';
|
||||
|
||||
for (const t of Object.keys(fieldTypeMapping)) {
|
||||
const postgresTypes = fieldTypeMapping[t as FieldType];
|
||||
if (postgresTypes?.includes(postgresType)) {
|
||||
mappedType = t as FieldType;
|
||||
}
|
||||
}
|
||||
return mappedType;
|
||||
}
|
||||
|
||||
export async function getMappingColumns(
|
||||
this: ILoadOptionsFunctions,
|
||||
): Promise<ResourceMapperFields> {
|
||||
const credentials = await this.getCredentials('postgres');
|
||||
|
||||
const { db } = await configurePostgres(credentials);
|
||||
|
||||
const schema = this.getNodeParameter('schema', 0, {
|
||||
extractValue: true,
|
||||
}) as string;
|
||||
|
||||
const table = this.getNodeParameter('table', 0, {
|
||||
extractValue: true,
|
||||
}) as string;
|
||||
|
||||
const operation = this.getNodeParameter('operation', 0, {
|
||||
extractValue: true,
|
||||
}) as string;
|
||||
|
||||
try {
|
||||
const columns = await getTableSchema(db, schema, table);
|
||||
const unique = operation === 'upsert' ? await uniqueColumns(db, table) : [];
|
||||
const enumInfo = await getEnums(db);
|
||||
const fields = await Promise.all(
|
||||
columns.map(async (col) => {
|
||||
const canBeUsedToMatch =
|
||||
operation === 'upsert' ? unique.some((u) => u.attname === col.column_name) : true;
|
||||
const type = mapPostgresType(col.data_type);
|
||||
const options = type === 'options' ? getEnumValues(enumInfo, col.udt_name) : undefined;
|
||||
const isAutoIncrement = col.column_default?.startsWith('nextval');
|
||||
return {
|
||||
id: col.column_name,
|
||||
displayName: col.column_name,
|
||||
required: col.is_nullable !== 'YES' && !isAutoIncrement,
|
||||
defaultMatch: col.column_name === 'id',
|
||||
display: true,
|
||||
type,
|
||||
canBeUsedToMatch,
|
||||
options,
|
||||
};
|
||||
}),
|
||||
);
|
||||
return { fields };
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user