feat(Data Table Node): Add Data Table Node (no-changelog) (#18556)

This commit is contained in:
Charlie Kolb
2025-08-25 15:57:55 +02:00
committed by GitHub
parent e6b6678128
commit 1777803e26
16 changed files with 741 additions and 4 deletions

View File

@@ -50,7 +50,7 @@ describe('DataStoreProxyService', () => {
id: PROJECT_ID, id: PROJECT_ID,
}); });
node = mock<INode>({ node = mock<INode>({
type: 'n8n-nodes-base.dataStore', type: 'n8n-nodes-base.dataTable',
}); });
ownershipServiceMock.getWorkflowProjectCached.mockResolvedValueOnce(project); ownershipServiceMock.getWorkflowProjectCached.mockResolvedValueOnce(project);

View File

@@ -34,8 +34,8 @@ export class DataStoreProxyService implements DataStoreProxyProvider {
} }
private validateRequest(node: INode) { private validateRequest(node: INode) {
if (node.type !== 'n8n-nodes-base.dataStore') { if (node.type !== 'n8n-nodes-base.dataTable') {
throw new Error('This proxy is only available for data store nodes'); throw new Error('This proxy is only available for data table nodes');
} }
} }

View File

@@ -224,7 +224,7 @@ export const SLACK_TRIGGER_NODE_TYPE = 'n8n-nodes-base.slackTrigger';
export const TELEGRAM_TRIGGER_NODE_TYPE = 'n8n-nodes-base.telegramTrigger'; export const TELEGRAM_TRIGGER_NODE_TYPE = 'n8n-nodes-base.telegramTrigger';
export const FACEBOOK_LEAD_ADS_TRIGGER_NODE_TYPE = 'n8n-nodes-base.facebookLeadAdsTrigger'; export const FACEBOOK_LEAD_ADS_TRIGGER_NODE_TYPE = 'n8n-nodes-base.facebookLeadAdsTrigger';
export const RESPOND_TO_WEBHOOK_NODE_TYPE = 'n8n-nodes-base.respondToWebhook'; export const RESPOND_TO_WEBHOOK_NODE_TYPE = 'n8n-nodes-base.respondToWebhook';
export const DATA_STORE_NODE_TYPE = 'n8n-nodes-base.dataStore'; export const DATA_STORE_NODE_TYPE = 'n8n-nodes-base.dataTable';
export const CREDENTIAL_ONLY_NODE_PREFIX = 'n8n-creds-base'; export const CREDENTIAL_ONLY_NODE_PREFIX = 'n8n-creds-base';
export const CREDENTIAL_ONLY_HTTP_NODE_VERSION = 4.1; export const CREDENTIAL_ONLY_HTTP_NODE_VERSION = 4.1;

View File

@@ -0,0 +1,18 @@
{
"node": "n8n-nodes-base.dataTable",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"details": "Data Table",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.datatable/"
}
]
},
"alias": ["data", "table", "knowledge"],
"subcategories": {
"Core Nodes": ["Data Transformation"]
}
}

View File

@@ -0,0 +1,60 @@
import type { IExecuteFunctions, INodeType, INodeTypeDescription } from 'n8n-workflow';
import { NodeConnectionTypes } from 'n8n-workflow';
import { router } from './actions/router';
import * as row from './actions/row/Row.resource';
import { getDataTableColumns, getDataTables, tableSearch } from './common/methods';
export class DataTable implements INodeType {
description: INodeTypeDescription = {
displayName: 'Data Table',
name: 'dataTable',
icon: 'fa:table',
iconColor: 'orange',
group: ['transform'],
version: 1,
subtitle: '={{$parameter["action"]}}',
description: 'Save data across workflow executions in a table',
defaults: {
name: 'Data Table',
},
usableAsTool: true,
// We have custom logic in the frontend to ignore `hidden` for this
// particular node type if the data table module is enabled
hidden: true,
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
properties: [
{
displayName: 'Resource',
name: 'resource',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Row',
value: 'row',
},
],
default: 'row',
},
...row.description,
],
};
methods = {
listSearch: {
tableSearch,
},
loadOptions: {
getDataTableColumns,
},
resourceMapping: {
getDataTables,
},
};
async execute(this: IExecuteFunctions) {
return await router.call(this);
}
}

View File

@@ -0,0 +1,52 @@
import type { IExecuteFunctions, INodeExecutionData, AllEntities } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import * as row from './row/Row.resource';
type DataTableNodeType = AllEntities<{ row: 'insert' | 'get' }>;
export async function router(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const operationResult: INodeExecutionData[] = [];
let responseData: INodeExecutionData[] = [];
const items = this.getInputData();
const resource = this.getNodeParameter('resource', 0);
const operation = this.getNodeParameter('operation', 0);
const dataTableNodeData = {
resource,
operation,
} as DataTableNodeType;
for (let i = 0; i < items.length; i++) {
try {
switch (dataTableNodeData.resource) {
case 'row':
responseData = await row[dataTableNodeData.operation].execute.call(this, i);
break;
default:
throw new NodeOperationError(
this.getNode(),
`The resource "${resource}" is not supported!`,
);
}
const executionData = this.helpers.constructExecutionMetaData(responseData, {
itemData: { item: i },
});
operationResult.push.apply(operationResult, executionData);
} catch (error) {
if (this.continueOnFail()) {
operationResult.push({
json: this.getInputData(i)[0].json,
error: error as NodeOperationError,
});
} else {
throw error;
}
}
}
return [operationResult];
}

View File

@@ -0,0 +1,81 @@
import type { INodeProperties } from 'n8n-workflow';
import * as get from './get.operation';
import * as insert from './insert.operation';
import { DATA_TABLE_ID_FIELD } from '../../common/fields';
export { insert, get };
export const description: INodeProperties[] = [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
displayOptions: {
show: {
resource: ['row'],
},
},
options: [
// {
// name: 'Create or Update',
// value: 'upsert',
// description: 'Create a new record, or update the current one if it already exists (upsert)',
// action: 'Create or update a row',
// },
// {
// name: 'Delete',
// value: 'delete',
// description: 'Delete a row',
// action: 'Delete a row',
// },
{
name: 'Get',
value: get.FIELD,
description: 'Get row(s)',
action: 'Get row(s)',
},
// {
// name: 'Get Many',
// value: 'getAll',
// description: 'Get many rows',
// action: 'Get many rows',
// },
{
name: 'Insert',
value: insert.FIELD,
description: 'Insert a new row',
action: 'Insert row',
},
],
default: 'insert',
},
{
displayName: 'Data Store',
name: DATA_TABLE_ID_FIELD,
type: 'resourceLocator',
default: { mode: 'list', value: '' },
required: true,
modes: [
{
displayName: 'From List',
name: 'list',
type: 'list',
typeOptions: {
searchListMethod: 'tableSearch',
searchable: true,
},
},
{
displayName: 'ID',
name: 'id',
type: 'string',
},
],
displayOptions: { show: { resource: ['row'] } },
},
...insert.description,
...get.description,
];

View File

@@ -0,0 +1,50 @@
import type {
IDisplayOptions,
IExecuteFunctions,
INodeExecutionData,
INodeProperties,
} from 'n8n-workflow';
import { getSelectFields, getSelectFilter } from '../../common/selectMany';
import { getDataTableProxyExecute } from '../../common/utils';
export const FIELD: string = 'get';
const displayOptions: IDisplayOptions = {
show: {
resource: ['row'],
operation: [FIELD],
},
};
export const description: INodeProperties[] = [...getSelectFields(displayOptions)];
export async function execute(
this: IExecuteFunctions,
index: number,
): Promise<INodeExecutionData[]> {
const dataStoreProxy = await getDataTableProxyExecute(this, index);
let take = 1000;
const result: INodeExecutionData[] = [];
const filter = getSelectFilter(this, index);
do {
const response = await dataStoreProxy.getManyRowsAndCount({
skip: result.length,
take,
filter,
});
const data = response.data.map((json) => ({ json }));
// Optimize common path of <1000 results
if (response.count === response.data.length) {
return data;
}
result.push.apply(result, data);
take = Math.min(take, response.count - result.length);
} while (take > 0);
return result;
}

View File

@@ -0,0 +1,51 @@
import type {
IDisplayOptions,
IDataObject,
IExecuteFunctions,
INodeExecutionData,
INodeProperties,
} from 'n8n-workflow';
import { COLUMNS } from '../../common/fields';
import { dataObjectToApiInput, getDataTableProxyExecute } from '../../common/utils';
export const FIELD: string = 'insert';
const displayOptions: IDisplayOptions = {
show: {
resource: ['row'],
operation: [FIELD],
},
};
export const description: INodeProperties[] = [
{
...COLUMNS,
displayOptions,
},
];
export async function execute(
this: IExecuteFunctions,
index: number,
): Promise<INodeExecutionData[]> {
const items = this.getInputData();
const dataStoreProxy = await getDataTableProxyExecute(this, index);
const dataMode = this.getNodeParameter('columns.mappingMode', index) as string;
let data: IDataObject;
if (dataMode === 'autoMapInputData') {
data = items[index].json;
} else {
const fields = this.getNodeParameter('columns.value', index, {}) as IDataObject;
data = fields;
}
const rows = dataObjectToApiInput(data, this.getNode(), index);
const insertedRowIds = await dataStoreProxy.insertRows([rows]);
return insertedRowIds.map((x) => ({ json: { id: x } }));
}

View File

@@ -0,0 +1,12 @@
import type { DataStoreColumnJsType } from 'n8n-workflow';
export const ANY_FILTER = 'anyFilter';
export const ALL_FILTERS = 'allFilters';
export type FilterType = typeof ANY_FILTER | typeof ALL_FILTERS;
export type FieldEntry = {
keyName: string;
condition: 'eq' | 'neq';
keyValue: DataStoreColumnJsType;
};

View File

@@ -0,0 +1,28 @@
import type { INodeProperties } from 'n8n-workflow';
export const DATA_TABLE_ID_FIELD = 'dataTableId';
export const COLUMNS: INodeProperties = {
displayName: 'Columns',
name: 'columns',
type: 'resourceMapper',
default: {
mappingMode: 'defineBelow',
value: null,
},
noDataExpression: true,
required: true,
typeOptions: {
loadOptionsDependsOn: [`${DATA_TABLE_ID_FIELD}.value`],
resourceMapper: {
resourceMapperMethod: 'getDataTables',
mode: 'add',
fieldWords: {
singular: 'column',
plural: 'columns',
},
addAllFields: true,
multiKeyMatch: true,
},
},
};

View File

@@ -0,0 +1,79 @@
import type {
ILoadOptionsFunctions,
INodeListSearchResult,
INodePropertyOptions,
ResourceMapperField,
ResourceMapperFields,
} from 'n8n-workflow';
import { getDataTableAggregateProxy, getDataTableProxyLoadOptions } from './utils';
// @ADO-3904: Pagination here does not work until a filter is entered or removed, suspected bug in ResourceLocator
export async function tableSearch(
this: ILoadOptionsFunctions,
filterString?: string,
prevPaginationToken?: string,
): Promise<INodeListSearchResult> {
const proxy = await getDataTableAggregateProxy(this);
const skip = prevPaginationToken === undefined ? 0 : parseInt(prevPaginationToken, 10);
const take = 100;
const filter = filterString === undefined ? {} : { filter: { name: filterString.toLowerCase() } };
const result = await proxy.getManyAndCount({
skip,
take,
...filter,
});
const results = result.data.map((row) => {
return {
name: row.name,
value: row.id,
};
});
const paginationToken = results.length === take ? `${skip + take}` : undefined;
return {
results,
paginationToken,
};
}
export async function getDataTableColumns(this: ILoadOptionsFunctions) {
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased-id, n8n-nodes-base/node-param-display-name-miscased
const returnData: INodePropertyOptions[] = [{ name: 'id - (string)', value: 'id' }];
const proxy = await getDataTableProxyLoadOptions(this);
const columns = await proxy.getColumns();
for (const column of columns) {
returnData.push({
name: `${column.name} - (${column.type})`,
value: column.name,
});
}
return returnData;
}
export async function getDataTables(this: ILoadOptionsFunctions): Promise<ResourceMapperFields> {
const proxy = await getDataTableProxyLoadOptions(this);
const result = await proxy.getColumns();
const fields: ResourceMapperField[] = [];
for (const field of result) {
const type = field.type === 'date' ? 'dateTime' : field.type;
fields.push({
id: field.name,
displayName: field.name,
required: false,
defaultMatch: false,
display: true,
type,
readOnly: false,
removed: false,
});
}
return { fields };
}

View File

@@ -0,0 +1,100 @@
import {
NodeOperationError,
type IDisplayOptions,
type IExecuteFunctions,
type INodeProperties,
} from 'n8n-workflow';
import type { FilterType } from './constants';
import { DATA_TABLE_ID_FIELD } from './fields';
import { buildGetManyFilter, isFieldArray, isMatchType } from './utils';
export function getSelectFields(displayOptions: IDisplayOptions): INodeProperties[] {
return [
{
displayName: 'Must Match',
name: 'matchType',
type: 'options',
options: [
{
name: 'Any Filter',
value: 'anyFilter',
},
{
name: 'All Filters',
value: 'allFilters',
},
] satisfies Array<{ value: FilterType; name: string }>,
displayOptions,
default: 'anyFilter',
},
{
displayName: 'Conditions',
name: 'filters',
type: 'fixedCollection',
typeOptions: {
multipleValues: true,
},
displayOptions,
default: {},
placeholder: 'Add Condition',
options: [
{
displayName: 'Conditions',
name: 'conditions',
values: [
{
displayName: 'Field Name or ID',
name: 'keyName',
type: 'options',
description:
'Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>',
typeOptions: {
loadOptionsDependsOn: [DATA_TABLE_ID_FIELD],
loadOptionsMethod: 'getDataTableColumns',
},
default: 'id',
},
{
displayName: 'Condition',
name: 'condition',
type: 'options',
options: [
{
name: 'Equals',
value: 'eq',
},
{
name: 'Not Equals',
value: 'neq',
},
],
default: 'eq',
},
{
displayName: 'Field Value',
name: 'keyValue',
type: 'string',
default: '',
},
],
},
],
description: 'Filter to decide which rows get',
},
];
}
export function getSelectFilter(ctx: IExecuteFunctions, index: number) {
const fields = ctx.getNodeParameter('filters.conditions', index, []);
const matchType = ctx.getNodeParameter('matchType', index, []);
if (!isMatchType(matchType)) {
throw new NodeOperationError(ctx.getNode(), 'unexpected match type');
}
if (!isFieldArray(fields)) {
throw new NodeOperationError(ctx.getNode(), 'unexpected fields input');
}
return buildGetManyFilter(fields, matchType);
}

View File

@@ -0,0 +1,112 @@
import type {
IDataObject,
INode,
ListDataStoreContentFilter,
IDataStoreProjectAggregateService,
IDataStoreProjectService,
IExecuteFunctions,
ILoadOptionsFunctions,
} from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import type { FieldEntry, FilterType } from './constants';
import { ALL_FILTERS, ANY_FILTER } from './constants';
import { DATA_TABLE_ID_FIELD } from './fields';
// We need two functions here since the available getNodeParameter
// overloads vary with the index
export async function getDataTableProxyExecute(
ctx: IExecuteFunctions,
index: number = 0,
): Promise<IDataStoreProjectService> {
if (ctx.helpers.getDataStoreProxy === undefined)
throw new NodeOperationError(
ctx.getNode(),
'Attempted to use Data Table node but the module is disabled',
);
const dataStoreId = ctx.getNodeParameter(DATA_TABLE_ID_FIELD, index, undefined, {
extractValue: true,
}) as string;
return await ctx.helpers.getDataStoreProxy(dataStoreId);
}
export async function getDataTableProxyLoadOptions(
ctx: ILoadOptionsFunctions,
): Promise<IDataStoreProjectService> {
if (ctx.helpers.getDataStoreProxy === undefined)
throw new NodeOperationError(
ctx.getNode(),
'Attempted to use Data Table node but the module is disabled',
);
const dataStoreId = ctx.getNodeParameter(DATA_TABLE_ID_FIELD, undefined, {
extractValue: true,
}) as string;
return await ctx.helpers.getDataStoreProxy(dataStoreId);
}
export async function getDataTableAggregateProxy(
ctx: IExecuteFunctions | ILoadOptionsFunctions,
): Promise<IDataStoreProjectAggregateService> {
if (ctx.helpers.getDataStoreAggregateProxy === undefined)
throw new NodeOperationError(
ctx.getNode(),
'Attempted to use Data Table node but the module is disabled',
);
return await ctx.helpers.getDataStoreAggregateProxy();
}
export function isFieldEntry(obj: unknown): obj is FieldEntry {
if (obj === null || typeof obj !== 'object') return false;
return 'keyName' in obj && 'condition' in obj && 'keyValue' in obj;
}
export function isMatchType(obj: unknown): obj is FilterType {
return typeof obj === 'string' && (obj === ANY_FILTER || obj === ALL_FILTERS);
}
export function buildGetManyFilter(
fieldEntries: FieldEntry[],
matchType: FilterType,
): ListDataStoreContentFilter {
const filters = fieldEntries.map((x) => ({
columnName: x.keyName,
condition: x.condition,
value: x.keyValue,
}));
return { type: matchType === 'allFilters' ? 'and' : 'or', filters };
}
export function isFieldArray(value: unknown): value is FieldEntry[] {
return (
value !== null && typeof value === 'object' && Array.isArray(value) && value.every(isFieldEntry)
);
}
export function dataObjectToApiInput(data: IDataObject, node: INode, row: number) {
return Object.fromEntries(
Object.entries(data).map(([k, v]) => {
if (v === undefined || v === null) return [k, null];
if (Array.isArray(v)) {
throw new NodeOperationError(
node,
`unexpected array input '${JSON.stringify(v)}' in row ${row}`,
);
}
if (!(v instanceof Date) && typeof v === 'object') {
throw new NodeOperationError(
node,
`unexpected object input '${JSON.stringify(v)}' in row ${row}`,
);
}
return [k, v];
}),
);
}

View File

@@ -0,0 +1,93 @@
import type { IExecuteFunctions } from 'n8n-workflow';
import { execute } from '../../../actions/row/get.operation';
import type { FieldEntry } from '../../../common/constants';
import { ANY_FILTER } from '../../../common/constants';
import { DATA_TABLE_ID_FIELD } from '../../../common/fields';
describe('Data Table get Operation', () => {
let mockExecuteFunctions: IExecuteFunctions;
const getManyRowsAndCount = jest.fn();
const dataTableId = 2345;
let filters: FieldEntry[];
beforeEach(() => {
filters = [
{
condition: 'eq',
keyName: 'id',
keyValue: 1,
},
];
mockExecuteFunctions = {
getNode: jest.fn().mockReturnValue({}),
getInputData: jest.fn().mockReturnValue([{}]),
getNodeParameter: jest.fn().mockImplementation((field) => {
switch (field) {
case DATA_TABLE_ID_FIELD:
return dataTableId;
case 'filters.conditions':
return filters;
case 'matchType':
return ANY_FILTER;
}
}),
helpers: {
getDataStoreProxy: jest.fn().mockReturnValue({
getManyRowsAndCount,
}),
},
} as unknown as IExecuteFunctions;
jest.clearAllMocks();
});
describe('execute', () => {
it('should get a few rows', async () => {
// ARRANGE
getManyRowsAndCount.mockReturnValue({ data: [{ id: 1 }], count: 1 });
// ACT
const result = await execute.call(mockExecuteFunctions, 0);
// ASSERT
expect(result).toEqual([{ json: { id: 1 } }]);
});
it('should get a paginated amount of rows', async () => {
// ARRANGE
getManyRowsAndCount.mockReturnValueOnce({
data: Array.from({ length: 1000 }, (_, k) => ({ id: k })),
count: 2345,
});
getManyRowsAndCount.mockReturnValueOnce({
data: Array.from({ length: 1000 }, (_, k) => ({ id: k + 1000 })),
count: 2345,
});
getManyRowsAndCount.mockReturnValueOnce({
data: Array.from({ length: 345 }, (_, k) => ({ id: k + 2000 })),
count: 2345,
});
filters = [];
// ACT
const result = await execute.call(mockExecuteFunctions, 0);
// ASSERT
expect(result.length).toBe(2345);
expect(result[0]).toEqual({ json: { id: 0 } });
expect(result[2344]).toEqual({ json: { id: 2344 } });
});
it('should pass null through correctly', async () => {
// ARRANGE
getManyRowsAndCount.mockReturnValue({ data: [{ id: 1, colA: null }], count: 1 });
// ACT
const result = await execute.call(mockExecuteFunctions, 0);
// ASSERT
expect(result).toEqual([{ json: { id: 1, colA: null } }]);
});
});
});

View File

@@ -488,6 +488,7 @@
"dist/nodes/Crypto/Crypto.node.js", "dist/nodes/Crypto/Crypto.node.js",
"dist/nodes/CustomerIo/CustomerIo.node.js", "dist/nodes/CustomerIo/CustomerIo.node.js",
"dist/nodes/CustomerIo/CustomerIoTrigger.node.js", "dist/nodes/CustomerIo/CustomerIoTrigger.node.js",
"dist/nodes/DataTable/DataTable.node.js",
"dist/nodes/DateTime/DateTime.node.js", "dist/nodes/DateTime/DateTime.node.js",
"dist/nodes/DebugHelper/DebugHelper.node.js", "dist/nodes/DebugHelper/DebugHelper.node.js",
"dist/nodes/DeepL/DeepL.node.js", "dist/nodes/DeepL/DeepL.node.js",