From fe99b7773dc03e95588c9bd349d0fbaf6994a9c5 Mon Sep 17 00:00:00 2001 From: Alex Grozav Date: Fri, 29 Aug 2025 13:50:45 +0100 Subject: [PATCH] feat(editor): Add boilerplate for SQLite WASM integration and `runData` worker (no-changelog) (#18959) --- packages/frontend/editor-ui/package.json | 1 + .../editor-ui/src/workers/database.ts | 48 ++++++++++ .../editor-ui/src/workers/run-data/db.ts | 19 ++++ .../src/workers/run-data/instance.ts | 8 ++ .../editor-ui/src/workers/run-data/worker.ts | 30 ++++++ .../editor-ui/src/workers/sqlite-wasm.d.ts | 96 +++++++++++++++++++ pnpm-lock.yaml | 54 ++++++----- 7 files changed, 231 insertions(+), 25 deletions(-) create mode 100644 packages/frontend/editor-ui/src/workers/database.ts create mode 100644 packages/frontend/editor-ui/src/workers/run-data/db.ts create mode 100644 packages/frontend/editor-ui/src/workers/run-data/instance.ts create mode 100644 packages/frontend/editor-ui/src/workers/run-data/worker.ts create mode 100644 packages/frontend/editor-ui/src/workers/sqlite-wasm.d.ts diff --git a/packages/frontend/editor-ui/package.json b/packages/frontend/editor-ui/package.json index 1a627c24f0..a5f12acfa0 100644 --- a/packages/frontend/editor-ui/package.json +++ b/packages/frontend/editor-ui/package.json @@ -48,6 +48,7 @@ "@n8n/utils": "workspace:*", "@replit/codemirror-indentation-markers": "^6.5.3", "@sentry/vue": "catalog:frontend", + "@sqlite.org/sqlite-wasm": "3.50.4-build1", "@types/semver": "^7.7.0", "@typescript/vfs": "^1.6.0", "@vue-flow/background": "^1.3.2", diff --git a/packages/frontend/editor-ui/src/workers/database.ts b/packages/frontend/editor-ui/src/workers/database.ts new file mode 100644 index 0000000000..23bbe165e8 --- /dev/null +++ b/packages/frontend/editor-ui/src/workers/database.ts @@ -0,0 +1,48 @@ +import { sqlite3Worker1Promiser } from '@sqlite.org/sqlite-wasm'; +import type { Promiser, DbId } from '@sqlite.org/sqlite-wasm'; + +export type DatabaseTable = { + name: string; + schema: string; +}; + +export type DatabaseConfig = { + filename: `file:${string}.sqlite3?vfs=opfs`; + tables: Record; +}; + +export async function initializeDatabase(config: DatabaseConfig) { + // Initialize the SQLite worker + const promiser: Promiser = await new Promise((resolve) => { + const _promiser = sqlite3Worker1Promiser({ + onready: () => resolve(_promiser), + }); + }); + + if (!promiser) throw new Error('Failed to initialize promiser'); + + // Get configuration and open database + const cfg = await promiser('config-get', {}); + const openResponse = await promiser('open', { + filename: config.filename, + }); + + if (openResponse.type === 'error') { + throw new Error(openResponse.result.message); + } + + const dbId: DbId = openResponse.result.dbId; + + for (const table of Object.values(config.tables)) { + await promiser('exec', { + dbId, + sql: table.schema, + }); + } + + return { + promiser, + dbId, + cfg, + }; +} diff --git a/packages/frontend/editor-ui/src/workers/run-data/db.ts b/packages/frontend/editor-ui/src/workers/run-data/db.ts new file mode 100644 index 0000000000..070b742621 --- /dev/null +++ b/packages/frontend/editor-ui/src/workers/run-data/db.ts @@ -0,0 +1,19 @@ +import type { DatabaseConfig } from '@/workers/database'; + +export const databaseConfig: DatabaseConfig = { + filename: 'file:n8n.sqlite3?vfs=opfs', + tables: { + executions: { + name: 'executions', + schema: ` + CREATE TABLE IF NOT EXISTS executions ( + id INTEGER PRIMARY KEY, + workflow_id INTEGER NOT NULL, + data TEXT CHECK (json_valid(data)) NOT NULL, + workflow TEXT CHECK (json_valid(workflow)) NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + `, + }, + }, +} as const; diff --git a/packages/frontend/editor-ui/src/workers/run-data/instance.ts b/packages/frontend/editor-ui/src/workers/run-data/instance.ts new file mode 100644 index 0000000000..0f6712b02a --- /dev/null +++ b/packages/frontend/editor-ui/src/workers/run-data/instance.ts @@ -0,0 +1,8 @@ +import * as Comlink from 'comlink'; +import type { RunDataWorker } from '@/workers/run-data/worker'; + +const worker = new Worker(new URL('./worker.ts', import.meta.url), { + type: 'module', +}); + +export const runDataWorker = Comlink.wrap(worker); diff --git a/packages/frontend/editor-ui/src/workers/run-data/worker.ts b/packages/frontend/editor-ui/src/workers/run-data/worker.ts new file mode 100644 index 0000000000..5dab723e41 --- /dev/null +++ b/packages/frontend/editor-ui/src/workers/run-data/worker.ts @@ -0,0 +1,30 @@ +import * as Comlink from 'comlink'; +import { databaseConfig } from '@/workers/run-data/db'; +import { initializeDatabase } from '@/workers/database'; +import type { Promiser, DbId } from '@sqlite.org/sqlite-wasm'; + +const state: { + initialized: boolean; + promiser: Promiser | undefined; + dbId: DbId; +} = { + initialized: false, + promiser: undefined, + dbId: undefined, +}; + +export const actions = { + async initialize() { + if (state.initialized) return; + + const { promiser, dbId } = await initializeDatabase(databaseConfig); + + state.promiser = promiser; + state.dbId = dbId; + state.initialized = true; + }, +}; + +export type RunDataWorker = typeof actions; + +Comlink.expose(actions); diff --git a/packages/frontend/editor-ui/src/workers/sqlite-wasm.d.ts b/packages/frontend/editor-ui/src/workers/sqlite-wasm.d.ts new file mode 100644 index 0000000000..0a1703d805 --- /dev/null +++ b/packages/frontend/editor-ui/src/workers/sqlite-wasm.d.ts @@ -0,0 +1,96 @@ +import type { Worker } from 'node:worker_threads'; + +declare module '@sqlite.org/sqlite-wasm' { + type OnreadyFunction = () => void; + + export type Sqlite3Worker1PromiserConfig = { + onready?: OnreadyFunction; + worker?: Worker | (() => Worker); + generateMessageId?: (messageObject: unknown) => string; + debug?: (...args: any[]) => void; + onunhandled?: (event: MessageEvent) => void; + }; + + export type DbId = string | undefined; + + export type PromiserMethods = { + 'config-get': { + args: Record; + result: { + dbID: DbId; + version: { + libVersion: string; + sourceId: string; + libVersionNumber: number; + downloadVersion: number; + }; + bigIntEnabled: boolean; + opfsEnabled: boolean; + vfsList: string[]; + }; + }; + open: { + args: Partial<{ + filename?: string; + vfs?: string; + }>; + result: { + dbId: DbId; + filename: string; + persistent: boolean; + vfs: string; + }; + }; + exec: { + args: { + sql: string; + dbId?: DbId; + bind?: unknown[]; + returnValue?: string; + }; + result: { + dbId: DbId; + sql: string; + bind: unknown[]; + returnValue: string; + resultRows?: unknown[][]; + }; + }; + }; + + export type PromiserResponseSuccess = { + type: T; + result: PromiserMethods[T]['result']; + messageId: string; + dbId: DbId; + workerReceivedTime: number; + workerRespondTime: number; + departureTime: number; + }; + + export type PromiserResponseError = { + type: 'error'; + result: { + operation: string; + message: string; + errorClass: string; + input: object; + stack: unknown[]; + }; + messageId: string; + dbId: DbId; + }; + + export type PromiserResponse = + | PromiserResponseSuccess + | PromiserResponseError; + + export type Promiser = ( + messageType: T, + messageArguments: PromiserMethods[T]['args'], + ) => Promise>; + + export function sqlite3Worker1Promiser( + config?: Sqlite3Worker1PromiserConfig | OnreadyFunction, + ): Promiser; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e04b833c88..d8f0239daa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1002,7 +1002,7 @@ importers: version: 4.3.0 '@getzep/zep-cloud': specifier: 1.0.12 - version: 1.0.12(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13)(langchain@0.3.30(316b19288832115574731e049dc7676a)) + version: 1.0.12(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13)(langchain@0.3.30(e7c2f10ddf33088da1e6affdf0fc6c0a)) '@getzep/zep-js': specifier: 0.9.0 version: 0.9.0 @@ -1029,7 +1029,7 @@ importers: version: 0.3.4(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13) '@langchain/community': specifier: 'catalog:' - version: 0.3.50(ccee17333f80550b1303d83de2b6f79a) + version: 0.3.50(7d9026709e640c92cdf2ea22646a0399) '@langchain/core': specifier: 'catalog:' version: 0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)) @@ -1146,7 +1146,7 @@ importers: version: 23.0.1 langchain: specifier: 0.3.30 - version: 0.3.30(316b19288832115574731e049dc7676a) + version: 0.3.30(e7c2f10ddf33088da1e6affdf0fc6c0a) lodash: specifier: 'catalog:' version: 4.17.21 @@ -2526,6 +2526,9 @@ importers: '@sentry/vue': specifier: catalog:frontend version: 9.42.1(pinia@2.2.4(typescript@5.9.2)(vue@3.5.13(typescript@5.9.2)))(vue@3.5.13(typescript@5.9.2)) + '@sqlite.org/sqlite-wasm': + specifier: 3.50.4-build1 + version: 3.50.4-build1 '@types/semver': specifier: ^7.7.0 version: 7.7.0 @@ -6894,6 +6897,10 @@ packages: resolution: {integrity: sha512-JtaY3FxmD+te+KSI2FJuEcfNC9T/DGGVf551babM7fAaXhjJUt7oSYurH1Devxd2+BOSUACCgt3buinx4UnmEA==} engines: {node: '>=18.0.0'} + '@sqlite.org/sqlite-wasm@3.50.4-build1': + resolution: {integrity: sha512-Qig2Wso7gPkU1PtXwFzndh+CTRzrIFxVGqv6eCetjU7YqxlHItj+GvQYwYTppCRgAPawtRN/4AJcEgB9xDHGug==} + hasBin: true + '@sqltools/formatter@1.2.5': resolution: {integrity: sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw==} @@ -18814,7 +18821,7 @@ snapshots: '@currents/commit-info': 1.0.1-beta.0 async-retry: 1.3.3 axios: 1.11.0(debug@4.4.1) - axios-retry: 4.5.0(axios@1.11.0(debug@4.4.1)) + axios-retry: 4.5.0(axios@1.11.0) c12: 1.11.2(magicast@0.3.5) chalk: 4.1.2 commander: 12.1.0 @@ -19127,7 +19134,7 @@ snapshots: '@gar/promisify@1.1.3': optional: true - '@getzep/zep-cloud@1.0.12(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13)(langchain@0.3.30(316b19288832115574731e049dc7676a))': + '@getzep/zep-cloud@1.0.12(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13)(langchain@0.3.30(e7c2f10ddf33088da1e6affdf0fc6c0a))': dependencies: form-data: 4.0.4 node-fetch: 2.7.0(encoding@0.1.13) @@ -19136,7 +19143,7 @@ snapshots: zod: 3.25.67 optionalDependencies: '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)) - langchain: 0.3.30(316b19288832115574731e049dc7676a) + langchain: 0.3.30(e7c2f10ddf33088da1e6affdf0fc6c0a) transitivePeerDependencies: - encoding @@ -19690,7 +19697,7 @@ snapshots: - aws-crt - encoding - '@langchain/community@0.3.50(ccee17333f80550b1303d83de2b6f79a)': + '@langchain/community@0.3.50(7d9026709e640c92cdf2ea22646a0399)': dependencies: '@browserbasehq/stagehand': 1.9.0(@playwright/test@1.54.2)(deepmerge@4.3.1)(dotenv@16.6.1)(encoding@0.1.13)(openai@5.12.2(ws@8.18.3)(zod@3.25.67))(zod@3.25.67) '@ibm-cloud/watsonx-ai': 1.1.2 @@ -19702,7 +19709,7 @@ snapshots: flat: 5.0.2 ibm-cloud-sdk-core: 5.3.2 js-yaml: 4.1.0 - langchain: 0.3.30(316b19288832115574731e049dc7676a) + langchain: 0.3.30(e7c2f10ddf33088da1e6affdf0fc6c0a) langsmith: 0.3.55(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)) openai: 5.12.2(ws@8.18.3)(zod@3.25.67) uuid: 10.0.0 @@ -19716,7 +19723,7 @@ snapshots: '@aws-sdk/credential-provider-node': 3.808.0 '@azure/storage-blob': 12.26.0 '@browserbasehq/sdk': 2.6.0(encoding@0.1.13) - '@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13)(langchain@0.3.30(316b19288832115574731e049dc7676a)) + '@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13)(langchain@0.3.30(e7c2f10ddf33088da1e6affdf0fc6c0a)) '@getzep/zep-js': 0.9.0 '@google-ai/generativelanguage': 2.6.0(encoding@0.1.13) '@google-cloud/storage': 7.12.1(encoding@0.1.13) @@ -21525,6 +21532,8 @@ snapshots: '@smithy/types': 4.2.0 tslib: 2.8.1 + '@sqlite.org/sqlite-wasm@3.50.4-build1': {} + '@sqltools/formatter@1.2.5': {} '@storybook/addon-a11y@8.6.4(storybook@8.6.4(prettier@3.6.2))': @@ -23481,14 +23490,9 @@ snapshots: axe-core@4.7.2: {} - axios-retry@4.5.0(axios@1.11.0(debug@4.4.1)): - dependencies: - axios: 1.11.0(debug@4.4.1) - is-retry-allowed: 2.2.0 - axios-retry@4.5.0(axios@1.11.0): dependencies: - axios: 1.11.0(debug@4.3.6) + axios: 1.11.0(debug@4.4.1) is-retry-allowed: 2.2.0 axios-retry@4.5.0(axios@1.8.3): @@ -23514,7 +23518,7 @@ snapshots: axios@1.11.0(debug@4.4.1): dependencies: - follow-redirects: 1.15.11(debug@4.4.1) + follow-redirects: 1.15.11(debug@4.3.6) form-data: 4.0.4 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -23853,7 +23857,7 @@ snapshots: bundlemon@3.1.0(typescript@5.9.2): dependencies: - axios: 1.11.0(debug@4.3.6) + axios: 1.11.0(debug@4.4.1) axios-retry: 4.5.0(axios@1.11.0) brotli-size: 4.0.0 bundlemon-utils: 2.0.1 @@ -26929,7 +26933,7 @@ snapshots: isstream: 0.1.2 jsonwebtoken: 9.0.2 mime-types: 2.1.35 - retry-axios: 2.6.0(axios@1.11.0) + retry-axios: 2.6.0(axios@1.11.0(debug@4.4.1)) tough-cookie: 4.1.4 transitivePeerDependencies: - supports-color @@ -26998,7 +27002,7 @@ snapshots: infisical-node@1.3.0: dependencies: - axios: 1.11.0(debug@4.3.6) + axios: 1.11.0(debug@4.4.1) dotenv: 16.3.1 tweetnacl: 1.0.3 tweetnacl-util: 0.15.1 @@ -28182,7 +28186,7 @@ snapshots: kuler@2.0.0: {} - langchain@0.3.30(316b19288832115574731e049dc7676a): + langchain@0.3.30(e7c2f10ddf33088da1e6affdf0fc6c0a): dependencies: '@langchain/core': 0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)) '@langchain/openai': 0.6.7(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(ws@8.18.3) @@ -28205,7 +28209,7 @@ snapshots: '@langchain/groq': 0.2.3(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(encoding@0.1.13) '@langchain/mistralai': 0.2.1(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67)))(zod@3.25.67) '@langchain/ollama': 0.2.3(@langchain/core@0.3.68(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0))(openai@5.12.2(ws@8.18.3)(zod@3.25.67))) - axios: 1.11.0(debug@4.3.6) + axios: 1.11.0(debug@4.4.1) cheerio: 1.0.0 handlebars: 4.7.8 transitivePeerDependencies: @@ -30257,7 +30261,7 @@ snapshots: posthog-node@3.2.1: dependencies: - axios: 1.11.0(debug@4.3.6) + axios: 1.11.0(debug@4.4.1) rusha: 0.8.14 transitivePeerDependencies: - debug @@ -30936,9 +30940,9 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 - retry-axios@2.6.0(axios@1.11.0): + retry-axios@2.6.0(axios@1.11.0(debug@4.4.1)): dependencies: - axios: 1.11.0(debug@4.3.6) + axios: 1.11.0(debug@4.4.1) retry-request@7.0.2(encoding@0.1.13): dependencies: @@ -31444,7 +31448,7 @@ snapshots: asn1.js: 5.4.1 asn1.js-rfc2560: 5.0.1(asn1.js@5.4.1) asn1.js-rfc5280: 3.0.0 - axios: 1.11.0(debug@4.3.6) + axios: 1.11.0(debug@4.4.1) big-integer: 1.6.52 bignumber.js: 9.1.2 binascii: 0.0.2