fix(editor): Fix highlighting in SQL editor (#19291)

This commit is contained in:
yehorkardash
2025-09-11 11:46:41 +00:00
committed by GitHub
parent 45e8209142
commit 03b865d4db
23 changed files with 1581 additions and 23 deletions

View File

@@ -0,0 +1,6 @@
/node_modules
/dist
/test/*.js
/test/*.d.ts
/test/*.d.ts.map
.tern-*

View File

@@ -0,0 +1,6 @@
/src
/test
/node_modules
.tern-*
rollup.config.js
tsconfig.json

View File

@@ -0,0 +1,21 @@
MIT License
Copyright (C) 2018-2021 by Marijn Haverbeke <marijn@haverbeke.berlin> and others
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@@ -0,0 +1,9 @@
# codemirror-lang-n8n-sql
SQL + n8n expression language support for CodeMirror 6.
Based on [`@codemirror/lang-sql`](https://github.com/codemirror/lang-sql).
## Author
© 2023 [Iván Ovejero](https://github.com/ivov)

View File

@@ -0,0 +1,13 @@
import { defineConfig, globalIgnores } from 'eslint/config';
import { baseConfig } from '@n8n/eslint-config/base';
export default defineConfig(baseConfig, globalIgnores(['src/grammar*.ts']), {
rules: {
'@typescript-eslint/naming-convention': 'warn',
'no-useless-escape': 'warn',
'@typescript-eslint/unbound-method': 'warn',
'@typescript-eslint/prefer-nullish-coalescing': 'warn',
'@typescript-eslint/no-unsafe-enum-comparison': 'off',
'@typescript-eslint/naming-convention': 'off',
},
});

View File

@@ -0,0 +1,6 @@
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
/** @type {import('jest').Config} */
export default require('../../../jest.config');

View File

@@ -0,0 +1,52 @@
{
"name": "@n8n/codemirror-lang-sql",
"version": "1.0.2",
"description": "SQL + n8n expression language support for CodeMirror 6",
"scripts": {
"clean": "rimraf dist .turbo",
"test": "jest",
"generate:sql:grammar": "lezer-generator --typeScript --output src/grammar.sql.ts src/sql.grammar",
"generate": "pnpm generate:sql:grammar && pnpm format",
"build": "tsc -p tsconfig.build.json",
"lint": "eslint . --quiet",
"lint:fix": "eslint . --fix",
"format": "biome format --write src test",
"format:check": "biome ci src test",
"typecheck": "tsc --noEmit"
},
"keywords": [
"editor",
"code"
],
"author": {
"name": "Iván Ovejero",
"email": "ivov.src@gmail.com",
"url": "https://ivov.dev"
},
"main": "dist/index.js",
"module": "src/index.ts",
"types": "dist/index.d.ts",
"exports": {
".": {
"require": "./dist/index.js",
"import": "./src/index.ts",
"types": "./dist/index.d.ts"
},
"./*": "./*"
},
"sideEffects": false,
"license": "MIT",
"dependencies": {
"@codemirror/autocomplete": "^6.0.0",
"@codemirror/language": "^6.0.0",
"@codemirror/state": "^6.0.0",
"@lezer/highlight": "^1.0.0",
"@lezer/lr": "^1.0.0",
"@n8n/codemirror-lang": "workspace:*"
},
"devDependencies": {
"@codemirror/text": "^0.19.6",
"@lezer/generator": "^1.7.0",
"@n8n/typescript-config": "workspace:*"
}
}

View File

@@ -0,0 +1,200 @@
import type { Completion, CompletionContext, CompletionSource } from '@codemirror/autocomplete';
import { completeFromList, ifNotIn } from '@codemirror/autocomplete';
import { syntaxTree } from '@codemirror/language';
import type { EditorState, Text } from '@codemirror/state';
import type { SyntaxNode } from '@lezer/common';
import { Type, Keyword } from './grammar.sql.terms';
const skippedTokens = ['Whitespace'];
function tokenBefore(tree: SyntaxNode) {
const cursor = tree.cursor().moveTo(tree.from, -1);
while (/Comment/.test(cursor.name)) cursor.moveTo(cursor.from, -1);
return cursor.node;
}
function idName(doc: Text, node: SyntaxNode): string {
const text = doc.sliceString(node.from, node.to);
const quoted = /^([`'"])(.*)\1$/.exec(text);
return quoted ? quoted[2] : text;
}
function plainID(node: SyntaxNode | null) {
return node && (node.name === 'Identifier' || node.name === 'QuotedIdentifier');
}
function pathFor(doc: Text, id: SyntaxNode) {
if (id.name === 'CompositeIdentifier') {
const path: string[] = [];
for (let ch = id.firstChild; ch; ch = ch.nextSibling)
if (plainID(ch)) path.push(idName(doc, ch));
return path;
}
return [idName(doc, id)];
}
function parentsFor(doc: Text, node: SyntaxNode | null) {
for (let path: string[] = []; ; ) {
if (!node || node.name !== '.') return path;
const name = tokenBefore(node);
if (!plainID(name)) return path;
path.unshift(idName(doc, name));
node = tokenBefore(name);
}
}
function sourceContext(state: EditorState, startPos: number) {
const pos = syntaxTree(state).resolveInner(startPos, -1);
const aliases = getAliases(state.doc, pos);
if (pos.name === 'Identifier' || pos.name === 'QuotedIdentifier' || pos.name === 'Keyword') {
return {
from: pos.from,
quoted:
pos.name === 'QuotedIdentifier' ? state.doc.sliceString(pos.from, pos.from + 1) : null,
parents: parentsFor(state.doc, tokenBefore(pos)),
aliases,
};
}
if (pos.name === '.') {
return { from: startPos, quoted: null, parents: parentsFor(state.doc, pos), aliases };
} else {
return { from: startPos, quoted: null, parents: [], empty: true, aliases };
}
}
const EndFrom = new Set(
'where group having order union intersect except all distinct limit offset fetch for'.split(' '),
);
function getAliases(doc: Text, at: SyntaxNode) {
let statement;
for (let parent: SyntaxNode | null = at; !statement; parent = parent.parent) {
if (!parent) return null;
if (parent.name === 'Statement') statement = parent;
}
let aliases: { [name: string]: string[] } | null = null;
for (
let scan = statement.firstChild, sawFrom = false, prevID: SyntaxNode | null = null;
scan;
scan = scan.nextSibling
) {
if (skippedTokens.includes(scan.name)) continue;
const kw = scan.name === 'Keyword' ? doc.sliceString(scan.from, scan.to).toLowerCase() : null;
let alias: string | null = null;
if (!sawFrom) {
sawFrom = kw === 'from';
} else if (kw === 'as' && prevID) {
let next = scan.nextSibling;
while (next && skippedTokens.includes(next.name)) next = next.nextSibling;
if (plainID(next)) alias = idName(doc, next!);
} else if (kw && EndFrom.has(kw)) {
break;
} else if (prevID && plainID(scan)) {
alias = idName(doc, scan);
}
if (alias) {
aliases ??= Object.create(null);
if (aliases) {
aliases[alias] = pathFor(doc, prevID!);
}
}
prevID = /Identifier$/.test(scan.name) ? scan : null;
}
return aliases;
}
function maybeQuoteCompletions(quote: string | null, completions: readonly Completion[]) {
if (!quote) return completions;
return completions.map((c) => ({ ...c, label: quote + c.label + quote, apply: undefined }));
}
const Span = /^\w*$/,
QuotedSpan = /^[`'"]?\w*[`'"]?$/;
class CompletionLevel {
list: readonly Completion[] = [];
children: { [name: string]: CompletionLevel } | undefined = undefined;
child(name: string): CompletionLevel {
const children =
this.children || (this.children = Object.create(null) as { [name: string]: CompletionLevel });
return children[name] || (children[name] = new CompletionLevel());
}
childCompletions(type: string) {
return this.children
? Object.keys(this.children)
.filter((x) => x)
.map((name) => ({ label: name, type }) as Completion)
: [];
}
}
export function completeFromSchema(
schema: { [table: string]: ReadonlyArray<string | Completion> },
tables?: readonly Completion[],
schemas?: readonly Completion[],
defaultTableName?: string,
defaultSchemaName?: string,
): CompletionSource {
const top = new CompletionLevel();
const defaultSchema = top.child(defaultSchemaName || '');
for (const table in schema) {
const dot = table.indexOf('.');
const schemaCompletions = dot > -1 ? top.child(table.slice(0, dot)) : defaultSchema;
const tableCompletions = schemaCompletions.child(dot > -1 ? table.slice(dot + 1) : table);
tableCompletions.list = schema[table].map((val) =>
typeof val === 'string' ? { label: val, type: 'property' } : val,
);
}
defaultSchema.list = (tables || defaultSchema.childCompletions('type')).concat(
defaultTableName ? defaultSchema.child(defaultTableName).list : [],
);
for (const sName in top.children) {
const schema = top.child(sName);
if (!schema.list.length) schema.list = schema.childCompletions('type');
}
top.list = defaultSchema.list.concat(schemas || top.childCompletions('type'));
return (context: CompletionContext) => {
// eslint-disable-next-line prefer-const
let { parents, from, quoted, empty, aliases } = sourceContext(context.state, context.pos);
if (empty && !context.explicit) return null;
if (aliases && parents.length === 1) parents = aliases[parents[0]] || parents;
let level = top;
for (const name of parents) {
while (!level.children?.[name]) {
if (level === top) level = defaultSchema;
else if (level === defaultSchema && defaultTableName) level = level.child(defaultTableName);
else return null;
}
level = level.child(name);
}
const quoteAfter = quoted && context.state.sliceDoc(context.pos, context.pos + 1) === quoted;
let options = level.list;
if (level === top && aliases)
options = options.concat(
Object.keys(aliases).map((name) => ({ label: name, type: 'constant' })),
);
return {
from,
to: quoteAfter ? context.pos + 1 : undefined,
options: maybeQuoteCompletions(quoted, options),
validFor: quoted ? QuotedSpan : Span,
};
};
}
export function completeKeywords(keywords: { [name: string]: number }, upperCase: boolean) {
const completions = Object.keys(keywords).map((keyword) => ({
label: upperCase ? keyword.toUpperCase() : keyword,
type:
keywords[keyword] === Type ? 'type' : keywords[keyword] === Keyword ? 'keyword' : 'variable',
boost: -1,
}));
return ifNotIn(
['QuotedIdentifier', 'SpecialVar', 'String', 'LineComment', 'BlockComment', '.'],
completeFromList(completions),
);
}

View File

@@ -0,0 +1,28 @@
// This file was generated by lezer-generator. You probably shouldn't edit it.
export const Whitespace = 1,
LineComment = 2,
BlockComment = 3,
String = 4,
Number = 5,
Bool = 6,
Null = 7,
ParenL = 8,
ParenR = 9,
BraceL = 10,
BraceR = 11,
BracketL = 12,
BracketR = 13,
Semi = 14,
Dot = 15,
Operator = 16,
Punctuation = 17,
SpecialVar = 18,
Identifier = 19,
QuotedIdentifier = 20,
Keyword = 21,
Type = 22,
Bits = 23,
Bytes = 24,
Builtin = 25,
Function = 26,
Script = 27;

View File

@@ -0,0 +1,20 @@
// This file was generated by lezer-generator. You probably shouldn't edit it.
import { LRParser } from '@lezer/lr';
import { tokens } from './tokens';
export const parser = LRParser.deserialize({
version: 14,
states:
"&SQ]QQOOO#wQRO'#DTO$OQQO'#CyO%eQQO'#CzO%lQQO'#C{O%sQQO'#C|OOQQ'#DT'#DTO%zQQO'#DTOOQQ'#DP'#DPO'ZQRO'#C}OOQQ'#Cx'#CxOOQQ'#DO'#DOQ]QQOOQOQQOOO'eQQO'#DQO(}QRO,59eO)UQQO,59eO)ZQQO'#DTOOQQ,59f,59fO)hQQO,59fOOQQ,59g,59gO)oQQO,59gOOQQ,59h,59hO)vQQO,59hOOQQ,59o,59oOOQQ-E6}-E6}OOQQ,59d,59dOOQQ-E6|-E6|OOQQ,59l,59lOOQQ-E7O-E7OO+[QRO1G/PO+cQQO,59eOOQQ1G/Q1G/QOOQQ1G/R1G/ROOQQ1G/S1G/SP+pQQO'#DPO+wQQO1G/PO)UQQO,59eO,UQQO'#Cy",
stateData:
",a~OPOSQOSROS~OSUOTUOUUOVUOWROYSO[TO^YO_QO`UOaUObPOcPOdPOeUOfUOgUOhUOiUO~O_^OSwXTwXUwXVwXWwXYwX[wX^wX`wXawXbwXcwXdwXewXfwXgwXhwXiwX~OuwX~P!jOb`Oc`Od`O~OSUOTUOUUOVUOWROYSO[TO_vO`UOaUObaOcaOdaOeUOfUOgUOhUOiUO~OXbO~P$ZOZdO~P$ZO]fO~P$ZOjhO~OSUOTUOUUOVUOWROYSO[TO_QO`UOaUObPOcPOdPOeUOfUOgUOhUOiUO~O^jOuqX~P&POblOclOdlO~O_^OSmaTmaUmaVmaWmaYma[ma^ma`maamabmacmadmaemafmagmahmaima~Ouma~P'pO_^O~OXwXZwX]wX~P!jOXpO~P$ZOZqO~P$ZO]rO~P$ZO_^OSmiTmiUmiVmiWmiYmi[mi^mi`miamibmicmidmiemifmigmihmiimi~Oumi~P)}OXmaZma]ma~P'pO^jO~P$ZOXmiZmi]mi~P)}ObuOcuOduO~O",
goto: '#uxPPPPPPPPPPPPPPPPPPPPPPPPPPPPy}}}!Z!g!k!q#VPP#iTZO[eUORSTX[cegseVORSTX[cegsT]O[Q[ORk[SXO[QcRQeSQgTZiXcegsQ_PWm_notQn`QoaRtueWORSTX[cegs',
nodeNames:
'⚠ Whitespace LineComment BlockComment String Number Bool Null ( ) { } [ ] ; . Operator Punctuation SpecialVar Identifier QuotedIdentifier Keyword Type Bits Bytes Builtin Function Script Statement CompositeIdentifier Parens Braces Brackets Statement',
maxTerm: 39,
skippedNodes: [0, 1, 2, 3],
repeatNodeCount: 3,
tokenData: 'RORO',
tokenizers: [0, tokens],
topRules: { Script: [0, 27] },
tokenPrec: 0,
});

View File

@@ -0,0 +1,2 @@
export * from './sql';
export * from './complete';

View File

@@ -0,0 +1,63 @@
@precedence { dot }
@top Script {
Statement { element* Semi }*
Statement { element+ }?
}
@skip { Whitespace | LineComment | BlockComment }
element {
String |
Number |
Bool |
Null |
Identifier |
QuotedIdentifier |
Bits |
Bytes |
Builtin |
SpecialVar |
CompositeIdentifier {
Dot? (QuotedIdentifier | Identifier | SpecialVar) (!dot Dot (QuotedIdentifier | Identifier | SpecialVar))+
} |
Keyword |
Type |
Operator |
Punctuation |
Parens { ParenL element* ParenR } |
Braces { BraceL element* BraceR } |
Brackets { BracketL element* BracketR }
Function
}
@external tokens tokens from "./tokens" {
Whitespace
LineComment
BlockComment
String
Number
Bool
Null
ParenL[@name="("]
ParenR[@name=")"]
BraceL[@name="{"]
BraceR[@name="}"]
BracketL[@name="["]
BracketR[@name="]"]
Semi[@name=";"]
Dot[@name="."]
Operator
Punctuation
SpecialVar
Identifier
QuotedIdentifier
Keyword
Type
Bits
Bytes
Builtin
Function
}
@detectDelim

View File

@@ -0,0 +1,364 @@
import type { Completion, CompletionSource } from '@codemirror/autocomplete';
import {
continuedIndent,
foldNodeProp,
indentNodeProp,
LanguageSupport,
LRLanguage,
} from '@codemirror/language';
import type { Extension } from '@codemirror/state';
import { parseMixed } from '@lezer/common';
import { styleTags, tags as t } from '@lezer/highlight';
import { expressionParser } from '@n8n/codemirror-lang';
import { completeFromSchema, completeKeywords } from './complete';
import { parser as baseParser } from './grammar.sql';
import type { Dialect } from './tokens';
import { dialect, SQLFunctions, SQLKeywords, SQLTypes, tokens, tokensFor } from './tokens';
const getSqlParser = () => {
return baseParser.configure({
props: [
indentNodeProp.add({
Statement: continuedIndent(),
}),
foldNodeProp.add({
Statement(tree) {
return { from: tree.firstChild!.to, to: tree.to };
},
BlockComment(tree) {
return { from: tree.from + 2, to: tree.to - 2 };
},
}),
styleTags({
Keyword: t.keyword,
Type: t.typeName,
Builtin: t.standard(t.name),
Bits: t.number,
Bytes: t.string,
Bool: t.bool,
Null: t.null,
// eslint-disable-next-line @typescript-eslint/naming-convention, id-denylist
Number: t.number,
// eslint-disable-next-line @typescript-eslint/naming-convention, id-denylist
String: t.string,
Identifier: t.name,
QuotedIdentifier: t.special(t.string),
SpecialVar: t.special(t.name),
LineComment: t.lineComment,
BlockComment: t.blockComment,
Operator: t.operator,
Function: t.function(t.variableName),
'Semi Punctuation': t.punctuation,
'( )': t.paren,
'{ }': t.brace,
'[ ]': t.squareBracket,
}),
],
});
};
export const getParser = (dialect: Dialect) => {
const sqlLangData = {
commentTokens: { line: '--', block: { open: '/*', close: '*/' } },
closeBrackets: { brackets: ['(', '[', '{', "'", '"', '`'] },
};
const sqlParser = getSqlParser().configure({
tokenizers: [{ from: tokens, to: tokensFor(dialect) }],
});
const sqlLanguage = LRLanguage.define({
name: 'sql',
parser: sqlParser,
languageData: sqlLangData,
});
const mixedParser = expressionParser.configure({
wrap: parseMixed((node) => {
return node.type.isTop
? {
parser: sqlLanguage.parser,
overlay: (node) => node.type.name === 'Plaintext',
}
: null;
}),
});
const mixedLanguage = LRLanguage.define({
parser: mixedParser,
name: 'expressionParser',
});
return { mixedLanguage, sqlLanguage };
};
/// Configuration for an [SQL Dialect](#lang-sql.SQLDialect).
export type SQLDialectSpec = {
/// A space-separated list of keywords for the dialect.
keywords?: string;
/// A space-separated list of functions for the dialect.
functions?: string;
/// A space-separated string of built-in identifiers for the dialect.
builtin?: string;
/// A space-separated string of type names for the dialect.
types?: string;
/// Controls whether regular strings allow backslash escapes.
backslashEscapes?: boolean;
/// Controls whether # creates a line comment.
hashComments?: boolean;
/// Controls whether `//` creates a line comment.
slashComments?: boolean;
/// When enabled `--` comments are only recognized when there's a
/// space after the dashes.
spaceAfterDashes?: boolean;
/// When enabled, things quoted with "$$" are treated as
/// strings, rather than identifiers.
doubleDollarQuotedStrings?: boolean;
/// When enabled, things quoted with double quotes are treated as
/// strings, rather than identifiers.
doubleQuotedStrings?: boolean;
/// Enables strings like `_utf8'str'` or `N'str'`.
charSetCasts?: boolean;
/// The set of characters that make up operators. Defaults to
/// `"*+\-%<>!=&|~^/"`.
operatorChars?: string;
/// The set of characters that start a special variable name.
/// Defaults to `"?"`.
specialVar?: string;
/// The characters that can be used to quote identifiers. Defaults
/// to `"\""`.
identifierQuotes?: string;
/// Controls whether bit values can be defined as 0b1010. Defaults
/// to false.
unquotedBitLiterals?: boolean;
/// Controls whether bit values can contain other characters than 0 and 1.
/// Defaults to false.
treatBitsAsBytes?: boolean;
};
/// Represents an SQL dialect.
export class SQLDialect {
private constructor(
/// @internal
readonly dialect: Dialect,
/// The mixed language for this dialect.
readonly language: LRLanguage,
/// The spec used to define this dialect.
readonly spec: SQLDialectSpec,
/// The sql language for this dialect.
readonly sqlLanguage: LRLanguage,
) {}
/// Returns the language for this dialect as an extension.
get extension() {
return this.language.extension;
}
/// Define a new dialect.
static define(spec: SQLDialectSpec) {
const d = dialect(spec, spec.keywords, spec.types, spec.builtin, spec.functions);
const { mixedLanguage, sqlLanguage } = getParser(d);
const language = mixedLanguage;
return new SQLDialect(d, language, spec, sqlLanguage);
}
}
/// Options used to configure an SQL extension.
export interface SQLConfig {
/// The [dialect](#lang-sql.SQLDialect) to use. Defaults to
/// [`StandardSQL`](#lang-sql.StandardSQL).
dialect?: SQLDialect;
/// An object that maps table names, optionally prefixed with a
/// schema name (`"schema.table"`) to options (columns) that can be
/// completed for that table. Use lower-case names here.
schema?: { [table: string]: ReadonlyArray<string | Completion> };
/// By default, the completions for the table names will be
/// generated from the `schema` object. But if you want to
/// customize them, you can pass an array of completions through
/// this option.
tables?: readonly Completion[];
/// Similar to `tables`, if you want to provide completion objects
/// for your schemas rather than using the generated ones, pass them
/// here.
schemas?: readonly Completion[];
/// When given, columns from the named table can be completed
/// directly at the top level.
defaultTable?: string;
/// When given, tables prefixed with this schema name can be
/// completed directly at the top level.
defaultSchema?: string;
/// When set to true, keyword completions will be upper-case.
upperCaseKeywords?: boolean;
}
/// Returns a completion source that provides keyword completion for
/// the given SQL dialect.
export function keywordCompletionSource(dialect: SQLDialect, upperCase = false): CompletionSource {
return completeKeywords(dialect.dialect.words, upperCase);
}
/// FIXME remove on 1.0 @internal
export function keywordCompletion(dialect: SQLDialect, upperCase = false): Extension {
return dialect.sqlLanguage.data.of({
autocomplete: keywordCompletionSource(dialect, upperCase),
});
}
/// Returns a completion sources that provides schema-based completion
/// for the given configuration.
export function schemaCompletionSource(config: SQLConfig): CompletionSource {
return config.schema
? completeFromSchema(
config.schema,
config.tables,
config.schemas,
config.defaultTable,
config.defaultSchema,
)
: () => null;
}
/// FIXME remove on 1.0 @internal
export function schemaCompletion(config: SQLConfig): Extension {
return config.schema
? (config.dialect || StandardSQL).sqlLanguage.data.of({
autocomplete: schemaCompletionSource(config),
})
: [];
}
/// SQL language support for the given SQL dialect, with keyword
/// completion, and, if provided, schema-based completion as extra
/// extensions.
export function sql(config: SQLConfig = {}) {
const lang = config.dialect || StandardSQL;
return new LanguageSupport(lang.language, [
schemaCompletion(config),
keywordCompletion(lang, !!config.upperCaseKeywords),
]);
}
/// The standard SQL dialect.
export const StandardSQL = SQLDialect.define({});
/// Dialect for [PostgreSQL](https://www.postgresql.org).
export const PostgreSQL = SQLDialect.define({
charSetCasts: true,
doubleDollarQuotedStrings: true,
operatorChars: '+-*/<>=~!@#%^&|`?',
specialVar: '',
functions:
SQLFunctions +
'abs aggregate array_agg array_max_cardinality avg decode encode bernoulli cardinality ceil ceiling char_length character_length coalesce corr degrees substring system xmlcomment xmlvalidate xmlexists length strip lower upper bit_length normalize mod octet_length overlay ln sqrt power exp log lower',
keywords:
SQLKeywords +
'abort absent access according ada admin alias also always analyse analyze asensitive assert assignment asymmetric atomic attach attribute attributes backward base64 begin_frame begin_partition bit_length blocked bom c cache called catalog_name chain character_set_catalog character_set_name character_set_schema characteristics characters checkpoint class class_origin cluster cobol collation_catalog collation_name collation_schema collect column_name columns command_function command_function_code comment comments committed concurrently condition_number configuration conflict connection_name constant constraint_catalog constraint_name constraint_schema contains content control conversion convert copy cost covar_pop covar_samp csv cume_dist current_catalog current_row current_schema cursor_name database datalink datatype datetime_interval_code datetime_interval_precision db debug defaults defined definer degree delimiter delimiters dense_rank depends derived detach detail dictionary disable discard dispatch dlnewcopy dlpreviouscopy dlurlcomplete dlurlcompleteonly dlurlcompletewrite dlurlpath dlurlpathonly dlurlpathwrite dlurlscheme dlurlserver dlvalue document dump dynamic_function dynamic_function_code element elsif empty enable encoding encrypted end_frame end_partition endexec enforced enum errcode error event every exclude excluding exclusive explain expression extension extract family file filter final first_value flag floor following force foreach fortran forward frame_row freeze fs functions fusion g generated granted greatest groups handler header hex hierarchy hint id ignore ilike immediately immutable implementation implicit import include including increment indent index indexes info inherit inherits inline insensitive instance instantiable instead integrity intersection invoker isnull k key_member key_type label lag last_value lead leakproof least length library like_regex link listen load location lock locked logged m mapping matched materialized max max_cardinality maxvalue member merge message message_length message_octet_length message_text min minvalue mode more move multiset mumps name namespace nfc nfd nfkc nfkd nil normalize normalized nothing notice notify notnull nowait nth_value ntile nullable nullif nulls number occurrences_regex octets off offset oids operator options ordering others over overriding owned owner p parallel parameter_mode parameter_name parameter_ordinal_position parameter_specific_catalog parameter_specific_name parameter_specific_schema parser partition pascal passing passthrough password percent percent_rank percentile_cont percentile_disc perform period permission pg_context pg_datatype_name pg_exception_context pg_exception_detail pg_exception_hint placing plans pli policy portion position position_regex precedes preceding prepared print_strict_params procedural procedures program publication query quote raise range rank reassign recheck recovery refresh regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy regr_syy reindex rename repeatable replace replica requiring reset respect restart restore result_oid returned_cardinality returned_length returned_octet_length returned_sqlstate returning reverse routine_catalog routine_name routine_schema routines row_count row_number rowtype rule scale schema_name schemas scope scope_catalog scope_name scope_schema security selective self sensitive sequence sequences serializable server server_name setof share show simple skip slice snapshot source specific_name sqlcode sqlerror stable stacked standalone statement statistics stddev_pop stddev_samp stdin stdout storage strict strip structure style subclass_origin submultiset subscription substring_regex succeeds sum symmetric sysid system system_time t table_name tables tablesample tablespace temp template ties token top_level_count transaction_active transactions_committed transactions_rolled_back transform transforms translate translate_regex trigger_catalog trigger_name trigger_schema trim trim_array truncate trusted type types uescape unbounded uncommitted unencrypted unlink unlisten unlogged unnamed untyped upper uri use_column use_variable user_defined_type_catalog user_defined_type_code user_defined_type_name user_defined_type_schema vacuum valid validate validator value_of var_pop var_samp varbinary variable_conflict variadic verbose version versioning views volatile warning whitespace width_bucket window within wrapper xmlagg xmlattributes xmlbinary xmlcast xmlcomment xmlconcat xmldeclaration xmldocument xmlelement xmlexists xmlforest xmliterate xmlnamespaces xmlparse xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltext xmlvalidate yes',
types:
SQLTypes +
'bigint int8 bigserial serial8 varbit bool box bytea cidr circle precision float8 inet int4 json jsonb line lseg macaddr macaddr8 money numeric pg_lsn point polygon float4 int2 smallserial serial2 serial serial4 text timetz timestamptz tsquery tsvector txid_snapshot uuid xml',
});
const MySQLKeywords =
'accessible algorithm analyze asensitive authors auto_increment autocommit avg avg_row_length binlog btree cache catalog_name chain change changed checkpoint checksum class_origin client_statistics coalesce code collations columns comment committed completion concurrent consistent contains contributors convert database databases day_hour day_microsecond day_minute day_second delay_key_write delayed delimiter des_key_file dev_pop dev_samp deviance directory disable discard distinctrow div dual dumpfile enable enclosed ends engine engines enum errors escaped even event events every explain extended fast field fields flush force found_rows fulltext grants handler hash high_priority hosts hour_microsecond hour_minute hour_second ignore ignore_server_ids import index index_statistics infile innodb insensitive insert_method install invoker iterate keys kill linear lines list load lock logs low_priority master master_heartbeat_period master_ssl_verify_server_cert masters max max_rows maxvalue message_text middleint migrate min min_rows minute_microsecond minute_second mod mode modify mutex mysql_errno no_write_to_binlog offline offset one online optimize optionally outfile pack_keys parser partition partitions password phase plugin plugins prev processlist profile profiles purge query quick range read_write rebuild recover regexp relaylog remove rename reorganize repair repeatable replace require resume rlike row_format rtree schedule schema_name schemas second_microsecond security sensitive separator serializable server share show slave slow snapshot soname spatial sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_no_cache sql_small_result ssl starting starts std stddev stddev_pop stddev_samp storage straight_join subclass_origin sum suspend table_name table_statistics tables tablespace terminated triggers truncate uncommitted uninstall unlock upgrade use use_frm user_resources user_statistics utc_date utc_time utc_timestamp variables views warnings xa xor year_month zerofill';
const MySQLTypes =
SQLTypes +
'bool blob long longblob longtext medium mediumblob mediumint mediumtext tinyblob tinyint tinytext text bigint int1 int2 int3 int4 int8 float4 float8 varbinary varcharacter precision datetime unsigned signed';
const MySQLBuiltin =
'charset clear edit ego help nopager notee nowarning pager print prompt quit rehash source status system tee';
/// [MySQL](https://dev.mysql.com/) dialect.
export const MySQL = SQLDialect.define({
operatorChars: '*+-%<>!=&|^',
charSetCasts: true,
doubleQuotedStrings: true,
unquotedBitLiterals: true,
hashComments: true,
spaceAfterDashes: true,
specialVar: '@?',
identifierQuotes: '`',
functions: SQLFunctions,
keywords: SQLKeywords + 'group_concat ' + MySQLKeywords,
types: MySQLTypes,
builtin: MySQLBuiltin,
});
/// Variant of [`MySQL`](#lang-sql.MySQL) for
/// [MariaDB](https://mariadb.org/).
export const MariaSQL = SQLDialect.define({
operatorChars: '*+-%<>!=&|^',
charSetCasts: true,
doubleQuotedStrings: true,
unquotedBitLiterals: true,
hashComments: true,
spaceAfterDashes: true,
specialVar: '@?',
identifierQuotes: '`',
functions: SQLFunctions,
keywords:
SQLKeywords +
'always generated groupby_concat hard persistent shutdown soft virtual ' +
MySQLKeywords,
types: MySQLTypes,
builtin: MySQLBuiltin,
});
/// SQL dialect for Microsoft [SQL
/// Server](https://www.microsoft.com/en-us/sql-server).
export const MSSQL = SQLDialect.define({
functions: SQLFunctions,
keywords:
SQLKeywords +
'trigger proc view index for add constraint key primary foreign collate clustered nonclustered declare exec go if use index holdlock nolock nowait paglock pivot readcommitted readcommittedlock readpast readuncommitted repeatableread rowlock serializable snapshot tablock tablockx unpivot updlock with',
types:
SQLTypes +
'bigint smallint smallmoney tinyint money real text nvarchar ntext varbinary image hierarchyid uniqueidentifier sql_variant xml',
builtin:
'binary_checksum checksum connectionproperty context_info current_request_id error_line error_message error_number error_procedure error_severity error_state formatmessage get_filestream_transaction_context getansinull host_id host_name isnull isnumeric min_active_rowversion newid newsequentialid rowcount_big xact_state object_id',
operatorChars: '*+-%<>!=^&|/',
specialVar: '@',
});
/// [SQLite](https://sqlite.org/) dialect.
export const SQLite = SQLDialect.define({
functions: SQLFunctions + 'isnull notnull',
keywords:
SQLKeywords +
'abort analyze attach autoincrement conflict database detach exclusive fail glob ignore index indexed instead offset plan pragma query raise regexp reindex rename replace temp vacuum virtual',
types:
SQLTypes +
'bool blob long longblob longtext medium mediumblob mediumint mediumtext tinyblob tinyint tinytext text bigint int2 int8 unsigned signed real',
builtin:
'auth backup bail changes clone databases dbinfo dump echo eqp explain fullschema headers help import imposter indexes iotrace lint load log mode nullvalue once print prompt quit restore save scanstats separator shell show stats system tables testcase timeout timer trace vfsinfo vfslist vfsname width',
operatorChars: '*+-%<>!=&|/~',
identifierQuotes: '`"',
specialVar: '@:?$',
});
/// Dialect for [Cassandra](https://cassandra.apache.org/)'s SQL-ish query language.
export const Cassandra = SQLDialect.define({
functions:
'to_timestamp to_unix_timestamp to_date cast abs round sqrt count sum max avg min log log10 now min_timeuuid max_timeuuid current_timestamp current_date current_time current_timeuuid',
keywords:
'add all allow alter and any apply as asc authorize batch begin by clustering columnfamily compact consistency count create custom delete desc distinct drop each_quorum exists filtering from grant if in index insert into key keyspace keyspaces level limit local_one local_quorum modify nan norecursive nosuperuser not of on one order password permission permissions primary quorum rename revoke schema select set storage superuser table three to token truncate ttl two type unlogged update use user users using values where with writetime infinity NaN',
types:
SQLTypes +
'ascii bigint blob counter frozen inet list map static text timeuuid tuple uuid varint',
slashComments: true,
});
/// [PL/SQL](https://en.wikipedia.org/wiki/PL/SQL) dialect.
export const PLSQL = SQLDialect.define({
functions: SQLFunctions,
keywords:
SQLKeywords +
'abort accept access add all alter and any arraylen as asc assert assign at attributes audit authorization avg base_table begin between binary_integer body by case cast char_base check close cluster clusters colauth column comment commit compress connected constant constraint crash create current currval cursor data_base database dba deallocate debugoff debugon declare default definition delay delete desc digits dispose distinct do drop else elseif elsif enable end entry exception exception_init exchange exclusive exists external fast fetch file for force form from function generic goto grant group having identified if immediate in increment index indexes indicator initial initrans insert interface intersect into is key level library like limited local lock log logging loop master maxextents maxtrans member minextents minus mislabel mode modify multiset new next no noaudit nocompress nologging noparallel not nowait number_base of off offline on online only option or order out package parallel partition pctfree pctincrease pctused pls_integer positive positiven pragma primary prior private privileges procedure public raise range raw rebuild record ref references refresh rename replace resource restrict return returning returns reverse revoke rollback row rowid rowlabel rownum rows run savepoint schema segment select separate set share snapshot some space split sql start statement storage subtype successful synonym tabauth table tables tablespace task terminate then to trigger truncate type union unique unlimited unrecoverable unusable update use using validate value values variable view views when whenever where while with work',
builtin:
'appinfo arraysize autocommit autoprint autorecovery autotrace blockterminator break btitle cmdsep colsep compatibility compute concat copycommit copytypecheck define echo editfile embedded feedback flagger flush heading headsep instance linesize lno loboffset logsource longchunksize markup native newpage numformat numwidth pagesize pause pno recsep recsepchar repfooter repheader serveroutput shiftinout show showmode spool sqlblanklines sqlcase sqlcode sqlcontinue sqlnumber sqlpluscompatibility sqlprefix sqlprompt sqlterminator suffix tab term termout timing trimout trimspool ttitle underline verify version wrap',
types:
SQLTypes +
'ascii bfile bfilename bigserial bit blob dec long number nvarchar nvarchar2 serial smallint string text uid varchar2 xml',
operatorChars: '*/+-%<>!=~',
doubleQuotedStrings: true,
charSetCasts: true,
});

View File

@@ -0,0 +1,373 @@
import type { InputStream } from '@lezer/lr';
import { ExternalTokenizer } from '@lezer/lr';
import {
LineComment,
BlockComment,
String as StringToken,
Number as NumberToken,
Bits,
Bytes,
Bool,
Null,
ParenL,
ParenR,
BraceL,
BraceR,
BracketL,
BracketR,
Semi,
Dot,
Operator,
Punctuation,
SpecialVar,
Identifier,
QuotedIdentifier,
Keyword,
Type,
Builtin,
Whitespace,
Function,
} from './grammar.sql.terms';
const enum Ch {
Newline = 10,
Space = 32,
DoubleQuote = 34,
Hash = 35,
Dollar = 36,
SingleQuote = 39,
ParenL = 40,
ParenR = 41,
Star = 42,
Plus = 43,
Comma = 44,
Dash = 45,
Dot = 46,
Slash = 47,
Colon = 58,
Semi = 59,
Question = 63,
At = 64,
BracketL = 91,
BracketR = 93,
Backslash = 92,
Underscore = 95,
Backtick = 96,
BraceL = 123,
BraceR = 125,
A = 65,
a = 97,
B = 66,
b = 98,
E = 69,
e = 101,
F = 70,
f = 102,
N = 78,
n = 110,
X = 88,
x = 120,
Z = 90,
z = 122,
_0 = 48,
_1 = 49,
_9 = 57,
}
function isAlpha(ch: number) {
return (ch >= Ch.A && ch <= Ch.Z) || (ch >= Ch.a && ch <= Ch.z) || (ch >= Ch._0 && ch <= Ch._9);
}
function isHexDigit(ch: number) {
return (ch >= Ch._0 && ch <= Ch._9) || (ch >= Ch.a && ch <= Ch.f) || (ch >= Ch.A && ch <= Ch.F);
}
function readLiteral(input: InputStream, endQuote: number, backslashEscapes: boolean) {
for (let escaped = false; ; ) {
if (input.next < 0) return;
if (input.next === endQuote && !escaped) {
input.advance();
return;
}
escaped = backslashEscapes && !escaped && input.next === Ch.Backslash;
input.advance();
}
}
function readDoubleDollarLiteral(input: InputStream) {
for (;;) {
if (input.next < 0 || input.peek(1) < 0) return;
if (input.next === Ch.Dollar && input.peek(1) === Ch.Dollar) {
input.advance(2);
return;
}
input.advance();
}
}
function readWord(input: InputStream): void;
function readWord(input: InputStream, result: string): string;
function readWord(input: InputStream, result?: string) {
for (;;) {
if (input.next !== Ch.Underscore && !isAlpha(input.next)) break;
if (result !== null) result += String.fromCharCode(input.next);
input.advance();
}
return result;
}
function readWordOrQuoted(input: InputStream) {
if (
input.next === Ch.SingleQuote ||
input.next === Ch.DoubleQuote ||
input.next === Ch.Backtick
) {
const quote = input.next;
input.advance();
readLiteral(input, quote, false);
} else {
readWord(input);
}
}
function readBits(input: InputStream, endQuote?: number) {
while (input.next === Ch._0 || input.next === Ch._1) input.advance();
if (endQuote && input.next === endQuote) input.advance();
}
function readNumber(input: InputStream, sawDot: boolean) {
for (;;) {
if (input.next === Ch.Dot) {
if (sawDot) break;
sawDot = true;
} else if (input.next < Ch._0 || input.next > Ch._9) {
break;
}
input.advance();
}
if (input.next === Ch.E || input.next === Ch.e) {
input.advance();
// advance() updates next, so we need to cast to unknown to avoid type errors
const advancedInput = input as unknown as InputStream;
if (advancedInput.next === Ch.Plus || advancedInput.next === Ch.Dash) input.advance();
while (input.next >= Ch._0 && input.next <= Ch._9) input.advance();
}
}
function eol(input: InputStream) {
while (!(input.next < 0 || input.next === Ch.Newline)) input.advance();
}
function inString(ch: number, str: string) {
for (let i = 0; i < str.length; i++) if (str.charCodeAt(i) === ch) return true;
return false;
}
const Space = ' \t\r\n';
function keywords(keywords: string, types: string, builtin?: string, functions?: string) {
const result = Object.create(null) as { [name: string]: number };
result['true'] = result['false'] = Bool;
result['null'] = result['unknown'] = Null;
for (const kw of keywords.split(' ')) if (kw) result[kw] = Keyword;
for (const tp of types.split(' ')) if (tp) result[tp] = Type;
for (const kw of (builtin || '').split(' ')) if (kw) result[kw] = Builtin;
for (const fn of (functions || '').split(' ')) if (fn) result[fn] = Function;
return result;
}
export interface Dialect {
backslashEscapes: boolean;
hashComments: boolean;
spaceAfterDashes: boolean;
slashComments: boolean;
doubleQuotedStrings: boolean;
doubleDollarQuotedStrings: boolean;
unquotedBitLiterals: boolean;
treatBitsAsBytes: boolean;
charSetCasts: boolean;
operatorChars: string;
specialVar: string;
identifierQuotes: string;
words: { [name: string]: number };
}
export const SQLTypes =
'array binary bit boolean char character clob date decimal double float int integer interval large national nchar nclob numeric object precision real smallint time timestamp varchar varying ';
export const SQLFunctions =
'abs absolute case check cast concat coalesce cube collate count current_date current_path current_role current_time current_timestamp current_user day exists grouping hour localtime localtimestamp minute month second trim session_user size system_user treat unnest user year equals lower upper pow floor ceil exp log ifnull min max avg sum sqrt round ';
export const SQLKeywords =
'action add after all allocate alter and any are as asc assertion at authorization before begin between both breadth by call cascade cascaded catalog close collation column commit condition connect connection constraint constraints constructor continue corresponding create cross current current_default_transform_group current_transform_group_for_type cursor cycle data deallocate declare default deferrable deferred delete depth deref desc describe descriptor deterministic diagnostics disconnect distinct do domain drop dynamic each else elseif end end-exec equals escape except exception exec execute exit external fetch first for foreign found from free full function general get global go goto grant group handle having hold identity if immediate in indicator initially inner inout input insert intersect into is isolation join key language last lateral leading leave left level like limit local locator loop map match method modifies module names natural nesting new next no none not of old on only open option or order ordinality out outer output overlaps pad parameter partial path prepare preserve primary prior privileges procedure public read reads recursive redo ref references referencing relative release repeat resignal restrict result return returns revoke right role rollback rollup routine row rows savepoint schema scroll search section select session set sets signal similar some space specific specifictype sql sqlexception sqlstate sqlwarning start state static table temporary then timezone_hour timezone_minute to trailing transaction translation trigger under undo union unique until update usage using value values view when whenever where while with without work write zone ';
const defaults: Dialect = {
backslashEscapes: false,
hashComments: false,
spaceAfterDashes: false,
slashComments: false,
doubleQuotedStrings: false,
doubleDollarQuotedStrings: false,
unquotedBitLiterals: false,
treatBitsAsBytes: false,
charSetCasts: false,
operatorChars: '*+-%<>!==&|~^/',
specialVar: '?',
identifierQuotes: '"',
words: keywords(SQLKeywords, SQLTypes, '', SQLFunctions),
};
export function dialect(
spec: Partial<Dialect>,
kws?: string,
types?: string,
builtin?: string,
functions?: string,
): Dialect {
const dialect = {} as Dialect;
for (const prop in defaults)
(dialect as unknown as Record<string, unknown>)[prop] = (
(Object.prototype.hasOwnProperty.call(spec, prop) ? spec : defaults) as unknown as Record<
string,
unknown
>
)[prop];
if (kws) dialect.words = keywords(kws, types || '', builtin, functions);
return dialect;
}
export function tokensFor(d: Dialect) {
return new ExternalTokenizer((input) => {
const { next } = input;
input.advance();
if (inString(next, Space)) {
while (inString(input.next, Space)) input.advance();
input.acceptToken(Whitespace);
} else if (next === Ch.Dollar && input.next === Ch.Dollar && d.doubleDollarQuotedStrings) {
readDoubleDollarLiteral(input);
input.acceptToken(StringToken);
} else if (next === Ch.SingleQuote || (next === Ch.DoubleQuote && d.doubleQuotedStrings)) {
readLiteral(input, next, d.backslashEscapes);
input.acceptToken(StringToken);
} else if (
(next === Ch.Hash && d.hashComments) ||
(next === Ch.Slash && input.next === Ch.Slash && d.slashComments)
) {
eol(input);
input.acceptToken(LineComment);
} else if (
next === Ch.Dash &&
input.next === Ch.Dash &&
(!d.spaceAfterDashes || input.peek(1) === Ch.Space)
) {
eol(input);
input.acceptToken(LineComment);
} else if (next === Ch.Slash && input.next === Ch.Star) {
input.advance();
for (let depth = 1; ; ) {
const cur: number = input.next;
if (input.next < 0) break;
input.advance();
if (cur === Ch.Star && (input as unknown as InputStream).next === Ch.Slash) {
depth--;
input.advance();
if (!depth) break;
} else if (cur === Ch.Slash && input.next === Ch.Star) {
depth++;
input.advance();
}
}
input.acceptToken(BlockComment);
} else if ((next === Ch.e || next === Ch.E) && input.next === Ch.SingleQuote) {
input.advance();
readLiteral(input, Ch.SingleQuote, true);
} else if (
(next === Ch.n || next === Ch.N) &&
input.next === Ch.SingleQuote &&
d.charSetCasts
) {
input.advance();
readLiteral(input, Ch.SingleQuote, d.backslashEscapes);
input.acceptToken(StringToken);
} else if (next === Ch.Underscore && d.charSetCasts) {
for (let i = 0; ; i++) {
if (input.next === Ch.SingleQuote && i > 1) {
input.advance();
readLiteral(input, Ch.SingleQuote, d.backslashEscapes);
input.acceptToken(StringToken);
break;
}
if (!isAlpha(input.next)) break;
input.advance();
}
} else if (next === Ch.ParenL) {
input.acceptToken(ParenL);
} else if (next === Ch.ParenR) {
input.acceptToken(ParenR);
} else if (next === Ch.BraceL) {
input.acceptToken(BraceL);
} else if (next === Ch.BraceR) {
input.acceptToken(BraceR);
} else if (next === Ch.BracketL) {
input.acceptToken(BracketL);
} else if (next === Ch.BracketR) {
input.acceptToken(BracketR);
} else if (next === Ch.Semi) {
input.acceptToken(Semi);
} else if (d.unquotedBitLiterals && next === Ch._0 && input.next === Ch.b) {
input.advance();
readBits(input);
input.acceptToken(Bits);
} else if (
(next === Ch.b || next === Ch.B) &&
(input.next === Ch.SingleQuote || input.next === Ch.DoubleQuote)
) {
const quoteStyle = input.next;
input.advance();
if (d.treatBitsAsBytes) {
readLiteral(input, quoteStyle, d.backslashEscapes);
input.acceptToken(Bytes);
} else {
readBits(input, quoteStyle);
input.acceptToken(Bits);
}
} else if (
(next === Ch._0 && (input.next === Ch.x || input.next === Ch.X)) ||
((next === Ch.x || next === Ch.X) && input.next === Ch.SingleQuote)
) {
const quoted = input.next === Ch.SingleQuote;
input.advance();
while (isHexDigit(input.next)) input.advance();
if (quoted && input.next === Ch.SingleQuote) input.advance();
input.acceptToken(NumberToken);
} else if (next === Ch.Dot && input.next >= Ch._0 && input.next <= Ch._9) {
readNumber(input, true);
input.acceptToken(NumberToken);
} else if (next === Ch.Dot) {
input.acceptToken(Dot);
} else if (next >= Ch._0 && next <= Ch._9) {
readNumber(input, false);
input.acceptToken(NumberToken);
} else if (inString(next, d.operatorChars)) {
while (inString(input.next, d.operatorChars)) input.advance();
input.acceptToken(Operator);
} else if (inString(next, d.specialVar)) {
if (input.next === next) input.advance();
readWordOrQuoted(input);
input.acceptToken(SpecialVar);
} else if (inString(next, d.identifierQuotes)) {
readLiteral(input, next, false);
input.acceptToken(QuotedIdentifier);
} else if (next === Ch.Colon || next === Ch.Comma) {
input.acceptToken(Punctuation);
} else if (isAlpha(next)) {
const word = readWord(input, String.fromCharCode(next));
input.acceptToken(
input.next === Ch.Dot ? Identifier : (d.words[word.toLowerCase()] ?? Identifier),
);
}
});
}
export const tokens = tokensFor(defaults);

View File

@@ -0,0 +1,200 @@
import type { CompletionResult, CompletionSource } from '@codemirror/autocomplete';
import { CompletionContext } from '@codemirror/autocomplete';
import { EditorState } from '@codemirror/state';
import type { SQLConfig } from '../src/sql';
import { MySQL, PostgreSQL, schemaCompletionSource } from '../src/sql';
function get(doc: string, conf: SQLConfig & { explicit?: boolean } = {}) {
const cur = doc.indexOf('|');
const dialect = conf.dialect || PostgreSQL;
doc = doc.slice(0, cur) + doc.slice(cur + 1);
const state = EditorState.create({
doc,
selection: { anchor: cur },
extensions: [
dialect,
dialect.sqlLanguage.data.of({
autocomplete: schemaCompletionSource(Object.assign({ dialect }, conf)),
}),
],
});
const result = state.languageDataAt<CompletionSource>('autocomplete', cur)[0](
new CompletionContext(state, cur, !!conf.explicit),
);
return result as CompletionResult | null;
}
function str(result: CompletionResult | null) {
return !result
? ''
: result.options
.slice()
.sort((a, b) => (b.boost || 0) - (a.boost || 0) || (a.label < b.label ? -1 : 1))
.map((o) => o.label)
.join(', ');
}
const schema1 = {
users: ['name', 'id', 'address'],
products: ['name', 'cost', 'description'],
};
const schema2 = {
'public.users': ['email', 'id'],
'other.users': ['name', 'id'],
};
describe('SQL completion', () => {
it('completes table names', () => {
expect(str(get('select u|', { schema: schema1 }))).toEqual('products, users');
});
it('completes quoted table names', () => {
expect(str(get('select "u|', { schema: schema1 }))).toEqual('"products", "users"');
});
it('completes table names under schema', () => {
expect(str(get('select public.u|', { schema: schema2 }))).toEqual('users');
});
it('completes quoted table names under schema', () => {
expect(str(get('select public."u|', { schema: schema2 }))).toEqual('"users"');
});
it('completes quoted table names under quoted schema', () => {
expect(str(get('select "public"."u|', { schema: schema2 }))).toEqual('"users"');
});
it('completes column names', () => {
expect(str(get('select users.|', { schema: schema1 }))).toEqual('address, id, name');
});
it('completes quoted column names', () => {
expect(str(get('select users."|', { schema: schema1 }))).toEqual('"address", "id", "name"');
});
it('completes column names in quoted tables', () => {
expect(str(get('select "users".|', { schema: schema1 }))).toEqual('address, id, name');
});
it('completes column names in tables for a specific schema', () => {
expect(str(get('select public.users.|', { schema: schema2 }))).toEqual('email, id');
expect(str(get('select other.users.|', { schema: schema2 }))).toEqual('id, name');
});
it('completes quoted column names in tables for a specific schema', () => {
expect(str(get('select public.users."|', { schema: schema2 }))).toEqual('"email", "id"');
expect(str(get('select other.users."|', { schema: schema2 }))).toEqual('"id", "name"');
});
it('completes column names in quoted tables for a specific schema', () => {
expect(str(get('select public."users".|', { schema: schema2 }))).toEqual('email, id');
expect(str(get('select other."users".|', { schema: schema2 }))).toEqual('id, name');
});
it('completes column names in quoted tables for a specific quoted schema', () => {
expect(str(get('select "public"."users".|', { schema: schema2 }))).toEqual('email, id');
expect(str(get('select "other"."users".|', { schema: schema2 }))).toEqual('id, name');
});
it('completes quoted column names in quoted tables for a specific quoted schema', () => {
expect(str(get('select "public"."users"."|', { schema: schema2 }))).toEqual('"email", "id"');
expect(str(get('select "other"."users"."|', { schema: schema2 }))).toEqual('"id", "name"');
});
it('completes column names of aliased tables', () => {
expect(str(get('select u.| from users u', { schema: schema1 }))).toEqual('address, id, name');
expect(str(get('select u.| from users as u', { schema: schema1 }))).toEqual(
'address, id, name',
);
expect(
str(get('select u.| from (SELECT * FROM something u) join users u', { schema: schema1 })),
).toEqual('address, id, name');
expect(str(get('select * from users u where u.|', { schema: schema1 }))).toEqual(
'address, id, name',
);
expect(str(get('select * from users as u where u.|', { schema: schema1 }))).toEqual(
'address, id, name',
);
expect(
str(
get('select * from (SELECT * FROM something u) join users u where u.|', {
schema: schema1,
}),
),
).toEqual('address, id, name');
});
it('completes column names of aliased quoted tables', () => {
expect(str(get('select u.| from "users" u', { schema: schema1 }))).toEqual('address, id, name');
expect(str(get('select u.| from "users" as u', { schema: schema1 }))).toEqual(
'address, id, name',
);
expect(str(get('select * from "users" u where u.|', { schema: schema1 }))).toEqual(
'address, id, name',
);
expect(str(get('select * from "users" as u where u.|', { schema: schema1 }))).toEqual(
'address, id, name',
);
});
it('completes column names of aliased tables for a specific schema', () => {
expect(str(get('select u.| from public.users u', { schema: schema2 }))).toEqual('email, id');
});
it('completes column names in aliased quoted tables for a specific schema', () => {
expect(str(get('select u.| from public."users" u', { schema: schema2 }))).toEqual('email, id');
});
it('completes column names in aliased quoted tables for a specific quoted schema', () => {
expect(str(get('select u.| from "public"."users" u', { schema: schema2 }))).toEqual(
'email, id',
);
});
it('completes aliased table names', () => {
expect(str(get('select a| from a.b as ab join auto au', { schema: schema2 }))).toEqual(
'ab, au, other, public',
);
});
it('includes closing quote in completion', () => {
const r = get('select "u|"', { schema: schema1 });
expect(r!.to).toEqual(10);
});
it('keeps extra table completion properties', () => {
const r = get('select u|', {
schema: { users: ['id'] },
tables: [{ label: 'users', type: 'keyword' }],
});
expect(r!.options[0].type).toEqual('keyword');
});
it('keeps extra column completion properties', () => {
const r = get('select users.|', { schema: { users: [{ label: 'id', type: 'keyword' }] } });
expect(r!.options[0].type).toEqual('keyword');
});
it('supports a default table', () => {
expect(str(get('select i|', { schema: schema1, defaultTable: 'users' }))).toEqual(
'address, id, name, products, users',
);
});
it('supports alternate quoting styles', () => {
expect(str(get('select `u|', { dialect: MySQL, schema: schema1 }))).toEqual(
'`products`, `users`',
);
});
it("doesn't complete without identifier", () => {
expect(str(get('select |', { schema: schema1 }))).toEqual('');
});
it('does complete explicitly without identifier', () => {
expect(str(get('select |', { schema: schema1, explicit: true }))).toEqual('products, users');
});
});

View File

@@ -0,0 +1,151 @@
import type { LRParser } from '@lezer/lr';
import { MySQL, PostgreSQL, SQLDialect } from '../src';
const mysqlTokens = MySQL.language;
const postgresqlTokens = PostgreSQL.language;
const bigQueryTokens = SQLDialect.define({
treatBitsAsBytes: true,
}).language;
const parse = (parser: LRParser, input: string) => {
const tree = parser.parse(input);
const props: Record<string, { tree: unknown }> = (
tree as unknown as { props: Record<string, { tree: unknown }> }
).props;
const key = Object.keys(props)[0];
return String(props[key].tree);
};
const parseMixed = (parser: LRParser, input: string) => {
return String(parser.parse(input) as unknown);
};
describe('Parse MySQL tokens', () => {
const parser = mysqlTokens.parser;
it('parses quoted bit-value literals', () => {
expect(parse(parser, "SELECT b'0101'")).toEqual('Script(Statement(Keyword,Whitespace,Bits))');
});
it('parses unquoted bit-value literals', () => {
expect(parse(parser, 'SELECT 0b01')).toEqual('Script(Statement(Keyword,Whitespace,Bits))');
});
});
describe('Parse PostgreSQL tokens', () => {
const parser = postgresqlTokens.parser;
it('parses quoted bit-value literals', () => {
expect(parse(parser, "SELECT b'0101'")).toEqual('Script(Statement(Keyword,Whitespace,Bits))');
});
it('parses quoted bit-value literals', () => {
expect(parse(parser, "SELECT B'0101'")).toEqual('Script(Statement(Keyword,Whitespace,Bits))');
});
it('parses double dollar quoted Whitespace literals', () => {
expect(parse(parser, 'SELECT $$hello$$')).toEqual(
'Script(Statement(Keyword,Whitespace,String))',
);
});
});
describe('Parse BigQuery tokens', () => {
const parser = bigQueryTokens.parser;
it('parses quoted bytes literals in single quotes', () => {
expect(parse(parser, "SELECT b'abcd'")).toEqual('Script(Statement(Keyword,Whitespace,Bytes))');
});
it('parses quoted bytes literals in double quotes', () => {
expect(parse(parser, 'SELECT b"abcd"')).toEqual('Script(Statement(Keyword,Whitespace,Bytes))');
});
it('parses bytes literals in single quotes', () => {
expect(parse(parser, "SELECT b'0101'")).toEqual('Script(Statement(Keyword,Whitespace,Bytes))');
});
it('parses bytes literals in double quotes', () => {
expect(parse(parser, 'SELECT b"0101"')).toEqual('Script(Statement(Keyword,Whitespace,Bytes))');
});
});
describe('Parse n8n resolvables', () => {
const parser = postgresqlTokens.parser;
it('parses resolvables with dots inside composite identifiers', () => {
expect(parseMixed(parser, "SELECT my_column FROM {{ 'schema' }}.{{ 'table' }}")).toEqual(
'Program(Plaintext,Resolvable,Plaintext,Resolvable)',
);
expect(
parseMixed(parser, "SELECT my_column FROM {{ 'schema' }}.{{ 'table' }}.{{ 'foo' }}"),
).toEqual('Program(Plaintext,Resolvable,Plaintext,Resolvable,Plaintext,Resolvable)');
expect(parseMixed(parser, "SELECT my_column FROM public.{{ 'table' }}")).toEqual(
'Program(Plaintext,Resolvable)',
);
expect(parseMixed(parser, "SELECT my_column FROM {{ 'schema' }}.users")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
});
it('parses 4-node SELECT variants', () => {
expect(parseMixed(parser, "{{ 'SELECT' }} my_column FROM my_table")).toEqual(
'Program(Resolvable,Plaintext)',
);
expect(parseMixed(parser, "SELECT {{ 'my_column' }} FROM my_table")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
expect(parseMixed(parser, "SELECT my_column {{ 'FROM' }} my_table")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
expect(parseMixed(parser, "SELECT my_column FROM {{ 'my_table' }}")).toEqual(
'Program(Plaintext,Resolvable)',
);
});
it('parses 5-node SELECT variants (with semicolon)', () => {
expect(parseMixed(parser, "{{ 'SELECT' }} my_column FROM my_table;")).toEqual(
'Program(Resolvable,Plaintext)',
);
expect(parseMixed(parser, "SELECT {{ 'my_column' }} FROM my_table;")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
expect(parseMixed(parser, "SELECT my_column {{ 'FROM' }} my_table;")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
expect(parseMixed(parser, "SELECT my_column FROM {{ 'my_table' }};")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
});
it('parses single-quoted resolvable with no whitespace', () => {
expect(parseMixed(parser, "SELECT my_column FROM '{{ 'my_table' }}';")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
});
it('parses single-quoted resolvable with leading whitespace', () => {
expect(parseMixed(parser, "SELECT my_column FROM ' {{ 'my_table' }}';")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
});
it('parses single-quoted resolvable with trailing whitespace', () => {
expect(parseMixed(parser, "SELECT my_column FROM '{{ 'my_table' }} ';")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
});
it('parses single-quoted resolvable with surrounding whitespace', () => {
expect(parseMixed(parser, "SELECT my_column FROM ' {{ 'my_table' }} ';")).toEqual(
'Program(Plaintext,Resolvable,Plaintext)',
);
});
});

View File

@@ -0,0 +1,10 @@
{
"extends": ["./tsconfig.json", "@n8n/typescript-config/tsconfig.build.json"],
"compilerOptions": {
"rootDir": "src",
"outDir": "dist",
"tsBuildInfoFile": "dist/build.tsbuildinfo"
},
"include": ["src/**/*.ts"],
"exclude": ["test/**"]
}

View File

@@ -0,0 +1,8 @@
{
"extends": "@n8n/typescript-config/tsconfig.common.json",
"compilerOptions": {
"rootDir": ".",
"tsBuildInfoFile": "dist/typecheck.tsbuildinfo"
},
"include": ["src/**/*.ts", "test/**/*.ts"]
}

View File

@@ -3,6 +3,8 @@ import { styleTags, tags as t } from '@lezer/highlight';
import { parser } from './grammar';
export const expressionParser = parser;
export const parserWithMetaData = parser.configure({
props: [
foldNodeProp.add({

View File

@@ -1 +1 @@
export { parserWithMetaData, n8nLanguage } from './expressions';
export { parserWithMetaData, n8nLanguage, expressionParser } from './expressions';

View File

@@ -37,7 +37,7 @@
"@n8n/api-types": "workspace:*",
"@n8n/chat": "workspace:*",
"@n8n/codemirror-lang": "workspace:*",
"@n8n/codemirror-lang-sql": "^1.0.2",
"@n8n/codemirror-lang-sql": "workspace:*",
"@n8n/composables": "workspace:*",
"@n8n/constants": "workspace:*",
"@n8n/design-system": "workspace:*",

View File

@@ -6,9 +6,8 @@ import { n8nCompletionSources } from '@/plugins/codemirror/completions/addComple
import { dropInExpressionEditor, mappingDropCursor } from '@/plugins/codemirror/dragAndDrop';
import { editorKeymap } from '@/plugins/codemirror/keymap';
import { n8nAutocompletion } from '@/plugins/codemirror/n8nLang';
import { ifNotIn } from '@codemirror/autocomplete';
import { history } from '@codemirror/commands';
import { LanguageSupport, bracketMatching, foldGutter, indentOnInput } from '@codemirror/language';
import { bracketMatching, foldGutter, indentOnInput, LanguageSupport } from '@codemirror/language';
import { Prec, type Line } from '@codemirror/state';
import {
EditorView,
@@ -79,9 +78,9 @@ const extensions = computed(() => {
const dialect = SQL_DIALECTS[props.dialect] ?? SQL_DIALECTS.StandardSQL;
function sqlWithN8nLanguageSupport() {
return new LanguageSupport(dialect.language, [
dialect.language.data.of({ closeBrackets: expressionCloseBracketsConfig }),
dialect.language.data.of({
autocomplete: ifNotIn(['Resolvable'], keywordCompletionSource(dialect, true)),
dialect.sqlLanguage.data.of({ closeBrackets: expressionCloseBracketsConfig }),
dialect.sqlLanguage.data.of({
autocomplete: keywordCompletionSource(dialect, true),
}),
n8nCompletionSources().map((source) => dialect.language.data.of(source)),
]);
@@ -114,6 +113,7 @@ const extensions = computed(() => {
mappingDropCursor(),
]);
}
return baseExtensions;
});
const {