From 4f07ac394b99e99469c5cca5558e950a08d9e171 Mon Sep 17 00:00:00 2001 From: Jaakko Husso Date: Wed, 4 Jun 2025 12:07:09 +0300 Subject: [PATCH] feat(Structured Output Parser Node): Add auto-fix support to Strucutred Output Parser (#15915) --- .../OutputParserAutofixing.node.ts | 2 +- .../OutputParserStructured.node.ts | 106 +++++++++- .../OutputParserStructured/prompt.ts | 16 ++ .../test/OutputParserStructured.node.test.ts | 197 +++++++++++++++++- .../frontend/@n8n/i18n/src/locales/en.json | 1 + .../Node/NodeCreator/ItemTypes/NodeItem.vue | 12 +- 6 files changed, 322 insertions(+), 12 deletions(-) create mode 100644 packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/prompt.ts diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/OutputParserAutofixing.node.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/OutputParserAutofixing.node.ts index 39d8d00e7e..feacb62c3d 100644 --- a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/OutputParserAutofixing.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserAutofixing/OutputParserAutofixing.node.ts @@ -24,7 +24,7 @@ export class OutputParserAutofixing implements INodeType { iconColor: 'black', group: ['transform'], version: 1, - description: 'Automatically fix the output if it is not in the correct format', + description: 'Deprecated, use structured output parser', defaults: { name: 'Auto-fixing Output Parser', }, diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts index 0367427a4e..af089f1c7b 100644 --- a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts @@ -1,3 +1,5 @@ +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import { PromptTemplate } from '@langchain/core/prompts'; import type { JSONSchema7 } from 'json-schema'; import { jsonParse, @@ -11,10 +13,15 @@ import { import type { z } from 'zod'; import { inputSchemaField, jsonSchemaExampleField, schemaTypeField } from '@utils/descriptions'; -import { N8nStructuredOutputParser } from '@utils/output_parsers/N8nOutputParser'; +import { + N8nOutputFixingParser, + N8nStructuredOutputParser, +} from '@utils/output_parsers/N8nOutputParser'; import { convertJsonSchemaToZod, generateSchema } from '@utils/schemaParsing'; import { getConnectionHintNoticeField } from '@utils/sharedFields'; +import { NAIVE_FIX_PROMPT } from './prompt'; + export class OutputParserStructured implements INodeType { description: INodeTypeDescription = { displayName: 'Structured Output Parser', @@ -43,8 +50,17 @@ export class OutputParserStructured implements INodeType { ], }, }, - // eslint-disable-next-line n8n-nodes-base/node-class-description-inputs-wrong-regular-node - inputs: [], + inputs: `={{ + ((parameters) => { + if (parameters?.autoFix) { + return [ + { displayName: 'Model', maxConnections: 1, type: "${NodeConnectionTypes.AiLanguageModel}", required: true } + ]; + } + + return []; + })($parameter) + }}`, // eslint-disable-next-line n8n-nodes-base/node-class-description-outputs-wrong outputs: [NodeConnectionTypes.AiOutputParser], outputNames: ['Output Parser'], @@ -116,6 +132,45 @@ export class OutputParserStructured implements INodeType { }, }, }, + { + displayName: 'Auto-Fix Format', + description: + 'Whether to automatically fix the output when it is not in the correct format. Will cause another LLM call.', + name: 'autoFix', + type: 'boolean', + default: false, + }, + { + displayName: 'Customize Retry Prompt', + name: 'customizeRetryPrompt', + type: 'boolean', + displayOptions: { + show: { + autoFix: [true], + }, + }, + default: false, + description: + 'Whether to customize the prompt used for retrying the output parsing. If disabled, a default prompt will be used.', + }, + { + displayName: 'Custom Prompt', + name: 'prompt', + type: 'string', + displayOptions: { + show: { + autoFix: [true], + customizeRetryPrompt: [true], + }, + }, + default: NAIVE_FIX_PROMPT, + typeOptions: { + rows: 10, + }, + hint: 'Should include "{error}", "{instructions}", and "{completion}" placeholders', + description: + 'Prompt template used for fixing the output. Uses placeholders: "{instructions}" for parsing rules, "{completion}" for the failed attempt, and "{error}" for the validation error message.', + }, ], }; @@ -124,6 +179,7 @@ export class OutputParserStructured implements INodeType { // We initialize these even though one of them will always be empty // it makes it easer to navigate the ternary operator const jsonExample = this.getNodeParameter('jsonSchemaExample', itemIndex, '') as string; + let inputSchema: string; if (this.getNode().typeVersion <= 1.1) { @@ -137,17 +193,51 @@ export class OutputParserStructured implements INodeType { const zodSchema = convertJsonSchemaToZod>(jsonSchema); const nodeVersion = this.getNode().typeVersion; + + const autoFix = this.getNodeParameter('autoFix', itemIndex, false) as boolean; + + let outputParser; try { - const parser = await N8nStructuredOutputParser.fromZodJsonSchema( + outputParser = await N8nStructuredOutputParser.fromZodJsonSchema( zodSchema, nodeVersion, this, ); - return { - response: parser, - }; } catch (error) { - throw new NodeOperationError(this.getNode(), 'Error during parsing of JSON Schema.'); + throw new NodeOperationError( + this.getNode(), + 'Error during parsing of JSON Schema. Please check the schema and try again.', + ); } + + if (!autoFix) { + return { + response: outputParser, + }; + } + + const model = (await this.getInputConnectionData( + NodeConnectionTypes.AiLanguageModel, + itemIndex, + )) as BaseLanguageModel; + + const prompt = this.getNodeParameter('prompt', itemIndex, NAIVE_FIX_PROMPT) as string; + + if (prompt.length === 0 || !prompt.includes('{error}')) { + throw new NodeOperationError( + this.getNode(), + 'Auto-fixing parser prompt has to contain {error} placeholder', + ); + } + const parser = new N8nOutputFixingParser( + this, + model, + outputParser, + PromptTemplate.fromTemplate(prompt), + ); + + return { + response: parser, + }; } } diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/prompt.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/prompt.ts new file mode 100644 index 0000000000..9e4431a68c --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/prompt.ts @@ -0,0 +1,16 @@ +export const NAIVE_FIX_PROMPT = `Instructions: +-------------- +{instructions} +-------------- +Completion: +-------------- +{completion} +-------------- + +Above, the Completion did not satisfy the constraints given in the Instructions. +Error: +-------------- +{error} +-------------- + +Please try again. Please only respond with an answer that satisfies the constraints laid out in the Instructions:`; diff --git a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/test/OutputParserStructured.node.test.ts b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/test/OutputParserStructured.node.test.ts index 67e5d63cdc..2ed5db7186 100644 --- a/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/test/OutputParserStructured.node.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/test/OutputParserStructured.node.test.ts @@ -1,15 +1,23 @@ -import { mock } from 'jest-mock-extended'; +import type { BaseLanguageModel } from '@langchain/core/language_models/base'; +import { OutputParserException } from '@langchain/core/output_parsers'; +import { mock, type MockProxy } from 'jest-mock-extended'; import { normalizeItems } from 'n8n-core'; import { jsonParse, + NodeConnectionTypes, + NodeOperationError, type INode, type ISupplyDataFunctions, type IWorkflowDataProxyData, } from 'n8n-workflow'; -import type { N8nStructuredOutputParser } from '@utils/output_parsers/N8nStructuredOutputParser'; +import { + N8nStructuredOutputParser, + type N8nOutputFixingParser, +} from '@utils/output_parsers/N8nOutputParser'; import { OutputParserStructured } from '../OutputParserStructured.node'; +import { NAIVE_FIX_PROMPT } from '../prompt'; describe('OutputParserStructured', () => { let outputParser: OutputParserStructured; @@ -388,6 +396,7 @@ describe('OutputParserStructured', () => { ), ).rejects.toThrow('Required'); }); + it('should throw on wrong type', async () => { const schema = `{ "type": "object", @@ -464,4 +473,188 @@ describe('OutputParserStructured', () => { }); }); }); + + describe('Auto-Fix', () => { + const model: BaseLanguageModel = jest.fn() as unknown as BaseLanguageModel; + + beforeEach(() => { + thisArg.getNodeParameter.calledWith('schemaType', 0).mockReturnValueOnce('fromJson'); + thisArg.getNodeParameter.calledWith('jsonSchemaExample', 0).mockReturnValueOnce(`{ + "user": { + "name": "Alice" + } + }`); + thisArg.getNode.mockReturnValue(mock({ typeVersion: 1.2 })); + thisArg.getInputConnectionData + .calledWith(NodeConnectionTypes.AiLanguageModel, 0) + .mockResolvedValueOnce(model); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('Configuration', () => { + it('should use default prompt when none specified', async () => { + thisArg.getNodeParameter.calledWith('autoFix', 0, false).mockReturnValueOnce(true); + thisArg.getNodeParameter + .calledWith('prompt', 0, NAIVE_FIX_PROMPT) + .mockReturnValueOnce(NAIVE_FIX_PROMPT); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + expect(response).toBeDefined(); + }); + + it('should use custom prompt if one is provided', async () => { + thisArg.getNodeParameter.calledWith('autoFix', 0, false).mockReturnValueOnce(true); + thisArg.getNodeParameter + .calledWith('prompt', 0, NAIVE_FIX_PROMPT) + .mockReturnValueOnce( + 'Some prompt with "{error}", "{instructions}", and "{completion}" placeholders', + ); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + expect(response).toBeDefined(); + }); + + it('should throw error when prompt template does not contain {error} placeholder', async () => { + thisArg.getNodeParameter.calledWith('autoFix', 0, false).mockReturnValueOnce(true); + thisArg.getNodeParameter + .calledWith('prompt', 0, NAIVE_FIX_PROMPT) + .mockReturnValueOnce('Invalid prompt without error placeholder'); + + await expect(outputParser.supplyData.call(thisArg, 0)).rejects.toThrow( + new NodeOperationError( + thisArg.getNode(), + 'Auto-fixing parser prompt has to contain {error} placeholder', + ), + ); + }); + + it('should throw error when prompt template is empty', async () => { + thisArg.getNodeParameter.calledWith('autoFix', 0, false).mockReturnValueOnce(true); + thisArg.getNodeParameter.calledWith('prompt', 0, NAIVE_FIX_PROMPT).mockReturnValueOnce(''); + + await expect(outputParser.supplyData.call(thisArg, 0)).rejects.toThrow( + new NodeOperationError( + thisArg.getNode(), + 'Auto-fixing parser prompt has to contain {error} placeholder', + ), + ); + }); + }); + + describe('Parsing', () => { + let mockStructuredOutputParser: MockProxy; + + beforeEach(() => { + mockStructuredOutputParser = mock(); + + jest + .spyOn(N8nStructuredOutputParser, 'fromZodJsonSchema') + .mockResolvedValue(mockStructuredOutputParser); + + thisArg.getNodeParameter.calledWith('autoFix', 0, false).mockReturnValueOnce(true); + thisArg.getNodeParameter + .calledWith('prompt', 0, NAIVE_FIX_PROMPT) + .mockReturnValueOnce(NAIVE_FIX_PROMPT); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + function getMockedRetryChain(output: string) { + return jest.fn().mockReturnValue({ + invoke: jest.fn().mockResolvedValue({ + content: output, + }), + }); + } + + it('should successfully parse valid output without needing to fix it', async () => { + const validOutput = { name: 'Alice', age: 25 }; + + mockStructuredOutputParser.parse.mockResolvedValueOnce(validOutput); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + const result = await response.parse('{"name": "Alice", "age": 25}'); + + expect(result).toEqual(validOutput); + expect(mockStructuredOutputParser.parse.mock.calls).toHaveLength(1); + }); + + it('should not retry on non-OutputParserException errors', async () => { + const error = new Error('Some other error'); + mockStructuredOutputParser.parse.mockRejectedValueOnce(error); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + await expect(response.parse('Invalid JSON string')).rejects.toThrow(error); + expect(mockStructuredOutputParser.parse.mock.calls).toHaveLength(1); + }); + + it('should retry on OutputParserException and succeed', async () => { + const validOutput = { name: 'Bob', age: 28 }; + + mockStructuredOutputParser.parse + .mockRejectedValueOnce(new OutputParserException('Invalid JSON')) + .mockResolvedValueOnce(validOutput); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + response.getRetryChain = getMockedRetryChain(JSON.stringify(validOutput)); + + const result = await response.parse('Invalid JSON string'); + + expect(result).toEqual(validOutput); + expect(mockStructuredOutputParser.parse.mock.calls).toHaveLength(2); + }); + + it('should handle failed retry attempt', async () => { + mockStructuredOutputParser.parse + .mockRejectedValueOnce(new OutputParserException('Invalid JSON')) + .mockRejectedValueOnce(new Error('Still invalid JSON')); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + response.getRetryChain = getMockedRetryChain('Still not valid JSON'); + + await expect(response.parse('Invalid JSON string')).rejects.toThrow('Still invalid JSON'); + expect(mockStructuredOutputParser.parse.mock.calls).toHaveLength(2); + }); + + it('should throw non-OutputParserException errors immediately without retry', async () => { + const customError = new Error('Database connection error'); + const retryChainSpy = jest.fn(); + + mockStructuredOutputParser.parse.mockRejectedValueOnce(customError); + + const { response } = (await outputParser.supplyData.call(thisArg, 0)) as { + response: N8nOutputFixingParser; + }; + + response.getRetryChain = retryChainSpy; + + await expect(response.parse('Some input')).rejects.toThrow('Database connection error'); + expect(mockStructuredOutputParser.parse.mock.calls).toHaveLength(1); + expect(retryChainSpy).not.toHaveBeenCalled(); + }); + }); + }); }); diff --git a/packages/frontend/@n8n/i18n/src/locales/en.json b/packages/frontend/@n8n/i18n/src/locales/en.json index 39eeae0ca0..c57cf9489d 100644 --- a/packages/frontend/@n8n/i18n/src/locales/en.json +++ b/packages/frontend/@n8n/i18n/src/locales/en.json @@ -1341,6 +1341,7 @@ "nodeCreator.aiPanel.workflowTriggerDescription": "Runs the flow when called by the Execute Workflow node from a different workflow", "nodeCreator.nodeItem.triggerIconTitle": "Trigger Node", "nodeCreator.nodeItem.aiIconTitle": "LangChain AI Node", + "nodeCreator.nodeItem.deprecated": "Deprecated", "nodeCredentials.createNew": "Create new credential", "nodeCredentials.credentialFor": "Credential for {credentialType}", "nodeCredentials.credentialsLabel": "Credential to connect with", diff --git a/packages/frontend/editor-ui/src/components/Node/NodeCreator/ItemTypes/NodeItem.vue b/packages/frontend/editor-ui/src/components/Node/NodeCreator/ItemTypes/NodeItem.vue index 56e07d49ec..278a5f40e7 100644 --- a/packages/frontend/editor-ui/src/components/Node/NodeCreator/ItemTypes/NodeItem.vue +++ b/packages/frontend/editor-ui/src/components/Node/NodeCreator/ItemTypes/NodeItem.vue @@ -124,6 +124,16 @@ const author = computed(() => { return communityNodeType.value?.displayName ?? displayName.value; }); +const tag = computed(() => { + if (props.nodeType.tag) { + return { text: props.nodeType.tag }; + } + if (description.value.toLowerCase().includes('deprecated')) { + return { text: i18n.baseText('nodeCreator.nodeItem.deprecated'), type: 'info' }; + } + return undefined; +}); + function onDragStart(event: DragEvent): void { if (event.dataTransfer) { event.dataTransfer.effectAllowed = 'copy'; @@ -163,7 +173,7 @@ function onCommunityNodeTooltipClick(event: MouseEvent) { :is-trigger="isTrigger" :is-official="isOfficial" :data-test-id="dataTestId" - :tag="nodeType.tag" + :tag="tag" @dragstart="onDragStart" @dragend="onDragEnd" >