admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-16 17:46:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add planning step to AI workflow builder (no-changelog) (#18737)

Browse Source 
Co-authored-by: Eugene Molodkin <eugene@n8n.io>
This commit is contained in:
oleg
2025-09-01 16:28:19 +02:00
committed by GitHub
parent caeaa679c6
commit 94f0048f02
31 changed files with 2977 additions and 1281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
packages/@n8n/ai-workflow-builder.ee/evaluations/core/test-runner.ts
View File

@@ -1,11 +1,13 @@
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
import type { SimpleWorkflow } from '../../src/types/workflow.js';
import type { WorkflowBuilderAgent, ChatPayload } from '../../src/workflow-builder-agent.js';
import { evaluateWorkflow } from '../chains/workflow-evaluator.js';
import type { EvaluationInput, EvaluationResult, TestCase } from '../types/evaluation.js';
import { isWorkflowStateValues } from '../types/langsmith.js';
import type { TestResult } from '../types/test-result.js';
import { PLAN_APPROVAL_MESSAGE } from '../../src/constants';
import type { SimpleWorkflow } from '../../src/types/workflow';
import type { WorkflowBuilderAgent } from '../../src/workflow-builder-agent';
import { evaluateWorkflow } from '../chains/workflow-evaluator';
import type { EvaluationInput, EvaluationResult, TestCase } from '../types/evaluation';
import { isWorkflowStateValues } from '../types/langsmith';
import type { TestResult } from '../types/test-result';
import { consumeGenerator, getChatPayload } from '../utils/evaluation-helpers';
/**
* Creates an error result for a failed test
@@ -48,19 +50,12 @@ export async function runSingleTest(
userId: string = 'test-user',
): Promise<TestResult> {
try {
const chatPayload: ChatPayload = {
message: testCase.prompt,
workflowContext: {
currentWorkflow: { id: testCase.id, nodes: [], connections: {} },
},
};
// Generate workflow
const startTime = Date.now();
let messageCount = 0;
for await (const _output of agent.chat(chatPayload, userId)) {
messageCount++;
}
// First generate plan
await consumeGenerator(agent.chat(getChatPayload(testCase.prompt, testCase.id), userId));
// Confirm plan
await consumeGenerator(agent.chat(getChatPayload(PLAN_APPROVAL_MESSAGE, testCase.id), userId));
const generationTime = Date.now() - startTime;
// Get generated workflow with validation

3
packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/evaluator.ts
View File

@@ -138,7 +138,8 @@ export function createLangsmithEvaluator(
for (const metric of usageMetrics) {
if (metric.value !== undefined) {
results.push({ key: metric.key, score: metric.value });
// Langsmith has a limitation on large scores (>99999) so we track in thousands
results.push({ key: metric.key, score: metric.value / 1000 });
}
}

38
packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/runner.ts
View File

@@ -1,20 +1,20 @@
import type { BaseChatModel } from '@langchain/core/language_models/chat_models.js';
import type { LangChainTracer } from '@langchain/core/tracers/tracer_langchain.js';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
import type { LangChainTracer } from '@langchain/core/tracers/tracer_langchain';
import { evaluate } from 'langsmith/evaluation';
import type { INodeTypeDescription } from 'n8n-workflow';
import pc from 'picocolors';
import { createLangsmithEvaluator } from './evaluator.js';
import type { ChatPayload } from '../../src/workflow-builder-agent.js';
import type { WorkflowState } from '../../src/workflow-state.js';
import { setupTestEnvironment, createAgent } from '../core/environment.js';
import { createLangsmithEvaluator } from './evaluator';
import { PLAN_APPROVAL_MESSAGE } from '../../src/constants';
import type { WorkflowState } from '../../src/workflow-state';
import { setupTestEnvironment, createAgent } from '../core/environment';
import {
generateRunId,
safeExtractUsage,
isWorkflowStateValues,
extractMessageContent,
} from '../types/langsmith.js';
import { formatHeader } from '../utils/evaluation-helpers.js';
} from '../types/langsmith';
import { consumeGenerator, formatHeader, getChatPayload } from '../utils/evaluation-helpers';
/**
* Creates a workflow generation function for Langsmith evaluation
@@ -44,18 +44,14 @@ function createWorkflowGenerator(
// Create agent for this run
const agent = createAgent(parsedNodeTypes, llm, tracer);
const chatPayload: ChatPayload = {
message: messageContent,
workflowContext: {
currentWorkflow: { id: runId, nodes: [], connections: {} },
},
};
// Generate workflow
let messageCount = 0;
for await (const _output of agent.chat(chatPayload, 'langsmith-eval-user')) {
messageCount++;
}
// First generate plan
await consumeGenerator(
agent.chat(getChatPayload(messageContent, runId), 'langsmith-eval-user'),
);
// Confirm plan
await consumeGenerator(
agent.chat(getChatPayload(PLAN_APPROVAL_MESSAGE, runId), 'langsmith-eval-user'),
);
// Get generated workflow with validation
const state = await agent.getState(runId, 'langsmith-eval-user');
@@ -77,7 +73,7 @@ function createWorkflowGenerator(
return {
workflow: generatedWorkflow,
prompt: chatPayload.message,
prompt: messageContent,
usage,
};
};

24
packages/@n8n/ai-workflow-builder.ee/evaluations/utils/evaluation-helpers.ts
View File

@@ -7,10 +7,11 @@ import type { INodeTypeDescription } from 'n8n-workflow';
import { join } from 'path';
import pc from 'picocolors';
import { anthropicClaudeSonnet4 } from '../../src/llm-config.js';
import { WorkflowBuilderAgent } from '../../src/workflow-builder-agent.js';
import type { Violation } from '../types/evaluation.js';
import type { TestResult } from '../types/test-result.js';
import { anthropicClaudeSonnet4 } from '../../src/llm-config';
import type { ChatPayload } from '../../src/workflow-builder-agent';
import { WorkflowBuilderAgent } from '../../src/workflow-builder-agent';
import type { Violation } from '../types/evaluation';
import type { TestResult } from '../types/test-result';
/**
* Sets up the LLM with proper configuration
@@ -268,3 +269,18 @@ export function saveEvaluationResults(
return { reportPath, resultsPath };
}
export async function consumeGenerator<T>(gen: AsyncGenerator<T>) {
for await (const _ of gen) {
/* consume all */
}
}
export function getChatPayload(message: string, id: string): ChatPayload {
return {
message,
workflowContext: {
currentWorkflow: { id, nodes: [], connections: {} },
},
};
}