admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 01:56:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(Token Splitter Node): Prevent tiktoken blocking on repetitive content (#16769)

Browse Source
This commit is contained in:
oleg
2025-06-27 16:08:14 +02:00
committed by GitHub
parent edf0fec444
commit c5ec056eb5
7 changed files with 812 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
packages/@n8n/nodes-langchain/nodes/llms/N8nLlmTracing.ts
View File

@@ -13,7 +13,7 @@ import type { IDataObject, ISupplyDataFunctions, JsonObject } from 'n8n-workflow
import { NodeConnectionTypes, NodeError, NodeOperationError } from 'n8n-workflow';
import { logAiEvent } from '@utils/helpers';
import { encodingForModel } from '@utils/tokenizer/tiktoken';
import { estimateTokensFromStringList } from '@utils/tokenizer/token-estimator';
type TokensUsageParser = (llmOutput: LLMResult['llmOutput']) => {
completionTokens: number;
@@ -84,13 +84,7 @@ export class N8nLlmTracing extends BaseCallbackHandler {
async estimateTokensFromStringList(list: string[]) {
const embeddingModel = getModelNameForTiktoken(TIKTOKEN_ESTIMATE_MODEL);
const encoder = await encodingForModel(embeddingModel);
const encodedListLength = await Promise.all(
list.map(async (text) => encoder.encode(text).length),
);
return encodedListLength.reduce((acc, curr) => acc + curr, 0);
return await estimateTokensFromStringList(list, embeddingModel);
}
async handleLLMEnd(output: LLMResult, runId: string) {