admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 10:02:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): Prevent race condition in tiktoken encoding cache (no-changelog) (#18780)

Browse Source 
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
jeanpaul
2025-08-26 13:47:42 +02:00
committed by GitHub
parent 6cccc4ab9f
commit df9521ff8e
1 changed files with 24 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
packages/@n8n/nodes-langchain/utils/tokenizer/tiktoken.ts
View File

@@ -4,7 +4,7 @@ import { Tiktoken, getEncodingNameForModel } from 'js-tiktoken/lite';
import { jsonParse } from 'n8n-workflow'; import { jsonParse } from 'n8n-workflow';
import { join } from 'path'; import { join } from 'path';
const cache: Record<string, Tiktoken> = {}; const cache: Record<string, Promise<Tiktoken>> = {};
const loadJSONFile = async (filename: string): Promise<TiktokenBPE> => { const loadJSONFile = async (filename: string): Promise<TiktokenBPE> => {
const filePath = join(__dirname, filename); const filePath = join(__dirname, filename);
@@ -13,10 +13,9 @@ const loadJSONFile = async (filename: string): Promise<TiktokenBPE> => {
}; };
export async function getEncoding(encoding: TiktokenEncoding): Promise<Tiktoken> { export async function getEncoding(encoding: TiktokenEncoding): Promise<Tiktoken> {
if (cache[encoding]) { if (!(encoding in cache)) {
return cache[encoding]; // Create and cache the promise for loading this encoding
} cache[encoding] = (async () => {
let jsonData: TiktokenBPE; let jsonData: TiktokenBPE;
switch (encoding) { switch (encoding) {
@@ -31,8 +30,14 @@ export async function getEncoding(encoding: TiktokenEncoding): Promise<Tiktoken>
jsonData = await loadJSONFile('./cl100k_base.json'); jsonData = await loadJSONFile('./cl100k_base.json');
} }
cache[encoding] = new Tiktoken(jsonData); return new Tiktoken(jsonData);
return cache[encoding]; })().catch((error) => {
delete cache[encoding];
throw error;
});
}
return await cache[encoding];
} }
export async function encodingForModel(model: TiktokenModel): Promise<Tiktoken> { export async function encodingForModel(model: TiktokenModel): Promise<Tiktoken> {