admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-17 01:56:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): Prevent race condition in tiktoken encoding cache (no-changelog) (#18780)

Browse Source 
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
jeanpaul
2025-08-26 13:47:42 +02:00
committed by GitHub
parent 6cccc4ab9f
commit df9521ff8e
1 changed files with 24 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
packages/@n8n/nodes-langchain/utils/tokenizer/tiktoken.ts
View File

@@ -4,7 +4,7 @@ import { Tiktoken, getEncodingNameForModel } from 'js-tiktoken/lite';
import { jsonParse } from 'n8n-workflow';
import { join } from 'path';
const cache: Record<string, Tiktoken> = {};
const cache: Record<string, Promise<Tiktoken>> = {};
const loadJSONFile = async (filename: string): Promise<TiktokenBPE> => {
const filePath = join(__dirname, filename);
@@ -13,26 +13,31 @@ const loadJSONFile = async (filename: string): Promise<TiktokenBPE> => {
};
export async function getEncoding(encoding: TiktokenEncoding): Promise<Tiktoken> {
if (cache[encoding]) {
return cache[encoding];
if (!(encoding in cache)) {
// Create and cache the promise for loading this encoding
cache[encoding] = (async () => {
let jsonData: TiktokenBPE;
switch (encoding) {
case 'o200k_base':
jsonData = await loadJSONFile('./o200k_base.json');
break;
case 'cl100k_base':
jsonData = await loadJSONFile('./cl100k_base.json');
break;
default:
// Fall back to cl100k_base for unsupported encodings
jsonData = await loadJSONFile('./cl100k_base.json');
}
return new Tiktoken(jsonData);
})().catch((error) => {
delete cache[encoding];
throw error;
});
}
let jsonData: TiktokenBPE;
switch (encoding) {
case 'o200k_base':
jsonData = await loadJSONFile('./o200k_base.json');
break;
case 'cl100k_base':
jsonData = await loadJSONFile('./cl100k_base.json');
break;
default:
// Fall back to cl100k_base for unsupported encodings
jsonData = await loadJSONFile('./cl100k_base.json');
}
cache[encoding] = new Tiktoken(jsonData);
return cache[encoding];
return await cache[encoding];
}
export async function encodingForModel(model: TiktokenModel): Promise<Tiktoken> {