admin/n8n-enterprise-unlocked
1
0
Fork 0
mirror of https://github.com/Abdulazizzn/n8n-enterprise-unlocked.git synced 2025-12-18 02:21:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(core): Convert tiktoken file loading to async (#18756)

Browse Source
This commit is contained in:
jeanpaul
2025-08-25 20:33:38 +02:00
committed by GitHub
parent fd12b3d5ce
commit e233dfa4a2
4 changed files with 37 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
packages/@n8n/nodes-langchain/utils/tokenizer/tiktoken.ts
View File

@@ -1,4 +1,4 @@
import { readFileSync } from 'fs';
import { readFile } from 'fs/promises';
import type { TiktokenBPE, TiktokenEncoding, TiktokenModel } from 'js-tiktoken/lite';
import { Tiktoken, getEncodingNameForModel } from 'js-tiktoken/lite';
import { jsonParse } from 'n8n-workflow';
@@ -6,13 +6,13 @@ import { join } from 'path';
const cache: Record<string, Tiktoken> = {};
const loadJSONFile = (filename: string): TiktokenBPE => {
const loadJSONFile = async (filename: string): Promise<TiktokenBPE> => {
const filePath = join(__dirname, filename);
const content = readFileSync(filePath, 'utf-8');
return jsonParse(content);
const content = await readFile(filePath, 'utf-8');
return await jsonParse(content);
};
export function getEncoding(encoding: TiktokenEncoding): Tiktoken {
export async function getEncoding(encoding: TiktokenEncoding): Promise<Tiktoken> {
if (cache[encoding]) {
return cache[encoding];
}
@@ -21,20 +21,20 @@ export function getEncoding(encoding: TiktokenEncoding): Tiktoken {
switch (encoding) {
case 'o200k_base':
jsonData = loadJSONFile('./o200k_base.json');
jsonData = await loadJSONFile('./o200k_base.json');
break;
case 'cl100k_base':
jsonData = loadJSONFile('./cl100k_base.json');
jsonData = await loadJSONFile('./cl100k_base.json');
break;
default:
// Fall back to cl100k_base for unsupported encodings
jsonData = loadJSONFile('./cl100k_base.json');
jsonData = await loadJSONFile('./cl100k_base.json');
}
cache[encoding] = new Tiktoken(jsonData);
return cache[encoding];
}
export function encodingForModel(model: TiktokenModel): Tiktoken {
return getEncoding(getEncodingNameForModel(model));
export async function encodingForModel(model: TiktokenModel): Promise<Tiktoken> {
return await getEncoding(getEncodingNameForModel(model));
}