From df9521ff8ec84e49030c586446caccb76e7f3ae8 Mon Sep 17 00:00:00 2001
From: jeanpaul <jeanpaul@users.noreply.github.com>
Date: Tue, 26 Aug 2025 13:47:42 +0200
Subject: [PATCH] fix(core): Prevent race condition in tiktoken encoding cache
 (no-changelog) (#18780)

Co-authored-by: Claude <noreply@anthropic.com>
---
 .../utils/tokenizer/tiktoken.ts               | 43 +++++++++++--------
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/packages/@n8n/nodes-langchain/utils/tokenizer/tiktoken.ts b/packages/@n8n/nodes-langchain/utils/tokenizer/tiktoken.ts
index 8bdfe27685..5bad1c2ff7 100644
--- a/packages/@n8n/nodes-langchain/utils/tokenizer/tiktoken.ts
+++ b/packages/@n8n/nodes-langchain/utils/tokenizer/tiktoken.ts
@@ -4,7 +4,7 @@ import { Tiktoken, getEncodingNameForModel } from 'js-tiktoken/lite';
 import { jsonParse } from 'n8n-workflow';
 import { join } from 'path';
 
-const cache: Record<string, Tiktoken> = {};
+const cache: Record<string, Promise<Tiktoken>> = {};
 
 const loadJSONFile = async (filename: string): Promise<TiktokenBPE> => {
 	const filePath = join(__dirname, filename);
@@ -13,26 +13,31 @@ const loadJSONFile = async (filename: string): Promise<TiktokenBPE> => {
 };
 
 export async function getEncoding(encoding: TiktokenEncoding): Promise<Tiktoken> {
-	if (cache[encoding]) {
-		return cache[encoding];
+	if (!(encoding in cache)) {
+		// Create and cache the promise for loading this encoding
+		cache[encoding] = (async () => {
+			let jsonData: TiktokenBPE;
+
+			switch (encoding) {
+				case 'o200k_base':
+					jsonData = await loadJSONFile('./o200k_base.json');
+					break;
+				case 'cl100k_base':
+					jsonData = await loadJSONFile('./cl100k_base.json');
+					break;
+				default:
+					// Fall back to cl100k_base for unsupported encodings
+					jsonData = await loadJSONFile('./cl100k_base.json');
+			}
+
+			return new Tiktoken(jsonData);
+		})().catch((error) => {
+			delete cache[encoding];
+			throw error;
+		});
 	}
 
-	let jsonData: TiktokenBPE;
-
-	switch (encoding) {
-		case 'o200k_base':
-			jsonData = await loadJSONFile('./o200k_base.json');
-			break;
-		case 'cl100k_base':
-			jsonData = await loadJSONFile('./cl100k_base.json');
-			break;
-		default:
-			// Fall back to cl100k_base for unsupported encodings
-			jsonData = await loadJSONFile('./cl100k_base.json');
-	}
-
-	cache[encoding] = new Tiktoken(jsonData);
-	return cache[encoding];
+	return await cache[encoding];
 }
 
 export async function encodingForModel(model: TiktokenModel): Promise<Tiktoken> {