feat(editor): Add node popularity scores to improve search ranking (#19561)

packages/@n8n/utils/src/search/reRankSearchResults.test.ts

@@ -0,0 +1,119 @@
+import { reRankSearchResults } from './reRankSearchResults';
+import topLevel from './snapshots/toplevel.snapshot.json';
+import { sublimeSearch } from './sublimeSearch';
+
+describe('reRankSearchResults', () => {
+	describe('should re-rank search results based on additional factors', () => {
+		it('should return Coda before Code without additional factors for query "cod"', () => {
+			const searchResults = sublimeSearch('cod', topLevel);
+			const resultNames = searchResults.map((result) => result.item.properties.displayName);
+
+			// Without re-ranking, Coda should appear before Code
+			expect(resultNames[0]).toBe('Coda');
+			expect(resultNames[1]).toBe('Code');
+		});
+
+		it('should return Code before Coda with additional factors favoring Code for query "cod"', () => {
+			const searchResults = sublimeSearch('cod', topLevel);
+
+			// Add popularity scores that heavily favor Code node
+			const additionalFactors = {
+				popularity: {
+					/* eslint-disable @typescript-eslint/naming-convention */
+					'n8n-nodes-base.code': 90, // High popularity for Code node
+					'n8n-nodes-base.coda': 10, // Lower popularity for Coda node
+					/* eslint-enable @typescript-eslint/naming-convention */
+				},
+			};
+
+			const reRankedResults = reRankSearchResults(searchResults, additionalFactors);
+			const resultNames = reRankedResults.map((result) => result.item.properties.displayName);
+
+			// After re-ranking with additional factors, Code should appear before Coda
+			expect(resultNames[0]).toBe('Code');
+			expect(resultNames[1]).toBe('Coda');
+		});
+
+		it('should handle multiple additional factors', () => {
+			const searchResults = sublimeSearch('cod', topLevel);
+
+			// Add multiple factors: popularity and recent usage
+			const additionalFactors = {
+				popularity: {
+					/* eslint-disable @typescript-eslint/naming-convention */
+					'n8n-nodes-base.code': 50,
+					'n8n-nodes-base.coda': 40,
+					/* eslint-enable @typescript-eslint/naming-convention */
+				},
+				recentUsage: {
+					/* eslint-disable @typescript-eslint/naming-convention */
+					'n8n-nodes-base.code': 80, // Code was used more recently
+					'n8n-nodes-base.coda': 20,
+					/* eslint-enable @typescript-eslint/naming-convention */
+				},
+			};
+
+			const reRankedResults = reRankSearchResults(searchResults, additionalFactors);
+			const resultNames = reRankedResults.map((result) => result.item.properties.displayName);
+
+			// Code should rank higher due to combined score (50 + 80 = 130 vs Coda's 40 + 20 = 60)
+			expect(resultNames[0]).toBe('Code');
+			expect(resultNames[1]).toBe('Coda');
+		});
+
+		it('should preserve original order when additional factors are equal', () => {
+			const searchResults = sublimeSearch('cod', topLevel);
+
+			// Add equal factors for both nodes
+			const additionalFactors = {
+				popularity: {
+					/* eslint-disable @typescript-eslint/naming-convention */
+					'n8n-nodes-base.code': 50,
+					'n8n-nodes-base.coda': 50,
+					/* eslint-enable @typescript-eslint/naming-convention */
+				},
+			};
+
+			const reRankedResults = reRankSearchResults(searchResults, additionalFactors);
+			const resultNames = reRankedResults.map((result) => result.item.properties.displayName);
+
+			// When additional factors are equal, original order should be preserved
+			// Since Coda has a higher base score from sublimeSearch, it should remain first
+			expect(resultNames[0]).toBe('Coda');
+			expect(resultNames[1]).toBe('Code');
+		});
+
+		it('should handle empty additional factors object', () => {
+			const searchResults = sublimeSearch('cod', topLevel);
+			const reRankedResults = reRankSearchResults(searchResults, {});
+
+			// Results should be identical to original search results
+			expect(reRankedResults).toEqual(searchResults);
+		});
+
+		it('should handle nodes not present in additional factors', () => {
+			const searchResults = sublimeSearch('git', topLevel);
+
+			// Only provide factor for some items
+			const additionalFactors = {
+				popularity: {
+					// eslint-disable-next-line @typescript-eslint/naming-convention
+					'n8n-nodes-base.github': 100,
+					// Other git-related nodes are not included
+				},
+			};
+
+			const reRankedResults = reRankSearchResults(searchResults, additionalFactors);
+
+			// GitHub should rank higher due to additional factor
+			const githubIndex = reRankedResults.findIndex(
+				(r) => r.item.properties.displayName === 'GitHub',
+			);
+			const gitIndex = reRankedResults.findIndex((r) => r.item.properties.displayName === 'Git');
+
+			if (githubIndex !== -1 && gitIndex !== -1) {
+				expect(githubIndex).toBeLessThan(gitIndex);
+			}
+		});
+	});
+});

packages/@n8n/utils/src/search/reRankSearchResults.ts

@@ -0,0 +1,26 @@
+export function reRankSearchResults<T extends { key: string }>(
+	searchResults: Array<{ score: number; item: T }>,
+	additionalFactors: Record<string, Record<string, number>>,
+): Array<{ score: number; item: T }> {
+	return searchResults
+		.map(({ score, item }) => {
+			// For each additional factor, we check if it exists for the item and type,
+			// and if so, we add the score to the item's score.
+			const additionalScore = Object.entries(additionalFactors).reduce((acc, [_, factorScores]) => {
+				const factorScore = factorScores[item.key];
+				if (factorScore) {
+					return acc + factorScore;
+				}
+
+				return acc;
+			}, 0);
+
+			return {
+				score: score + additionalScore,
+				item,
+			};
+		})
+		.sort((a, b) => {
+			return b.score - a.score;
+		});
+}

packages/frontend/editor-ui/.gitignore

@@ -25,3 +25,6 @@ yarn-error.log*
 
 # Auto-generated files
 src/components.d.ts
+
+# Build artifacts
+.build/

packages/frontend/editor-ui/package.json

@@ -5,7 +5,9 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "clean": "rimraf dist .turbo",
+    "clean": "rimraf dist .turbo .build",
+    "popularity-cache-marker": "mkdir -p .build && date +%G-W%V > .build/cache-marker",
+    "fetch-popularity": "node scripts/fetch-node-popularity.mjs",
     "build": "cross-env VUE_APP_PUBLIC_PATH=\"/{{BASE_PATH}}/\" NODE_OPTIONS=\"--max-old-space-size=8192\" vite build",
     "typecheck": "vue-tsc --noEmit",
     "typecheck:watch": "vue-tsc --watch --noEmit",

packages/frontend/editor-ui/scripts/fetch-node-popularity.mjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+import { promises as fs } from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const POPULARITY_ENDPOINT =
+	process.env.NODE_POPULARITY_ENDPOINT ||
+	'https://internal.users.n8n.cloud/webhook/nodes-popularity-scores';
+const BUILD_DIR = path.join(__dirname, '..', '.build');
+const OUTPUT_FILE = path.join(BUILD_DIR, 'node-popularity.json');
+
+async function ensureBuildDir() {
+	try {
+		await fs.mkdir(BUILD_DIR, { recursive: true });
+	} catch (error) {
+		// Directory might already exist, that's fine
+	}
+}
+
+async function fetchPopularityData() {
+	try {
+		console.log('Fetching node popularity data from:', POPULARITY_ENDPOINT);
+		const response = await fetch(POPULARITY_ENDPOINT, {
+			signal: AbortSignal.timeout(5000),
+		});
+
+		if (!response.ok) {
+			throw new Error(`HTTP error! status: ${response.status}`);
+		}
+
+		const data = await response.json();
+		console.log(`Successfully fetched popularity data for ${data.length} nodes`);
+		return data;
+	} catch (error) {
+		console.warn('Failed to fetch node popularity data:', error.message);
+		return null;
+	}
+}
+
+async function getExistingData() {
+	try {
+		const content = await fs.readFile(OUTPUT_FILE, 'utf-8');
+		return JSON.parse(content);
+	} catch (error) {
+		// File doesn't exist or is invalid
+		return null;
+	}
+}
+
+async function savePopularityData(data) {
+	await ensureBuildDir();
+	await fs.writeFile(OUTPUT_FILE, JSON.stringify(data, null, 2));
+	console.log(`Saved popularity data to ${OUTPUT_FILE} with ${data.length} nodes`);
+}
+
+async function fallbackToExistingData() {
+	const existingData = await getExistingData();
+
+	if (existingData) {
+		console.log('Using existing cached data - no changes made');
+		// Don't regenerate the file, keep the existing one
+	} else {
+		console.error('No data available - neither from API nor cache');
+		console.error('Creating empty placeholder file to avoid build failure');
+		await savePopularityData([]);
+	}
+}
+
+async function main() {
+	try {
+		// Try to fetch fresh data
+		const freshData = await fetchPopularityData();
+
+		if (freshData && Array.isArray(freshData) && freshData.length > 0) {
+			// Save the fresh data
+			await savePopularityData(freshData);
+		} else {
+			// Fetching failed, check if we have existing data
+			console.log('API unavailable, checking for existing cached data');
+			await fallbackToExistingData();
+		}
+	} catch (error) {
+		console.error('Error in fetch-node-popularity script:', error);
+
+		await fallbackToExistingData();
+	}
+}
+
+main();

packages/frontend/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts

@@ -61,6 +61,8 @@ import { useUIStore } from '@/stores/ui.store';
 import { type NodeIconSource } from '@/utils/nodeIcon';
 import { getThemedValue } from '@/utils/nodeTypesUtils';
 
+import nodePopularity from 'virtual:node-popularity-data';
+
 export interface ViewStack {
 	uuid?: string;
 	title?: string;
@@ -88,6 +90,13 @@ export interface ViewStack {
 	communityNodeDetails?: CommunityNodeDetails;
 }
 
+const nodePopularityMap = Object.values(nodePopularity).reduce((acc, node) => {
+	return {
+		...acc,
+		[node.id]: node.popularity * 100, // Scale the popularity score
+	};
+}, {});
+
 export const useViewStacks = defineStore('nodeCreatorViewStacks', () => {
 	const nodeCreatorStore = useNodeCreatorStore();
 	const { getActiveItemIndex } = useKeyboardNavigation();
@@ -121,7 +130,11 @@ export const useViewStacks = defineStore('nodeCreatorViewStacks', () => {
 				searchBase = filterOutAiNodes(searchBase);
 			}
 
-			const searchResults = extendItemsWithUUID(searchNodes(stack.search || '', searchBase));
+			const searchResults = extendItemsWithUUID(
+				searchNodes(stack.search || '', searchBase, {
+					popularity: nodePopularityMap,
+				}),
+			);
 
 			const groupedNodes = groupIfAiNodes(searchResults, stack.title, false) ?? searchResults;
 			// Set the active index to the second item if there's a section
@@ -181,8 +194,11 @@ export const useViewStacks = defineStore('nodeCreatorViewStacks', () => {
 		const filteredNodes = isAiRootView(stack) ? allNodes : filterOutAiNodes(allNodes);
 
 		let globalSearchResult: INodeCreateElement[] = extendItemsWithUUID(
-			searchNodes(stack.search || '', filteredNodes),
+			searchNodes(stack.search || '', filteredNodes, {
+				popularity: nodePopularityMap,
+			}),
 		);
+
 		if (isAiRootView(stack)) {
 			globalSearchResult = groupIfAiNodes(globalSearchResult, stack.title, false);
 		}

packages/frontend/editor-ui/src/components/Node/NodeCreator/utils.ts

@@ -29,6 +29,7 @@ import {
 import { v4 as uuidv4 } from 'uuid';
 
 import { sublimeSearch } from '@n8n/utils/search/sublimeSearch';
+import { reRankSearchResults } from '@n8n/utils/search/reRankSearchResults';
 import type { NodeViewItemSection } from './viewsData';
 import { i18n } from '@n8n/i18n';
 import sortBy from 'lodash/sortBy';
@@ -122,7 +123,11 @@ export function removeTrailingTrigger(searchFilter: string) {
 	return searchFilter;
 }
 
-export function searchNodes(searchFilter: string, items: INodeCreateElement[]) {
+export function searchNodes(
+	searchFilter: string,
+	items: INodeCreateElement[],
+	additionalFactors = {},
+) {
 	const askAiEnabled = useSettingsStore().isAskAiEnabled;
 	if (!askAiEnabled) {
 		items = items.filter((item) => item.key !== AI_TRANSFORM_NODE_TYPE);
@@ -131,12 +136,12 @@ export function searchNodes(searchFilter: string, items: INodeCreateElement[]) {
 	const trimmedFilter = removeTrailingTrigger(searchFilter).toLowerCase();
 
 	// We have a snapshot of this call in sublimeSearch.test.ts to assert practical order for some cases
-	// Please update the snapshots per the README next to the the snapshots if you modify items significantly.
-	const result = (sublimeSearch<INodeCreateElement>(trimmedFilter, items) || []).map(
-		({ item }) => item,
-	);
+	// Please update the snapshots per the README next to the snapshots if you modify items significantly.
+	const searchResults = sublimeSearch<INodeCreateElement>(trimmedFilter, items) || [];
 
-	return result;
+	const reRankedResults = reRankSearchResults(searchResults, additionalFactors);
+
+	return reRankedResults.map(({ item }) => item);
 }
 
 export function flattenCreateElements(items: INodeCreateElement[]): INodeCreateElement[] {

packages/frontend/editor-ui/src/vite-virtual-nodes-popularity.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="vite/client" />
+
+declare module 'virtual:node-popularity-data' {
+	const data: Array<{
+		id: string;
+		popularity: number;
+	}>;
+	export default data;
+}

packages/frontend/editor-ui/turbo.json

@@ -0,0 +1,19 @@
+{
+	"extends": ["//"],
+	"tasks": {
+		"popularity-cache-marker": {
+			"cache": false,
+			"outputs": [".build/cache-marker"]
+		},
+		"fetch-popularity": {
+			"dependsOn": ["popularity-cache-marker"],
+			"cache": true,
+			"outputs": [".build/node-popularity.json"],
+			"inputs": ["scripts/fetch-node-popularity.mjs", ".build/cache-marker"]
+		},
+		"build": {
+			"dependsOn": ["^build", "fetch-popularity"],
+			"outputs": ["dist/**"]
+		}
+	}
+}

packages/frontend/editor-ui/vite.config.mts

@@ -13,6 +13,7 @@ import browserslistToEsbuild from 'browserslist-to-esbuild';
 import legacy from '@vitejs/plugin-legacy';
 import browserslist from 'browserslist';
 import { isLocaleFile, sendLocaleUpdate } from './vite/i18n-locales-hmr-helpers';
+import { nodePopularityPlugin } from './vite/vite-plugin-node-popularity.mjs';
 
 const publicPath = process.env.VUE_APP_PUBLIC_PATH || '/';
 
@@ -75,6 +76,7 @@ const alias = [
 ];
 
 const plugins: UserConfig['plugins'] = [
+	nodePopularityPlugin(),
 	icons({
 		compiler: 'vue3',
 		autoInstall: true,

packages/frontend/editor-ui/vite/vite-plugin-node-popularity.mts

@@ -0,0 +1,32 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import type { Plugin } from 'vite';
+
+const VIRTUAL_MODULE_ID = 'virtual:node-popularity-data';
+const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID;
+
+export function nodePopularityPlugin(): Plugin {
+	return {
+		name: 'node-popularity-plugin',
+		resolveId(id) {
+			if (id === VIRTUAL_MODULE_ID) {
+				return RESOLVED_VIRTUAL_MODULE_ID;
+			}
+		},
+		async load(id) {
+			if (id === RESOLVED_VIRTUAL_MODULE_ID) {
+				// Try to load the data from the build directory
+				const buildDataPath = path.join(process.cwd(), '.build', 'node-popularity.json');
+
+				try {
+					const data = await fs.readFile(buildDataPath, 'utf-8');
+					return `export default ${data}`;
+				} catch (error) {
+					// If file doesn't exist, return empty array
+					console.warn('Node popularity data not found at', buildDataPath, '- using empty array');
+					return 'export default []';
+				}
+			}
+		},
+	};
+}