feat(editor): Add node popularity scores to improve search ranking (#19561)

packages/frontend/editor-ui/.gitignore

@@ -25,3 +25,6 @@ yarn-error.log*
 
 # Auto-generated files
 src/components.d.ts
+
+# Build artifacts
+.build/

packages/frontend/editor-ui/package.json

@@ -5,7 +5,9 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "clean": "rimraf dist .turbo",
+    "clean": "rimraf dist .turbo .build",
+    "popularity-cache-marker": "mkdir -p .build && date +%G-W%V > .build/cache-marker",
+    "fetch-popularity": "node scripts/fetch-node-popularity.mjs",
     "build": "cross-env VUE_APP_PUBLIC_PATH=\"/{{BASE_PATH}}/\" NODE_OPTIONS=\"--max-old-space-size=8192\" vite build",
     "typecheck": "vue-tsc --noEmit",
     "typecheck:watch": "vue-tsc --watch --noEmit",

packages/frontend/editor-ui/scripts/fetch-node-popularity.mjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+import { promises as fs } from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const POPULARITY_ENDPOINT =
+	process.env.NODE_POPULARITY_ENDPOINT ||
+	'https://internal.users.n8n.cloud/webhook/nodes-popularity-scores';
+const BUILD_DIR = path.join(__dirname, '..', '.build');
+const OUTPUT_FILE = path.join(BUILD_DIR, 'node-popularity.json');
+
+async function ensureBuildDir() {
+	try {
+		await fs.mkdir(BUILD_DIR, { recursive: true });
+	} catch (error) {
+		// Directory might already exist, that's fine
+	}
+}
+
+async function fetchPopularityData() {
+	try {
+		console.log('Fetching node popularity data from:', POPULARITY_ENDPOINT);
+		const response = await fetch(POPULARITY_ENDPOINT, {
+			signal: AbortSignal.timeout(5000),
+		});
+
+		if (!response.ok) {
+			throw new Error(`HTTP error! status: ${response.status}`);
+		}
+
+		const data = await response.json();
+		console.log(`Successfully fetched popularity data for ${data.length} nodes`);
+		return data;
+	} catch (error) {
+		console.warn('Failed to fetch node popularity data:', error.message);
+		return null;
+	}
+}
+
+async function getExistingData() {
+	try {
+		const content = await fs.readFile(OUTPUT_FILE, 'utf-8');
+		return JSON.parse(content);
+	} catch (error) {
+		// File doesn't exist or is invalid
+		return null;
+	}
+}
+
+async function savePopularityData(data) {
+	await ensureBuildDir();
+	await fs.writeFile(OUTPUT_FILE, JSON.stringify(data, null, 2));
+	console.log(`Saved popularity data to ${OUTPUT_FILE} with ${data.length} nodes`);
+}
+
+async function fallbackToExistingData() {
+	const existingData = await getExistingData();
+
+	if (existingData) {
+		console.log('Using existing cached data - no changes made');
+		// Don't regenerate the file, keep the existing one
+	} else {
+		console.error('No data available - neither from API nor cache');
+		console.error('Creating empty placeholder file to avoid build failure');
+		await savePopularityData([]);
+	}
+}
+
+async function main() {
+	try {
+		// Try to fetch fresh data
+		const freshData = await fetchPopularityData();
+
+		if (freshData && Array.isArray(freshData) && freshData.length > 0) {
+			// Save the fresh data
+			await savePopularityData(freshData);
+		} else {
+			// Fetching failed, check if we have existing data
+			console.log('API unavailable, checking for existing cached data');
+			await fallbackToExistingData();
+		}
+	} catch (error) {
+		console.error('Error in fetch-node-popularity script:', error);
+
+		await fallbackToExistingData();
+	}
+}
+
+main();

packages/frontend/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts

@@ -61,6 +61,8 @@ import { useUIStore } from '@/stores/ui.store';
 import { type NodeIconSource } from '@/utils/nodeIcon';
 import { getThemedValue } from '@/utils/nodeTypesUtils';
 
+import nodePopularity from 'virtual:node-popularity-data';
+
 export interface ViewStack {
 	uuid?: string;
 	title?: string;
@@ -88,6 +90,13 @@ export interface ViewStack {
 	communityNodeDetails?: CommunityNodeDetails;
 }
 
+const nodePopularityMap = Object.values(nodePopularity).reduce((acc, node) => {
+	return {
+		...acc,
+		[node.id]: node.popularity * 100, // Scale the popularity score
+	};
+}, {});
+
 export const useViewStacks = defineStore('nodeCreatorViewStacks', () => {
 	const nodeCreatorStore = useNodeCreatorStore();
 	const { getActiveItemIndex } = useKeyboardNavigation();
@@ -121,7 +130,11 @@ export const useViewStacks = defineStore('nodeCreatorViewStacks', () => {
 				searchBase = filterOutAiNodes(searchBase);
 			}
 
-			const searchResults = extendItemsWithUUID(searchNodes(stack.search || '', searchBase));
+			const searchResults = extendItemsWithUUID(
+				searchNodes(stack.search || '', searchBase, {
+					popularity: nodePopularityMap,
+				}),
+			);
 
 			const groupedNodes = groupIfAiNodes(searchResults, stack.title, false) ?? searchResults;
 			// Set the active index to the second item if there's a section
@@ -181,8 +194,11 @@ export const useViewStacks = defineStore('nodeCreatorViewStacks', () => {
 		const filteredNodes = isAiRootView(stack) ? allNodes : filterOutAiNodes(allNodes);
 
 		let globalSearchResult: INodeCreateElement[] = extendItemsWithUUID(
-			searchNodes(stack.search || '', filteredNodes),
+			searchNodes(stack.search || '', filteredNodes, {
+				popularity: nodePopularityMap,
+			}),
 		);
+
 		if (isAiRootView(stack)) {
 			globalSearchResult = groupIfAiNodes(globalSearchResult, stack.title, false);
 		}

packages/frontend/editor-ui/src/components/Node/NodeCreator/utils.ts

@@ -29,6 +29,7 @@ import {
 import { v4 as uuidv4 } from 'uuid';
 
 import { sublimeSearch } from '@n8n/utils/search/sublimeSearch';
+import { reRankSearchResults } from '@n8n/utils/search/reRankSearchResults';
 import type { NodeViewItemSection } from './viewsData';
 import { i18n } from '@n8n/i18n';
 import sortBy from 'lodash/sortBy';
@@ -122,7 +123,11 @@ export function removeTrailingTrigger(searchFilter: string) {
 	return searchFilter;
 }
 
-export function searchNodes(searchFilter: string, items: INodeCreateElement[]) {
+export function searchNodes(
+	searchFilter: string,
+	items: INodeCreateElement[],
+	additionalFactors = {},
+) {
 	const askAiEnabled = useSettingsStore().isAskAiEnabled;
 	if (!askAiEnabled) {
 		items = items.filter((item) => item.key !== AI_TRANSFORM_NODE_TYPE);
@@ -131,12 +136,12 @@ export function searchNodes(searchFilter: string, items: INodeCreateElement[]) {
 	const trimmedFilter = removeTrailingTrigger(searchFilter).toLowerCase();
 
 	// We have a snapshot of this call in sublimeSearch.test.ts to assert practical order for some cases
-	// Please update the snapshots per the README next to the the snapshots if you modify items significantly.
-	const result = (sublimeSearch<INodeCreateElement>(trimmedFilter, items) || []).map(
-		({ item }) => item,
-	);
+	// Please update the snapshots per the README next to the snapshots if you modify items significantly.
+	const searchResults = sublimeSearch<INodeCreateElement>(trimmedFilter, items) || [];
 
-	return result;
+	const reRankedResults = reRankSearchResults(searchResults, additionalFactors);
+
+	return reRankedResults.map(({ item }) => item);
 }
 
 export function flattenCreateElements(items: INodeCreateElement[]): INodeCreateElement[] {

packages/frontend/editor-ui/src/vite-virtual-nodes-popularity.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="vite/client" />
+
+declare module 'virtual:node-popularity-data' {
+	const data: Array<{
+		id: string;
+		popularity: number;
+	}>;
+	export default data;
+}

packages/frontend/editor-ui/turbo.json

@@ -0,0 +1,19 @@
+{
+	"extends": ["//"],
+	"tasks": {
+		"popularity-cache-marker": {
+			"cache": false,
+			"outputs": [".build/cache-marker"]
+		},
+		"fetch-popularity": {
+			"dependsOn": ["popularity-cache-marker"],
+			"cache": true,
+			"outputs": [".build/node-popularity.json"],
+			"inputs": ["scripts/fetch-node-popularity.mjs", ".build/cache-marker"]
+		},
+		"build": {
+			"dependsOn": ["^build", "fetch-popularity"],
+			"outputs": ["dist/**"]
+		}
+	}
+}

packages/frontend/editor-ui/vite.config.mts

@@ -13,6 +13,7 @@ import browserslistToEsbuild from 'browserslist-to-esbuild';
 import legacy from '@vitejs/plugin-legacy';
 import browserslist from 'browserslist';
 import { isLocaleFile, sendLocaleUpdate } from './vite/i18n-locales-hmr-helpers';
+import { nodePopularityPlugin } from './vite/vite-plugin-node-popularity.mjs';
 
 const publicPath = process.env.VUE_APP_PUBLIC_PATH || '/';
 
@@ -75,6 +76,7 @@ const alias = [
 ];
 
 const plugins: UserConfig['plugins'] = [
+	nodePopularityPlugin(),
 	icons({
 		compiler: 'vue3',
 		autoInstall: true,

packages/frontend/editor-ui/vite/vite-plugin-node-popularity.mts

@@ -0,0 +1,32 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import type { Plugin } from 'vite';
+
+const VIRTUAL_MODULE_ID = 'virtual:node-popularity-data';
+const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID;
+
+export function nodePopularityPlugin(): Plugin {
+	return {
+		name: 'node-popularity-plugin',
+		resolveId(id) {
+			if (id === VIRTUAL_MODULE_ID) {
+				return RESOLVED_VIRTUAL_MODULE_ID;
+			}
+		},
+		async load(id) {
+			if (id === RESOLVED_VIRTUAL_MODULE_ID) {
+				// Try to load the data from the build directory
+				const buildDataPath = path.join(process.cwd(), '.build', 'node-popularity.json');
+
+				try {
+					const data = await fs.readFile(buildDataPath, 'utf-8');
+					return `export default ${data}`;
+				} catch (error) {
+					// If file doesn't exist, return empty array
+					console.warn('Node popularity data not found at', buildDataPath, '- using empty array');
+					return 'export default []';
+				}
+			}
+		},
+	};
+}