diff --git a/packages/@n8n/utils/src/search/reRankSearchResults.test.ts b/packages/@n8n/utils/src/search/reRankSearchResults.test.ts new file mode 100644 index 0000000000..b1b32a3e90 --- /dev/null +++ b/packages/@n8n/utils/src/search/reRankSearchResults.test.ts @@ -0,0 +1,119 @@ +import { reRankSearchResults } from './reRankSearchResults'; +import topLevel from './snapshots/toplevel.snapshot.json'; +import { sublimeSearch } from './sublimeSearch'; + +describe('reRankSearchResults', () => { + describe('should re-rank search results based on additional factors', () => { + it('should return Coda before Code without additional factors for query "cod"', () => { + const searchResults = sublimeSearch('cod', topLevel); + const resultNames = searchResults.map((result) => result.item.properties.displayName); + + // Without re-ranking, Coda should appear before Code + expect(resultNames[0]).toBe('Coda'); + expect(resultNames[1]).toBe('Code'); + }); + + it('should return Code before Coda with additional factors favoring Code for query "cod"', () => { + const searchResults = sublimeSearch('cod', topLevel); + + // Add popularity scores that heavily favor Code node + const additionalFactors = { + popularity: { + /* eslint-disable @typescript-eslint/naming-convention */ + 'n8n-nodes-base.code': 90, // High popularity for Code node + 'n8n-nodes-base.coda': 10, // Lower popularity for Coda node + /* eslint-enable @typescript-eslint/naming-convention */ + }, + }; + + const reRankedResults = reRankSearchResults(searchResults, additionalFactors); + const resultNames = reRankedResults.map((result) => result.item.properties.displayName); + + // After re-ranking with additional factors, Code should appear before Coda + expect(resultNames[0]).toBe('Code'); + expect(resultNames[1]).toBe('Coda'); + }); + + it('should handle multiple additional factors', () => { + const searchResults = sublimeSearch('cod', topLevel); + + // Add multiple factors: popularity and recent usage + const additionalFactors = { + popularity: { + /* eslint-disable @typescript-eslint/naming-convention */ + 'n8n-nodes-base.code': 50, + 'n8n-nodes-base.coda': 40, + /* eslint-enable @typescript-eslint/naming-convention */ + }, + recentUsage: { + /* eslint-disable @typescript-eslint/naming-convention */ + 'n8n-nodes-base.code': 80, // Code was used more recently + 'n8n-nodes-base.coda': 20, + /* eslint-enable @typescript-eslint/naming-convention */ + }, + }; + + const reRankedResults = reRankSearchResults(searchResults, additionalFactors); + const resultNames = reRankedResults.map((result) => result.item.properties.displayName); + + // Code should rank higher due to combined score (50 + 80 = 130 vs Coda's 40 + 20 = 60) + expect(resultNames[0]).toBe('Code'); + expect(resultNames[1]).toBe('Coda'); + }); + + it('should preserve original order when additional factors are equal', () => { + const searchResults = sublimeSearch('cod', topLevel); + + // Add equal factors for both nodes + const additionalFactors = { + popularity: { + /* eslint-disable @typescript-eslint/naming-convention */ + 'n8n-nodes-base.code': 50, + 'n8n-nodes-base.coda': 50, + /* eslint-enable @typescript-eslint/naming-convention */ + }, + }; + + const reRankedResults = reRankSearchResults(searchResults, additionalFactors); + const resultNames = reRankedResults.map((result) => result.item.properties.displayName); + + // When additional factors are equal, original order should be preserved + // Since Coda has a higher base score from sublimeSearch, it should remain first + expect(resultNames[0]).toBe('Coda'); + expect(resultNames[1]).toBe('Code'); + }); + + it('should handle empty additional factors object', () => { + const searchResults = sublimeSearch('cod', topLevel); + const reRankedResults = reRankSearchResults(searchResults, {}); + + // Results should be identical to original search results + expect(reRankedResults).toEqual(searchResults); + }); + + it('should handle nodes not present in additional factors', () => { + const searchResults = sublimeSearch('git', topLevel); + + // Only provide factor for some items + const additionalFactors = { + popularity: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'n8n-nodes-base.github': 100, + // Other git-related nodes are not included + }, + }; + + const reRankedResults = reRankSearchResults(searchResults, additionalFactors); + + // GitHub should rank higher due to additional factor + const githubIndex = reRankedResults.findIndex( + (r) => r.item.properties.displayName === 'GitHub', + ); + const gitIndex = reRankedResults.findIndex((r) => r.item.properties.displayName === 'Git'); + + if (githubIndex !== -1 && gitIndex !== -1) { + expect(githubIndex).toBeLessThan(gitIndex); + } + }); + }); +}); diff --git a/packages/@n8n/utils/src/search/reRankSearchResults.ts b/packages/@n8n/utils/src/search/reRankSearchResults.ts new file mode 100644 index 0000000000..7dd608cf5e --- /dev/null +++ b/packages/@n8n/utils/src/search/reRankSearchResults.ts @@ -0,0 +1,26 @@ +export function reRankSearchResults( + searchResults: Array<{ score: number; item: T }>, + additionalFactors: Record>, +): Array<{ score: number; item: T }> { + return searchResults + .map(({ score, item }) => { + // For each additional factor, we check if it exists for the item and type, + // and if so, we add the score to the item's score. + const additionalScore = Object.entries(additionalFactors).reduce((acc, [_, factorScores]) => { + const factorScore = factorScores[item.key]; + if (factorScore) { + return acc + factorScore; + } + + return acc; + }, 0); + + return { + score: score + additionalScore, + item, + }; + }) + .sort((a, b) => { + return b.score - a.score; + }); +} diff --git a/packages/frontend/editor-ui/.gitignore b/packages/frontend/editor-ui/.gitignore index 137e9ef8d0..f226a6d86a 100644 --- a/packages/frontend/editor-ui/.gitignore +++ b/packages/frontend/editor-ui/.gitignore @@ -25,3 +25,6 @@ yarn-error.log* # Auto-generated files src/components.d.ts + +# Build artifacts +.build/ diff --git a/packages/frontend/editor-ui/package.json b/packages/frontend/editor-ui/package.json index 04cbb620d2..610448a583 100644 --- a/packages/frontend/editor-ui/package.json +++ b/packages/frontend/editor-ui/package.json @@ -5,7 +5,9 @@ "main": "index.js", "type": "module", "scripts": { - "clean": "rimraf dist .turbo", + "clean": "rimraf dist .turbo .build", + "popularity-cache-marker": "mkdir -p .build && date +%G-W%V > .build/cache-marker", + "fetch-popularity": "node scripts/fetch-node-popularity.mjs", "build": "cross-env VUE_APP_PUBLIC_PATH=\"/{{BASE_PATH}}/\" NODE_OPTIONS=\"--max-old-space-size=8192\" vite build", "typecheck": "vue-tsc --noEmit", "typecheck:watch": "vue-tsc --watch --noEmit", diff --git a/packages/frontend/editor-ui/scripts/fetch-node-popularity.mjs b/packages/frontend/editor-ui/scripts/fetch-node-popularity.mjs new file mode 100755 index 0000000000..1e54462372 --- /dev/null +++ b/packages/frontend/editor-ui/scripts/fetch-node-popularity.mjs @@ -0,0 +1,90 @@ +#!/usr/bin/env node +import { promises as fs } from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const POPULARITY_ENDPOINT = + process.env.NODE_POPULARITY_ENDPOINT || + 'https://internal.users.n8n.cloud/webhook/nodes-popularity-scores'; +const BUILD_DIR = path.join(__dirname, '..', '.build'); +const OUTPUT_FILE = path.join(BUILD_DIR, 'node-popularity.json'); + +async function ensureBuildDir() { + try { + await fs.mkdir(BUILD_DIR, { recursive: true }); + } catch (error) { + // Directory might already exist, that's fine + } +} + +async function fetchPopularityData() { + try { + console.log('Fetching node popularity data from:', POPULARITY_ENDPOINT); + const response = await fetch(POPULARITY_ENDPOINT, { + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const data = await response.json(); + console.log(`Successfully fetched popularity data for ${data.length} nodes`); + return data; + } catch (error) { + console.warn('Failed to fetch node popularity data:', error.message); + return null; + } +} + +async function getExistingData() { + try { + const content = await fs.readFile(OUTPUT_FILE, 'utf-8'); + return JSON.parse(content); + } catch (error) { + // File doesn't exist or is invalid + return null; + } +} + +async function savePopularityData(data) { + await ensureBuildDir(); + await fs.writeFile(OUTPUT_FILE, JSON.stringify(data, null, 2)); + console.log(`Saved popularity data to ${OUTPUT_FILE} with ${data.length} nodes`); +} + +async function fallbackToExistingData() { + const existingData = await getExistingData(); + + if (existingData) { + console.log('Using existing cached data - no changes made'); + // Don't regenerate the file, keep the existing one + } else { + console.error('No data available - neither from API nor cache'); + console.error('Creating empty placeholder file to avoid build failure'); + await savePopularityData([]); + } +} + +async function main() { + try { + // Try to fetch fresh data + const freshData = await fetchPopularityData(); + + if (freshData && Array.isArray(freshData) && freshData.length > 0) { + // Save the fresh data + await savePopularityData(freshData); + } else { + // Fetching failed, check if we have existing data + console.log('API unavailable, checking for existing cached data'); + await fallbackToExistingData(); + } + } catch (error) { + console.error('Error in fetch-node-popularity script:', error); + + await fallbackToExistingData(); + } +} + +main(); diff --git a/packages/frontend/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts b/packages/frontend/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts index 4e24346f53..c4100c8842 100644 --- a/packages/frontend/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts +++ b/packages/frontend/editor-ui/src/components/Node/NodeCreator/composables/useViewStacks.ts @@ -61,6 +61,8 @@ import { useUIStore } from '@/stores/ui.store'; import { type NodeIconSource } from '@/utils/nodeIcon'; import { getThemedValue } from '@/utils/nodeTypesUtils'; +import nodePopularity from 'virtual:node-popularity-data'; + export interface ViewStack { uuid?: string; title?: string; @@ -88,6 +90,13 @@ export interface ViewStack { communityNodeDetails?: CommunityNodeDetails; } +const nodePopularityMap = Object.values(nodePopularity).reduce((acc, node) => { + return { + ...acc, + [node.id]: node.popularity * 100, // Scale the popularity score + }; +}, {}); + export const useViewStacks = defineStore('nodeCreatorViewStacks', () => { const nodeCreatorStore = useNodeCreatorStore(); const { getActiveItemIndex } = useKeyboardNavigation(); @@ -121,7 +130,11 @@ export const useViewStacks = defineStore('nodeCreatorViewStacks', () => { searchBase = filterOutAiNodes(searchBase); } - const searchResults = extendItemsWithUUID(searchNodes(stack.search || '', searchBase)); + const searchResults = extendItemsWithUUID( + searchNodes(stack.search || '', searchBase, { + popularity: nodePopularityMap, + }), + ); const groupedNodes = groupIfAiNodes(searchResults, stack.title, false) ?? searchResults; // Set the active index to the second item if there's a section @@ -181,8 +194,11 @@ export const useViewStacks = defineStore('nodeCreatorViewStacks', () => { const filteredNodes = isAiRootView(stack) ? allNodes : filterOutAiNodes(allNodes); let globalSearchResult: INodeCreateElement[] = extendItemsWithUUID( - searchNodes(stack.search || '', filteredNodes), + searchNodes(stack.search || '', filteredNodes, { + popularity: nodePopularityMap, + }), ); + if (isAiRootView(stack)) { globalSearchResult = groupIfAiNodes(globalSearchResult, stack.title, false); } diff --git a/packages/frontend/editor-ui/src/components/Node/NodeCreator/utils.ts b/packages/frontend/editor-ui/src/components/Node/NodeCreator/utils.ts index ee06a486fd..3278945fcf 100644 --- a/packages/frontend/editor-ui/src/components/Node/NodeCreator/utils.ts +++ b/packages/frontend/editor-ui/src/components/Node/NodeCreator/utils.ts @@ -29,6 +29,7 @@ import { import { v4 as uuidv4 } from 'uuid'; import { sublimeSearch } from '@n8n/utils/search/sublimeSearch'; +import { reRankSearchResults } from '@n8n/utils/search/reRankSearchResults'; import type { NodeViewItemSection } from './viewsData'; import { i18n } from '@n8n/i18n'; import sortBy from 'lodash/sortBy'; @@ -122,7 +123,11 @@ export function removeTrailingTrigger(searchFilter: string) { return searchFilter; } -export function searchNodes(searchFilter: string, items: INodeCreateElement[]) { +export function searchNodes( + searchFilter: string, + items: INodeCreateElement[], + additionalFactors = {}, +) { const askAiEnabled = useSettingsStore().isAskAiEnabled; if (!askAiEnabled) { items = items.filter((item) => item.key !== AI_TRANSFORM_NODE_TYPE); @@ -131,12 +136,12 @@ export function searchNodes(searchFilter: string, items: INodeCreateElement[]) { const trimmedFilter = removeTrailingTrigger(searchFilter).toLowerCase(); // We have a snapshot of this call in sublimeSearch.test.ts to assert practical order for some cases - // Please update the snapshots per the README next to the the snapshots if you modify items significantly. - const result = (sublimeSearch(trimmedFilter, items) || []).map( - ({ item }) => item, - ); + // Please update the snapshots per the README next to the snapshots if you modify items significantly. + const searchResults = sublimeSearch(trimmedFilter, items) || []; - return result; + const reRankedResults = reRankSearchResults(searchResults, additionalFactors); + + return reRankedResults.map(({ item }) => item); } export function flattenCreateElements(items: INodeCreateElement[]): INodeCreateElement[] { diff --git a/packages/frontend/editor-ui/src/vite-virtual-nodes-popularity.d.ts b/packages/frontend/editor-ui/src/vite-virtual-nodes-popularity.d.ts new file mode 100644 index 0000000000..aed387a82a --- /dev/null +++ b/packages/frontend/editor-ui/src/vite-virtual-nodes-popularity.d.ts @@ -0,0 +1,9 @@ +/// + +declare module 'virtual:node-popularity-data' { + const data: Array<{ + id: string; + popularity: number; + }>; + export default data; +} diff --git a/packages/frontend/editor-ui/turbo.json b/packages/frontend/editor-ui/turbo.json new file mode 100644 index 0000000000..848da77109 --- /dev/null +++ b/packages/frontend/editor-ui/turbo.json @@ -0,0 +1,19 @@ +{ + "extends": ["//"], + "tasks": { + "popularity-cache-marker": { + "cache": false, + "outputs": [".build/cache-marker"] + }, + "fetch-popularity": { + "dependsOn": ["popularity-cache-marker"], + "cache": true, + "outputs": [".build/node-popularity.json"], + "inputs": ["scripts/fetch-node-popularity.mjs", ".build/cache-marker"] + }, + "build": { + "dependsOn": ["^build", "fetch-popularity"], + "outputs": ["dist/**"] + } + } +} diff --git a/packages/frontend/editor-ui/vite.config.mts b/packages/frontend/editor-ui/vite.config.mts index 4c8e214227..3d5dbc4fdb 100644 --- a/packages/frontend/editor-ui/vite.config.mts +++ b/packages/frontend/editor-ui/vite.config.mts @@ -13,6 +13,7 @@ import browserslistToEsbuild from 'browserslist-to-esbuild'; import legacy from '@vitejs/plugin-legacy'; import browserslist from 'browserslist'; import { isLocaleFile, sendLocaleUpdate } from './vite/i18n-locales-hmr-helpers'; +import { nodePopularityPlugin } from './vite/vite-plugin-node-popularity.mjs'; const publicPath = process.env.VUE_APP_PUBLIC_PATH || '/'; @@ -75,6 +76,7 @@ const alias = [ ]; const plugins: UserConfig['plugins'] = [ + nodePopularityPlugin(), icons({ compiler: 'vue3', autoInstall: true, diff --git a/packages/frontend/editor-ui/vite/vite-plugin-node-popularity.mts b/packages/frontend/editor-ui/vite/vite-plugin-node-popularity.mts new file mode 100644 index 0000000000..6b4ced9f34 --- /dev/null +++ b/packages/frontend/editor-ui/vite/vite-plugin-node-popularity.mts @@ -0,0 +1,32 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import type { Plugin } from 'vite'; + +const VIRTUAL_MODULE_ID = 'virtual:node-popularity-data'; +const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID; + +export function nodePopularityPlugin(): Plugin { + return { + name: 'node-popularity-plugin', + resolveId(id) { + if (id === VIRTUAL_MODULE_ID) { + return RESOLVED_VIRTUAL_MODULE_ID; + } + }, + async load(id) { + if (id === RESOLVED_VIRTUAL_MODULE_ID) { + // Try to load the data from the build directory + const buildDataPath = path.join(process.cwd(), '.build', 'node-popularity.json'); + + try { + const data = await fs.readFile(buildDataPath, 'utf-8'); + return `export default ${data}`; + } catch (error) { + // If file doesn't exist, return empty array + console.warn('Node popularity data not found at', buildDataPath, '- using empty array'); + return 'export default []'; + } + } + }, + }; +}