feat(Email Trigger (IMAP) Node): Limit new mails fetched (#16926)

Co-authored-by: Oleg Ivaniv <me@olegivaniv.com>
Co-authored-by: Michael Kret <michael.k@radency.com>

packages/@n8n/imap/jest.config.js

@@ -1,2 +0,0 @@
-/** @type {import('jest').Config} */
-module.exports = require('../../../jest.config');

packages/@n8n/imap/package.json

@@ -11,7 +11,8 @@
     "lint": "eslint . --quiet",
     "lintfix": "eslint . --fix",
     "watch": "tsc -p tsconfig.build.json --watch",
-    "test": "jest"
+    "test": "vitest run",
+    "test:dev": "vitest --silent=false"
   },
   "main": "dist/index.js",
   "module": "src/index.ts",
@@ -28,9 +29,11 @@
   },
   "devDependencies": {
     "@n8n/typescript-config": "workspace:*",
+    "@n8n/vitest-config": "workspace:*",
     "@types/imap": "^0.8.40",
     "@types/quoted-printable": "^1.0.2",
     "@types/utf8": "^3.0.3",
-    "@types/uuencode": "^0.0.3"
+    "@types/uuencode": "^0.0.3",
+     "vitest-mock-extended": "catalog:"
   }
 }

packages/@n8n/imap/src/imap-simple.test.ts

@@ -0,0 +1,266 @@
+import { EventEmitter } from 'events';
+import Imap, { type Box, type MailBoxes } from 'imap';
+import { Readable } from 'stream';
+import type { Mocked } from 'vitest';
+import { mock } from 'vitest-mock-extended';
+
+import { ImapSimple } from './imap-simple';
+import { PartData } from './part-data';
+
+type MockImap = EventEmitter & {
+	connect: Mocked<() => unknown>;
+	fetch: Mocked<() => unknown>;
+	end: Mocked<() => unknown>;
+	search: Mocked<(...args: Parameters<Imap['search']>) => unknown>;
+	sort: Mocked<(...args: Parameters<Imap['sort']>) => unknown>;
+	openBox: Mocked<
+		(boxName: string, onOpen: (error: Error | null, box?: Box) => unknown) => unknown
+	>;
+	closeBox: Mocked<(...args: Parameters<Imap['closeBox']>) => unknown>;
+	getBoxes: Mocked<(onBoxes: (error: Error | null, boxes?: MailBoxes) => unknown) => unknown>;
+	addFlags: Mocked<(...args: Parameters<Imap['addFlags']>) => unknown>;
+};
+
+vi.mock('imap', () => {
+	return {
+		default: class InlineMockImap extends EventEmitter implements MockImap {
+			connect = vi.fn();
+			fetch = vi.fn();
+			end = vi.fn();
+			search = vi.fn();
+			sort = vi.fn();
+			openBox = vi.fn();
+			closeBox = vi.fn();
+			addFlags = vi.fn();
+			getBoxes = vi.fn();
+		},
+	};
+});
+
+vi.mock('./part-data', () => ({
+	// eslint-disable-next-line @typescript-eslint/naming-convention
+	PartData: { fromData: vi.fn(() => 'decoded') },
+}));
+
+describe('ImapSimple', () => {
+	function createImap() {
+		const imap = new Imap({ user: 'testuser', password: 'testpass' });
+		return { imapSimple: new ImapSimple(imap), mockImap: imap as unknown as MockImap };
+	}
+
+	describe('constructor', () => {
+		it('should forward nonerror events', () => {
+			const { imapSimple, mockImap } = createImap();
+			const onMail = vi.fn();
+			imapSimple.on('mail', onMail);
+			mockImap.emit('mail', 3);
+			expect(onMail).toHaveBeenCalledWith(3);
+		});
+
+		it('should suppress ECONNRESET errors if ending', () => {
+			const { imapSimple, mockImap } = createImap();
+			const onError = vi.fn();
+			imapSimple.on('error', onError);
+			imapSimple.end();
+
+			mockImap.emit('error', { message: 'reset', code: 'ECONNRESET' });
+			expect(onError).not.toHaveBeenCalled();
+		});
+
+		it('should forward ECONNRESET errors if not ending', () => {
+			const { imapSimple, mockImap } = createImap();
+			const onError = vi.fn();
+			imapSimple.on('error', onError);
+
+			const error = { message: 'reset', code: 'ECONNRESET' };
+			mockImap.emit('error', error);
+			expect(onError).toHaveBeenCalledWith(error);
+		});
+	});
+
+	describe('search', () => {
+		it('should resolve with messages returned from fetch', async () => {
+			const { imapSimple, mockImap } = createImap();
+
+			const fetchEmitter = new EventEmitter();
+			const mockMessages = [{ uid: 1 }, { uid: 2 }, { uid: 3 }];
+			vi.mocked(mockImap.search).mockImplementation((_criteria, onResult) =>
+				onResult(
+					null as unknown as Error,
+					mockMessages.map((m) => m.uid),
+				),
+			);
+			mockImap.fetch = vi.fn(() => fetchEmitter);
+
+			const searchPromise = imapSimple.search(['UNSEEN', ['FROM', 'test@n8n.io']], {
+				bodies: ['BODY'],
+			});
+			expect(mockImap.search).toHaveBeenCalledWith(
+				['UNSEEN', ['FROM', 'test@n8n.io']],
+				expect.any(Function),
+			);
+
+			for (const message of mockMessages) {
+				const messageEmitter = new EventEmitter();
+				const body = 'body' + message.uid;
+				const bodyStream = Readable.from(body);
+				fetchEmitter.emit('message', messageEmitter, message.uid);
+				messageEmitter.emit('body', bodyStream, { which: 'TEXT', size: Buffer.byteLength(body) });
+				messageEmitter.emit('attributes', { uid: message.uid });
+				await new Promise((resolve) => {
+					bodyStream.on('end', resolve);
+				});
+				messageEmitter.emit('end');
+			}
+
+			fetchEmitter.emit('end');
+
+			const messages = await searchPromise;
+
+			expect(messages).toEqual([
+				{
+					attributes: { uid: 1 },
+					parts: [{ body: 'body1', size: 5, which: 'TEXT' }],
+					seqNo: 1,
+				},
+				{
+					attributes: { uid: 2 },
+					parts: [{ body: 'body2', size: 5, which: 'TEXT' }],
+					seqNo: 2,
+				},
+				{
+					attributes: { uid: 3 },
+					parts: [{ body: 'body3', size: 5, which: 'TEXT' }],
+					seqNo: 3,
+				},
+			]);
+		});
+	});
+
+	describe('getPartData', () => {
+		it('should return decoded part data', async () => {
+			const { imapSimple, mockImap } = createImap();
+
+			const fetchEmitter = new EventEmitter();
+			mockImap.fetch = vi.fn(() => fetchEmitter);
+
+			const message = { attributes: { uid: 123 } };
+			const part = { partID: '1.2', encoding: 'BASE64' };
+
+			const partDataPromise = imapSimple.getPartData(mock(message), mock(part));
+
+			const body = 'encoded-body';
+			const messageEmitter = new EventEmitter();
+			const bodyStream = Readable.from(body);
+
+			fetchEmitter.emit('message', messageEmitter);
+
+			messageEmitter.emit('body', bodyStream, {
+				which: part.partID,
+				size: Buffer.byteLength(body),
+			});
+			messageEmitter.emit('attributes', {});
+			await new Promise((resolve) => bodyStream.on('end', resolve));
+			messageEmitter.emit('end');
+
+			fetchEmitter.emit('end');
+
+			const result = await partDataPromise;
+			expect(PartData.fromData).toHaveBeenCalledWith('encoded-body', 'BASE64');
+			expect(result).toBe('decoded');
+		});
+	});
+
+	describe('openBox', () => {
+		it('should open the mailbox', async () => {
+			const { imapSimple, mockImap } = createImap();
+			const box = mock<Box>({ name: 'INBOX' });
+			vi.mocked(mockImap.openBox).mockImplementation((_boxName, onOpen) =>
+				onOpen(null as unknown as Error, box),
+			);
+			await expect(imapSimple.openBox('INBOX')).resolves.toEqual(box);
+		});
+
+		it('should reject on error', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.openBox).mockImplementation((_boxName, onOpen) =>
+				onOpen(new Error('nope')),
+			);
+			await expect(imapSimple.openBox('INBOX')).rejects.toThrow('nope');
+		});
+	});
+
+	describe('closeBox', () => {
+		it('should close the mailbox with default autoExpunge=true', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) =>
+				onClose(null as unknown as Error),
+			);
+			await expect(imapSimple.closeBox()).resolves.toBeUndefined();
+			expect(mockImap.closeBox).toHaveBeenCalledWith(true, expect.any(Function));
+		});
+
+		it('should close the mailbox with autoExpunge=false', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) =>
+				onClose(null as unknown as Error),
+			);
+			await expect(imapSimple.closeBox(false)).resolves.toBeUndefined();
+			expect(mockImap.closeBox).toHaveBeenCalledWith(false, expect.any(Function));
+		});
+
+		it('should reject on error', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) =>
+				onClose(new Error('fail')),
+			);
+			await expect(imapSimple.closeBox()).rejects.toThrow('fail');
+		});
+	});
+
+	describe('addFlags', () => {
+		it('should add flags to messages and resolve', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.addFlags).mockImplementation((_uids, _flags, onAdd) =>
+				onAdd(null as unknown as Error),
+			);
+
+			await expect(imapSimple.addFlags([1, 2], ['\\Seen'])).resolves.toBeUndefined();
+			expect(mockImap.addFlags).toHaveBeenCalledWith([1, 2], ['\\Seen'], expect.any(Function));
+		});
+
+		it('should reject on error', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.addFlags).mockImplementation((_uids, _flags, onAdd) =>
+				onAdd(new Error('add flags failed')),
+			);
+
+			await expect(imapSimple.addFlags([1], '\\Seen')).rejects.toThrow('add flags failed');
+		});
+	});
+
+	describe('getBoxes', () => {
+		it('should resolve with list of mailboxes', async () => {
+			const { imapSimple, mockImap } = createImap();
+			// eslint-disable-next-line @typescript-eslint/naming-convention
+			const boxes = mock<MailBoxes>({ INBOX: {}, Archive: {} });
+
+			vi.mocked(mockImap.getBoxes).mockImplementation((onBoxes) =>
+				onBoxes(null as unknown as Error, boxes),
+			);
+
+			await expect(imapSimple.getBoxes()).resolves.toEqual(boxes);
+			expect(mockImap.getBoxes).toHaveBeenCalledWith(expect.any(Function));
+		});
+
+		it('should reject on error', async () => {
+			const { imapSimple, mockImap } = createImap();
+
+			vi.mocked(mockImap.getBoxes).mockImplementation((onBoxes) =>
+				onBoxes(new Error('getBoxes failed')),
+			);
+
+			await expect(imapSimple.getBoxes()).rejects.toThrow('getBoxes failed');
+		});
+	});
+});

packages/@n8n/imap/src/imap-simple.ts

@@ -4,7 +4,7 @@ import { type ImapMessage } from 'imap';
 
 import { getMessage } from './helpers/get-message';
 import { PartData } from './part-data';
-import type { Message, MessagePart } from './types';
+import type { Message, MessagePart, SearchCriteria } from './types';
 
 const IMAP_EVENTS = ['alert', 'mail', 'expunge', 'uidvalidity', 'update', 'close', 'end'] as const;
 
@@ -63,10 +63,11 @@ export class ImapSimple extends EventEmitter {
 	 */
 	async search(
 		/** Criteria to use to search. Passed to node-imap's .search() 1:1 */
-		// eslint-disable-next-line @typescript-eslint/no-explicit-any
-		searchCriteria: any[],
+		searchCriteria: SearchCriteria[],
 		/** Criteria to use to fetch the search results. Passed to node-imap's .fetch() 1:1 */
 		fetchOptions: Imap.FetchOptions,
+		/** Optional limit to restrict the number of messages fetched */
+		limit?: number,
 	) {
 		return await new Promise<Message[]>((resolve, reject) => {
 			this.imap.search(searchCriteria, (e, uids) => {
@@ -80,17 +81,23 @@ export class ImapSimple extends EventEmitter {
 					return;
 				}
 
-				const fetch = this.imap.fetch(uids, fetchOptions);
+				// If limit is specified, take only the first N UIDs
+				let uidsToFetch = uids;
+				if (limit && limit > 0 && uids.length > limit) {
+					uidsToFetch = uids.slice(0, limit);
+				}
+
+				const fetch = this.imap.fetch(uidsToFetch, fetchOptions);
 				let messagesRetrieved = 0;
 				const messages: Message[] = [];
 
 				const fetchOnMessage = async (message: Imap.ImapMessage, seqNo: number) => {
 					const msg: Message = await getMessage(message);
 					msg.seqNo = seqNo;
-					messages[seqNo] = msg;
+					messages.push(msg);
 
 					messagesRetrieved++;
-					if (messagesRetrieved === uids.length) {
+					if (messagesRetrieved === uidsToFetch.length) {
 						resolve(messages.filter((m) => !!m));
 					}
 				};

packages/@n8n/imap/src/types.ts

@@ -39,3 +39,5 @@ export interface Message {
 	parts: MessageBodyPart[];
 	seqNo?: number;
 }
+
+export type SearchCriteria = string | [string, string];

packages/@n8n/imap/tsconfig.build.json

@@ -7,5 +7,5 @@
 		"tsBuildInfoFile": "dist/build.tsbuildinfo"
 	},
 	"include": ["src/**/*.ts"],
-	"exclude": ["test/**"]
+	"exclude": ["src/**/*.test.ts"]
 }

packages/@n8n/imap/tsconfig.json

@@ -2,9 +2,9 @@
 	"extends": "@n8n/typescript-config/tsconfig.common.json",
 	"compilerOptions": {
 		"rootDir": ".",
-		"types": ["node", "jest"],
+		"types": ["node", "vite/client", "vitest/globals"],
 		"baseUrl": "src",
 		"tsBuildInfoFile": "dist/typecheck.tsbuildinfo"
 	},
-	"include": ["src/**/*.ts", "test/**/*.ts"]
+	"include": ["src/**/*.ts"]
 }

packages/@n8n/imap/vite.config.ts

@@ -0,0 +1,3 @@
+import { vitestConfig } from '@n8n/vitest-config/node';
+
+export default vitestConfig;