feat(Email Trigger (IMAP) Node): Limit new mails fetched (#16926)

Co-authored-by: Oleg Ivaniv <me@olegivaniv.com>
Co-authored-by: Michael Kret <michael.k@radency.com>

packages/@n8n/eslint-config/src/configs/base.ts

@@ -19,6 +19,8 @@ export const baseConfig = tseslint.config(
 		'tsup.config.ts',
 		'jest.config.js',
 		'cypress.config.js',
+		'vite.config.ts',
+		'vitest.config.ts',
 	]),
 	eslint.configs.recommended,
 	tseslint.configs.recommended,
@@ -409,6 +411,7 @@ export const baseConfig = tseslint.config(
 		files: ['test/**/*.ts', '**/__tests__/*.ts', '**/*.test.ts', '**/*.cy.ts'],
 		rules: {
 			'n8n-local-rules/no-plain-errors': 'off',
+			'@typescript-eslint/unbound-method': 'off',
 			'n8n-local-rules/no-skipped-tests': process.env.NODE_ENV === 'development' ? 'warn' : 'error',
 		},
 	},

packages/@n8n/imap/jest.config.js

@@ -1,2 +0,0 @@
-/** @type {import('jest').Config} */
-module.exports = require('../../../jest.config');

packages/@n8n/imap/package.json

@@ -11,7 +11,8 @@
     "lint": "eslint . --quiet",
     "lintfix": "eslint . --fix",
     "watch": "tsc -p tsconfig.build.json --watch",
-    "test": "jest"
+    "test": "vitest run",
+    "test:dev": "vitest --silent=false"
   },
   "main": "dist/index.js",
   "module": "src/index.ts",
@@ -28,9 +29,11 @@
   },
   "devDependencies": {
     "@n8n/typescript-config": "workspace:*",
+    "@n8n/vitest-config": "workspace:*",
     "@types/imap": "^0.8.40",
     "@types/quoted-printable": "^1.0.2",
     "@types/utf8": "^3.0.3",
-    "@types/uuencode": "^0.0.3"
+    "@types/uuencode": "^0.0.3",
+     "vitest-mock-extended": "catalog:"
   }
 }

packages/@n8n/imap/src/imap-simple.test.ts

@@ -0,0 +1,266 @@
+import { EventEmitter } from 'events';
+import Imap, { type Box, type MailBoxes } from 'imap';
+import { Readable } from 'stream';
+import type { Mocked } from 'vitest';
+import { mock } from 'vitest-mock-extended';
+
+import { ImapSimple } from './imap-simple';
+import { PartData } from './part-data';
+
+type MockImap = EventEmitter & {
+	connect: Mocked<() => unknown>;
+	fetch: Mocked<() => unknown>;
+	end: Mocked<() => unknown>;
+	search: Mocked<(...args: Parameters<Imap['search']>) => unknown>;
+	sort: Mocked<(...args: Parameters<Imap['sort']>) => unknown>;
+	openBox: Mocked<
+		(boxName: string, onOpen: (error: Error | null, box?: Box) => unknown) => unknown
+	>;
+	closeBox: Mocked<(...args: Parameters<Imap['closeBox']>) => unknown>;
+	getBoxes: Mocked<(onBoxes: (error: Error | null, boxes?: MailBoxes) => unknown) => unknown>;
+	addFlags: Mocked<(...args: Parameters<Imap['addFlags']>) => unknown>;
+};
+
+vi.mock('imap', () => {
+	return {
+		default: class InlineMockImap extends EventEmitter implements MockImap {
+			connect = vi.fn();
+			fetch = vi.fn();
+			end = vi.fn();
+			search = vi.fn();
+			sort = vi.fn();
+			openBox = vi.fn();
+			closeBox = vi.fn();
+			addFlags = vi.fn();
+			getBoxes = vi.fn();
+		},
+	};
+});
+
+vi.mock('./part-data', () => ({
+	// eslint-disable-next-line @typescript-eslint/naming-convention
+	PartData: { fromData: vi.fn(() => 'decoded') },
+}));
+
+describe('ImapSimple', () => {
+	function createImap() {
+		const imap = new Imap({ user: 'testuser', password: 'testpass' });
+		return { imapSimple: new ImapSimple(imap), mockImap: imap as unknown as MockImap };
+	}
+
+	describe('constructor', () => {
+		it('should forward nonerror events', () => {
+			const { imapSimple, mockImap } = createImap();
+			const onMail = vi.fn();
+			imapSimple.on('mail', onMail);
+			mockImap.emit('mail', 3);
+			expect(onMail).toHaveBeenCalledWith(3);
+		});
+
+		it('should suppress ECONNRESET errors if ending', () => {
+			const { imapSimple, mockImap } = createImap();
+			const onError = vi.fn();
+			imapSimple.on('error', onError);
+			imapSimple.end();
+
+			mockImap.emit('error', { message: 'reset', code: 'ECONNRESET' });
+			expect(onError).not.toHaveBeenCalled();
+		});
+
+		it('should forward ECONNRESET errors if not ending', () => {
+			const { imapSimple, mockImap } = createImap();
+			const onError = vi.fn();
+			imapSimple.on('error', onError);
+
+			const error = { message: 'reset', code: 'ECONNRESET' };
+			mockImap.emit('error', error);
+			expect(onError).toHaveBeenCalledWith(error);
+		});
+	});
+
+	describe('search', () => {
+		it('should resolve with messages returned from fetch', async () => {
+			const { imapSimple, mockImap } = createImap();
+
+			const fetchEmitter = new EventEmitter();
+			const mockMessages = [{ uid: 1 }, { uid: 2 }, { uid: 3 }];
+			vi.mocked(mockImap.search).mockImplementation((_criteria, onResult) =>
+				onResult(
+					null as unknown as Error,
+					mockMessages.map((m) => m.uid),
+				),
+			);
+			mockImap.fetch = vi.fn(() => fetchEmitter);
+
+			const searchPromise = imapSimple.search(['UNSEEN', ['FROM', 'test@n8n.io']], {
+				bodies: ['BODY'],
+			});
+			expect(mockImap.search).toHaveBeenCalledWith(
+				['UNSEEN', ['FROM', 'test@n8n.io']],
+				expect.any(Function),
+			);
+
+			for (const message of mockMessages) {
+				const messageEmitter = new EventEmitter();
+				const body = 'body' + message.uid;
+				const bodyStream = Readable.from(body);
+				fetchEmitter.emit('message', messageEmitter, message.uid);
+				messageEmitter.emit('body', bodyStream, { which: 'TEXT', size: Buffer.byteLength(body) });
+				messageEmitter.emit('attributes', { uid: message.uid });
+				await new Promise((resolve) => {
+					bodyStream.on('end', resolve);
+				});
+				messageEmitter.emit('end');
+			}
+
+			fetchEmitter.emit('end');
+
+			const messages = await searchPromise;
+
+			expect(messages).toEqual([
+				{
+					attributes: { uid: 1 },
+					parts: [{ body: 'body1', size: 5, which: 'TEXT' }],
+					seqNo: 1,
+				},
+				{
+					attributes: { uid: 2 },
+					parts: [{ body: 'body2', size: 5, which: 'TEXT' }],
+					seqNo: 2,
+				},
+				{
+					attributes: { uid: 3 },
+					parts: [{ body: 'body3', size: 5, which: 'TEXT' }],
+					seqNo: 3,
+				},
+			]);
+		});
+	});
+
+	describe('getPartData', () => {
+		it('should return decoded part data', async () => {
+			const { imapSimple, mockImap } = createImap();
+
+			const fetchEmitter = new EventEmitter();
+			mockImap.fetch = vi.fn(() => fetchEmitter);
+
+			const message = { attributes: { uid: 123 } };
+			const part = { partID: '1.2', encoding: 'BASE64' };
+
+			const partDataPromise = imapSimple.getPartData(mock(message), mock(part));
+
+			const body = 'encoded-body';
+			const messageEmitter = new EventEmitter();
+			const bodyStream = Readable.from(body);
+
+			fetchEmitter.emit('message', messageEmitter);
+
+			messageEmitter.emit('body', bodyStream, {
+				which: part.partID,
+				size: Buffer.byteLength(body),
+			});
+			messageEmitter.emit('attributes', {});
+			await new Promise((resolve) => bodyStream.on('end', resolve));
+			messageEmitter.emit('end');
+
+			fetchEmitter.emit('end');
+
+			const result = await partDataPromise;
+			expect(PartData.fromData).toHaveBeenCalledWith('encoded-body', 'BASE64');
+			expect(result).toBe('decoded');
+		});
+	});
+
+	describe('openBox', () => {
+		it('should open the mailbox', async () => {
+			const { imapSimple, mockImap } = createImap();
+			const box = mock<Box>({ name: 'INBOX' });
+			vi.mocked(mockImap.openBox).mockImplementation((_boxName, onOpen) =>
+				onOpen(null as unknown as Error, box),
+			);
+			await expect(imapSimple.openBox('INBOX')).resolves.toEqual(box);
+		});
+
+		it('should reject on error', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.openBox).mockImplementation((_boxName, onOpen) =>
+				onOpen(new Error('nope')),
+			);
+			await expect(imapSimple.openBox('INBOX')).rejects.toThrow('nope');
+		});
+	});
+
+	describe('closeBox', () => {
+		it('should close the mailbox with default autoExpunge=true', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) =>
+				onClose(null as unknown as Error),
+			);
+			await expect(imapSimple.closeBox()).resolves.toBeUndefined();
+			expect(mockImap.closeBox).toHaveBeenCalledWith(true, expect.any(Function));
+		});
+
+		it('should close the mailbox with autoExpunge=false', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) =>
+				onClose(null as unknown as Error),
+			);
+			await expect(imapSimple.closeBox(false)).resolves.toBeUndefined();
+			expect(mockImap.closeBox).toHaveBeenCalledWith(false, expect.any(Function));
+		});
+
+		it('should reject on error', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) =>
+				onClose(new Error('fail')),
+			);
+			await expect(imapSimple.closeBox()).rejects.toThrow('fail');
+		});
+	});
+
+	describe('addFlags', () => {
+		it('should add flags to messages and resolve', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.addFlags).mockImplementation((_uids, _flags, onAdd) =>
+				onAdd(null as unknown as Error),
+			);
+
+			await expect(imapSimple.addFlags([1, 2], ['\\Seen'])).resolves.toBeUndefined();
+			expect(mockImap.addFlags).toHaveBeenCalledWith([1, 2], ['\\Seen'], expect.any(Function));
+		});
+
+		it('should reject on error', async () => {
+			const { imapSimple, mockImap } = createImap();
+			vi.mocked(mockImap.addFlags).mockImplementation((_uids, _flags, onAdd) =>
+				onAdd(new Error('add flags failed')),
+			);
+
+			await expect(imapSimple.addFlags([1], '\\Seen')).rejects.toThrow('add flags failed');
+		});
+	});
+
+	describe('getBoxes', () => {
+		it('should resolve with list of mailboxes', async () => {
+			const { imapSimple, mockImap } = createImap();
+			// eslint-disable-next-line @typescript-eslint/naming-convention
+			const boxes = mock<MailBoxes>({ INBOX: {}, Archive: {} });
+
+			vi.mocked(mockImap.getBoxes).mockImplementation((onBoxes) =>
+				onBoxes(null as unknown as Error, boxes),
+			);
+
+			await expect(imapSimple.getBoxes()).resolves.toEqual(boxes);
+			expect(mockImap.getBoxes).toHaveBeenCalledWith(expect.any(Function));
+		});
+
+		it('should reject on error', async () => {
+			const { imapSimple, mockImap } = createImap();
+
+			vi.mocked(mockImap.getBoxes).mockImplementation((onBoxes) =>
+				onBoxes(new Error('getBoxes failed')),
+			);
+
+			await expect(imapSimple.getBoxes()).rejects.toThrow('getBoxes failed');
+		});
+	});
+});

packages/@n8n/imap/src/imap-simple.ts

@@ -4,7 +4,7 @@ import { type ImapMessage } from 'imap';
 
 import { getMessage } from './helpers/get-message';
 import { PartData } from './part-data';
-import type { Message, MessagePart } from './types';
+import type { Message, MessagePart, SearchCriteria } from './types';
 
 const IMAP_EVENTS = ['alert', 'mail', 'expunge', 'uidvalidity', 'update', 'close', 'end'] as const;
 
@@ -63,10 +63,11 @@ export class ImapSimple extends EventEmitter {
 	 */
 	async search(
 		/** Criteria to use to search. Passed to node-imap's .search() 1:1 */
-		// eslint-disable-next-line @typescript-eslint/no-explicit-any
-		searchCriteria: any[],
+		searchCriteria: SearchCriteria[],
 		/** Criteria to use to fetch the search results. Passed to node-imap's .fetch() 1:1 */
 		fetchOptions: Imap.FetchOptions,
+		/** Optional limit to restrict the number of messages fetched */
+		limit?: number,
 	) {
 		return await new Promise<Message[]>((resolve, reject) => {
 			this.imap.search(searchCriteria, (e, uids) => {
@@ -80,17 +81,23 @@ export class ImapSimple extends EventEmitter {
 					return;
 				}
 
-				const fetch = this.imap.fetch(uids, fetchOptions);
+				// If limit is specified, take only the first N UIDs
+				let uidsToFetch = uids;
+				if (limit && limit > 0 && uids.length > limit) {
+					uidsToFetch = uids.slice(0, limit);
+				}
+
+				const fetch = this.imap.fetch(uidsToFetch, fetchOptions);
 				let messagesRetrieved = 0;
 				const messages: Message[] = [];
 
 				const fetchOnMessage = async (message: Imap.ImapMessage, seqNo: number) => {
 					const msg: Message = await getMessage(message);
 					msg.seqNo = seqNo;
-					messages[seqNo] = msg;
+					messages.push(msg);
 
 					messagesRetrieved++;
-					if (messagesRetrieved === uids.length) {
+					if (messagesRetrieved === uidsToFetch.length) {
 						resolve(messages.filter((m) => !!m));
 					}
 				};

packages/@n8n/imap/src/types.ts

@@ -39,3 +39,5 @@ export interface Message {
 	parts: MessageBodyPart[];
 	seqNo?: number;
 }
+
+export type SearchCriteria = string | [string, string];

packages/@n8n/imap/tsconfig.build.json

@@ -7,5 +7,5 @@
 		"tsBuildInfoFile": "dist/build.tsbuildinfo"
 	},
 	"include": ["src/**/*.ts"],
-	"exclude": ["test/**"]
+	"exclude": ["src/**/*.test.ts"]
 }

packages/@n8n/imap/tsconfig.json

@@ -2,9 +2,9 @@
 	"extends": "@n8n/typescript-config/tsconfig.common.json",
 	"compilerOptions": {
 		"rootDir": ".",
-		"types": ["node", "jest"],
+		"types": ["node", "vite/client", "vitest/globals"],
 		"baseUrl": "src",
 		"tsBuildInfoFile": "dist/typecheck.tsbuildinfo"
 	},
-	"include": ["src/**/*.ts", "test/**/*.ts"]
+	"include": ["src/**/*.ts"]
 }

packages/@n8n/imap/vite.config.ts

@@ -0,0 +1,3 @@
+import { vitestConfig } from '@n8n/vitest-config/node';
+
+export default vitestConfig;

packages/@n8n/utils/vite.config.ts

@@ -1,4 +1,3 @@
-import { defineConfig, mergeConfig } from 'vite';
 import { vitestConfig } from '@n8n/vitest-config/frontend';
 
-export default mergeConfig(defineConfig({}), vitestConfig);
+export default vitestConfig;

packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.ts

@@ -12,12 +12,13 @@ export class EmailReadImap extends VersionedNodeType {
 			icon: 'fa:inbox',
 			group: ['trigger'],
 			description: 'Triggers the workflow when a new email is received',
-			defaultVersion: 2,
+			defaultVersion: 2.1,
 		};
 
 		const nodeVersions: IVersionedNodeType['nodeVersions'] = {
 			1: new EmailReadImapV1(baseDescription),
 			2: new EmailReadImapV2(baseDescription),
+			2.1: new EmailReadImapV2(baseDescription),
 		};
 
 		super(nodeVersions, baseDescription);

packages/nodes-base/nodes/EmailReadImap/test/v2/utils.test.ts

@@ -1,7 +1,7 @@
 import { type ImapSimple } from '@n8n/imap';
-import { mock } from 'jest-mock-extended';
+import { mock, mockDeep } from 'jest-mock-extended';
 import { returnJsonArray } from 'n8n-core';
-import { type IDataObject, type ITriggerFunctions } from 'n8n-workflow';
+import type { INode, ITriggerFunctions } from 'n8n-workflow';
 
 import { getNewEmails } from '../../v2/utils';
 
@@ -9,7 +9,7 @@ describe('Test IMap V2 utils', () => {
 	afterEach(() => jest.resetAllMocks());
 
 	describe('getNewEmails', () => {
-		const triggerFunctions = mock<ITriggerFunctions>({
+		const triggerFunctions = mockDeep<ITriggerFunctions>({
 			helpers: { returnJsonArray },
 		});
 
@@ -73,27 +73,27 @@ describe('Test IMap V2 utils', () => {
 			];
 
 			expectedResults.forEach(async (expectedResult) => {
-				// use new staticData for each iteration
-				const staticData: IDataObject = {};
-
+				triggerFunctions.getNode.mockReturnValue(mock<INode>({ typeVersion: 2.1 }));
 				triggerFunctions.getNodeParameter
 					.calledWith('format')
 					.mockReturnValue(expectedResult.format);
 				triggerFunctions.getNodeParameter
 					.calledWith('dataPropertyAttachmentsPrefixName')
 					.mockReturnValue('resolved');
+				triggerFunctions.getWorkflowStaticData.mockReturnValue({});
 
-				const result = getNewEmails.call(
-					triggerFunctions,
+				const onEmailBatch = jest.fn();
+				await getNewEmails.call(triggerFunctions, {
 					imapConnection,
-					[],
-					staticData,
-					'',
+					searchCriteria: [],
+					postProcessAction: '',
 					getText,
 					getAttachment,
-				);
+					onEmailBatch,
+				});
 
-				await expect(result).resolves.toEqual([expectedResult.expected]);
+				expect(onEmailBatch).toHaveBeenCalledTimes(1);
+				expect(onEmailBatch).toHaveBeenCalledWith([expectedResult.expected]);
 			});
 		});
 	});

packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts

@@ -1,4 +1,4 @@
-import type { ImapSimple, ImapSimpleOptions, Message } from '@n8n/imap';
+import type { ImapSimple, ImapSimpleOptions, Message, SearchCriteria } from '@n8n/imap';
 import { connect as imapConnect, getParts } from '@n8n/imap';
 import find from 'lodash/find';
 import isEmpty from 'lodash/isEmpty';
@@ -343,7 +343,7 @@ export class EmailReadImapV1 implements INodeType {
 		// Returns all the new unseen messages
 		const getNewEmails = async (
 			imapConnection: ImapSimple,
-			searchCriteria: Array<string | string[]>,
+			searchCriteria: SearchCriteria[],
 		): Promise<INodeExecutionData[]> => {
 			const format = this.getNodeParameter('format', 0) as string;
 
@@ -508,7 +508,7 @@ export class EmailReadImapV1 implements INodeType {
 		const returnedPromise = this.helpers.createDeferredPromise();
 
 		const establishConnection = async (): Promise<ImapSimple> => {
-			let searchCriteria = ['UNSEEN'] as Array<string | string[]>;
+			let searchCriteria: SearchCriteria[] = ['UNSEEN'];
 			if (options.customEmailConfig !== undefined) {
 				try {
 					searchCriteria = JSON.parse(options.customEmailConfig as string);

packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts

@@ -1,6 +1,15 @@
-import type { ImapSimple, ImapSimpleOptions, Message, MessagePart } from '@n8n/imap';
+import type { ICredentialsDataImap } from '@credentials/Imap.credentials';
+import { isCredentialsDataImap } from '@credentials/Imap.credentials';
+import type {
+	ImapSimple,
+	ImapSimpleOptions,
+	Message,
+	MessagePart,
+	SearchCriteria,
+} from '@n8n/imap';
 import { connect as imapConnect } from '@n8n/imap';
 import isEmpty from 'lodash/isEmpty';
+import { DateTime } from 'luxon';
 import type {
 	ITriggerFunctions,
 	IBinaryData,
@@ -13,13 +22,11 @@ import type {
 	INodeTypeDescription,
 	ITriggerResponse,
 	JsonObject,
+	INodeExecutionData,
 } from 'n8n-workflow';
 import { NodeConnectionTypes, NodeOperationError, TriggerCloseError } from 'n8n-workflow';
 import rfc2047 from 'rfc2047';
 
-import type { ICredentialsDataImap } from '@credentials/Imap.credentials';
-import { isCredentialsDataImap } from '@credentials/Imap.credentials';
-
 import { getNewEmails } from './utils';
 
 const versionDescription: INodeTypeDescription = {
@@ -28,7 +35,7 @@ const versionDescription: INodeTypeDescription = {
 	icon: 'fa:inbox',
 	iconColor: 'green',
 	group: ['trigger'],
-	version: 2,
+	version: [2, 2.1],
 	description: 'Triggers the workflow when a new email is received',
 	eventTriggerDescription: 'Waiting for you to receive an email',
 	defaults: {
@@ -248,6 +255,7 @@ export class EmailReadImapV2 implements INodeType {
 		const mailbox = this.getNodeParameter('mailbox') as string;
 		const postProcessAction = this.getNodeParameter('postProcessAction') as string;
 		const options = this.getNodeParameter('options', {}) as IDataObject;
+		const activatedAt = DateTime.now();
 
 		const staticData = this.getWorkflowStaticData('node');
 		this.logger.debug('Loaded static data for node "EmailReadImap"', { staticData });
@@ -333,12 +341,10 @@ export class EmailReadImapV2 implements INodeType {
 		const returnedPromise = this.helpers.createDeferredPromise();
 
 		const establishConnection = async (): Promise<ImapSimple> => {
-			let searchCriteria = ['UNSEEN'] as Array<string | string[]>;
+			let searchCriteria: SearchCriteria[] = ['UNSEEN'];
 			if (options.customEmailConfig !== undefined) {
 				try {
-					searchCriteria = JSON.parse(options.customEmailConfig as string) as Array<
-						string | string[]
-					>;
+					searchCriteria = JSON.parse(options.customEmailConfig as string) as SearchCriteria[];
 				} catch (error) {
 					throw new NodeOperationError(this.getNode(), 'Custom email config is not valid JSON.');
 				}
@@ -353,10 +359,21 @@ export class EmailReadImapV2 implements INodeType {
 					tls: credentials.secure,
 					authTimeout: 20000,
 				},
-				onMail: async () => {
+				onMail: async (numEmails) => {
+					this.logger.debug('New emails received in node "EmailReadImap"', {
+						numEmails,
+					});
+
 					if (connection) {
+						/**
+						 * Only process new emails:
+						 * - If we've seen emails before (lastMessageUid is set), fetch messages higher UID.
+						 * - Otherwise, fetch emails received since the workflow activation date.
+						 *
+						 * Note: IMAP 'SINCE' only filters by date (not time),
+						 * so it may include emails from earlier on the activation day.
+						 */
 						if (staticData.lastMessageUid !== undefined) {
-							searchCriteria.push(['UID', `${staticData.lastMessageUid as number}:*`]);
 							/**
 							 * A short explanation about UIDs and how they work
 							 * can be found here: https://dev.to/kehers/imap-new-messages-since-last-check-44gm
@@ -369,24 +386,28 @@ export class EmailReadImapV2 implements INodeType {
 							 * - You can check if UIDs changed in the above example
 							 * by checking UIDValidity.
 							 */
-							this.logger.debug('Querying for new messages on node "EmailReadImap"', {
-								searchCriteria,
-							});
+							searchCriteria.push(['UID', `${staticData.lastMessageUid as number}:*`]);
+						} else {
+							searchCriteria.push(['SINCE', activatedAt.toFormat('dd-LLL-yyyy')]);
 						}
 
+						this.logger.debug('Querying for new messages on node "EmailReadImap"', {
+							searchCriteria,
+						});
+
 						try {
-							const returnData = await getNewEmails.call(
-								this,
-								connection,
+							await getNewEmails.call(this, {
+								imapConnection: connection,
 								searchCriteria,
-								staticData,
 								postProcessAction,
 								getText,
 								getAttachment,
-							);
-							if (returnData.length) {
-								this.emit([returnData]);
-							}
+								onEmailBatch: async (returnData: INodeExecutionData[]) => {
+									if (returnData.length) {
+										this.emit([returnData]);
+									}
+								},
+							});
 						} catch (error) {
 							this.logger.error('Email Read Imap node encountered an error fetching new emails', {
 								error: error as Error,
@@ -399,7 +420,7 @@ export class EmailReadImapV2 implements INodeType {
 						}
 					}
 				},
-				onUpdate: async (seqNo: number, info) => {
+				onUpdate: (seqNo: number, info) => {
 					this.logger.debug(`Email Read Imap:update ${seqNo}`, info);
 				},
 			};
@@ -420,8 +441,8 @@ export class EmailReadImapV2 implements INodeType {
 
 			// Connect to the IMAP server and open the mailbox
 			// that we get informed whenever a new email arrives
-			return await imapConnect(config).then(async (conn) => {
-				conn.on('close', async (_hadError: boolean) => {
+			return await imapConnect(config).then((conn) => {
+				conn.on('close', (_hadError: boolean) => {
 					if (isCurrentlyReconnecting) {
 						this.logger.debug('Email Read Imap: Connected closed for forced reconnecting');
 					} else if (closeFunctionWasCalled) {
@@ -431,7 +452,7 @@ export class EmailReadImapV2 implements INodeType {
 						this.emitError(new Error('Imap connection closed unexpectedly'));
 					}
 				});
-				conn.on('error', async (error) => {
+				conn.on('error', (error) => {
 					const errorCode = ((error as JsonObject).code as string).toUpperCase();
 					this.logger.debug(`IMAP connection experienced an error: (${errorCode})`, {
 						error: error as Error,

packages/nodes-base/nodes/EmailReadImap/v2/utils.ts

@@ -1,4 +1,10 @@
-import { getParts, type ImapSimple, type Message, type MessagePart } from '@n8n/imap';
+import {
+	getParts,
+	type ImapSimple,
+	type Message,
+	type MessagePart,
+	type SearchCriteria,
+} from '@n8n/imap';
 import find from 'lodash/find';
 import { simpleParser, type Source as ParserSource } from 'mailparser';
 import {
@@ -46,19 +52,30 @@ async function parseRawEmail(
 	} as INodeExecutionData;
 }
 
+const EMAIL_BATCH_SIZE = 20;
+
 export async function getNewEmails(
 	this: ITriggerFunctions,
-	imapConnection: ImapSimple,
-	searchCriteria: Array<string | string[]>,
-	staticData: IDataObject,
-	postProcessAction: string,
-	getText: (parts: MessagePart[], message: Message, subtype: string) => Promise<string>,
-	getAttachment: (
-		imapConnection: ImapSimple,
-		parts: MessagePart[],
-		message: Message,
-	) => Promise<IBinaryData[]>,
-): Promise<INodeExecutionData[]> {
+	{
+		getAttachment,
+		getText,
+		onEmailBatch,
+		imapConnection,
+		postProcessAction,
+		searchCriteria,
+	}: {
+		imapConnection: ImapSimple;
+		searchCriteria: SearchCriteria[];
+		postProcessAction: string;
+		getText: (parts: MessagePart[], message: Message, subtype: string) => Promise<string>;
+		getAttachment: (
+			imapConnection: ImapSimple,
+			parts: MessagePart[],
+			message: Message,
+		) => Promise<IBinaryData[]>;
+		onEmailBatch: (data: INodeExecutionData[]) => Promise<void>;
+	},
+) {
 	const format = this.getNodeParameter('format', 0) as string;
 
 	let fetchOptions = {};
@@ -77,150 +94,167 @@ export async function getNewEmails(
 		};
 	}
 
-	const results = await imapConnection.search(searchCriteria, fetchOptions);
+	let results: Message[] = [];
+	let maxUid = 0;
 
-	const newEmails: INodeExecutionData[] = [];
-	let newEmail: INodeExecutionData;
-	let attachments: IBinaryData[];
-	let propertyName: string;
+	const staticData = this.getWorkflowStaticData('node');
+	const limit = this.getNode().typeVersion >= 2.1 ? EMAIL_BATCH_SIZE : undefined;
 
-	// All properties get by default moved to metadata except the ones
-	// which are defined here which get set on the top level.
-	const topLevelProperties = ['cc', 'date', 'from', 'subject', 'to'];
+	do {
+		if (maxUid) {
+			searchCriteria = searchCriteria.filter((criteria: SearchCriteria) => {
+				if (Array.isArray(criteria)) {
+					return !['UID', 'SINCE'].includes(criteria[0]);
+				}
+				return true;
+			});
 
-	if (format === 'resolved') {
-		const dataPropertyAttachmentsPrefixName = this.getNodeParameter(
-			'dataPropertyAttachmentsPrefixName',
-		) as string;
-
-		for (const message of results) {
-			if (
-				staticData.lastMessageUid !== undefined &&
-				message.attributes.uid <= (staticData.lastMessageUid as number)
-			) {
-				continue;
-			}
-			if (
-				staticData.lastMessageUid === undefined ||
-				(staticData.lastMessageUid as number) < message.attributes.uid
-			) {
-				staticData.lastMessageUid = message.attributes.uid;
-			}
-			const part = find(message.parts, { which: '' });
-
-			if (part === undefined) {
-				throw new NodeOperationError(this.getNode(), 'Email part could not be parsed.');
-			}
-			const parsedEmail = await parseRawEmail.call(
-				this,
-				part.body as Buffer,
-				dataPropertyAttachmentsPrefixName,
-			);
-
-			parsedEmail.json.attributes = {
-				uid: message.attributes.uid,
-			};
-
-			newEmails.push(parsedEmail);
+			searchCriteria.push(['UID', `${maxUid}:*`]);
 		}
-	} else if (format === 'simple') {
-		const downloadAttachments = this.getNodeParameter('downloadAttachments') as boolean;
+		results = await imapConnection.search(searchCriteria, fetchOptions, limit);
 
-		let dataPropertyAttachmentsPrefixName = '';
-		if (downloadAttachments) {
-			dataPropertyAttachmentsPrefixName = this.getNodeParameter(
+		this.logger.debug(`Process ${results.length} new emails in node "EmailReadImap"`);
+
+		const newEmails: INodeExecutionData[] = [];
+		let newEmail: INodeExecutionData;
+		let attachments: IBinaryData[];
+		let propertyName: string;
+
+		// All properties get by default moved to metadata except the ones
+		// which are defined here which get set on the top level.
+		const topLevelProperties = ['cc', 'date', 'from', 'subject', 'to'];
+
+		if (format === 'resolved') {
+			const dataPropertyAttachmentsPrefixName = this.getNodeParameter(
 				'dataPropertyAttachmentsPrefixName',
 			) as string;
-		}
 
-		for (const message of results) {
-			if (
-				staticData.lastMessageUid !== undefined &&
-				message.attributes.uid <= (staticData.lastMessageUid as number)
-			) {
-				continue;
-			}
-			if (
-				staticData.lastMessageUid === undefined ||
-				(staticData.lastMessageUid as number) < message.attributes.uid
-			) {
-				staticData.lastMessageUid = message.attributes.uid;
-			}
-			const parts = getParts(message.attributes.struct as IDataObject[]);
-
-			newEmail = {
-				json: {
-					textHtml: await getText(parts, message, 'html'),
-					textPlain: await getText(parts, message, 'plain'),
-					metadata: {} as IDataObject,
-					attributes: {
-						uid: message.attributes.uid,
-					} as IDataObject,
-				},
-			};
-
-			const messageHeader = message.parts.filter((part) => part.which === 'HEADER');
-
-			const messageBody = messageHeader[0].body as Record<string, string[]>;
-			for (propertyName of Object.keys(messageBody)) {
-				if (messageBody[propertyName].length) {
-					if (topLevelProperties.includes(propertyName)) {
-						newEmail.json[propertyName] = messageBody[propertyName][0];
-					} else {
-						(newEmail.json.metadata as IDataObject)[propertyName] = messageBody[propertyName][0];
-					}
+			for (const message of results) {
+				const lastMessageUid = this.getWorkflowStaticData('node').lastMessageUid as number;
+				if (lastMessageUid !== undefined && message.attributes.uid <= lastMessageUid) {
+					continue;
 				}
-			}
 
+				// Track the maximum UID to update staticData later
+				if (message.attributes.uid > maxUid) {
+					maxUid = message.attributes.uid;
+				}
+				const part = find(message.parts, { which: '' });
+
+				if (part === undefined) {
+					throw new NodeOperationError(this.getNode(), 'Email part could not be parsed.');
+				}
+				const parsedEmail = await parseRawEmail.call(
+					this,
+					part.body as Buffer,
+					dataPropertyAttachmentsPrefixName,
+				);
+
+				parsedEmail.json.attributes = {
+					uid: message.attributes.uid,
+				};
+
+				newEmails.push(parsedEmail);
+			}
+		} else if (format === 'simple') {
+			const downloadAttachments = this.getNodeParameter('downloadAttachments') as boolean;
+
+			let dataPropertyAttachmentsPrefixName = '';
 			if (downloadAttachments) {
-				// Get attachments and add them if any get found
-				attachments = await getAttachment(imapConnection, parts, message);
-				if (attachments.length) {
-					newEmail.binary = {};
-					for (let i = 0; i < attachments.length; i++) {
-						newEmail.binary[`${dataPropertyAttachmentsPrefixName}${i}`] = attachments[i];
+				dataPropertyAttachmentsPrefixName = this.getNodeParameter(
+					'dataPropertyAttachmentsPrefixName',
+				) as string;
+			}
+
+			for (const message of results) {
+				const lastMessageUid = this.getWorkflowStaticData('node').lastMessageUid as number;
+				if (lastMessageUid !== undefined && message.attributes.uid <= lastMessageUid) {
+					continue;
+				}
+
+				// Track the maximum UID to update staticData later
+				if (message.attributes.uid > maxUid) {
+					maxUid = message.attributes.uid;
+				}
+				const parts = getParts(message.attributes.struct as IDataObject[]);
+
+				newEmail = {
+					json: {
+						textHtml: await getText(parts, message, 'html'),
+						textPlain: await getText(parts, message, 'plain'),
+						metadata: {} as IDataObject,
+						attributes: {
+							uid: message.attributes.uid,
+						} as IDataObject,
+					},
+				};
+
+				const messageHeader = message.parts.filter((part) => part.which === 'HEADER');
+
+				const messageBody = messageHeader[0].body as Record<string, string[]>;
+				for (propertyName of Object.keys(messageBody)) {
+					if (messageBody[propertyName].length) {
+						if (topLevelProperties.includes(propertyName)) {
+							newEmail.json[propertyName] = messageBody[propertyName][0];
+						} else {
+							(newEmail.json.metadata as IDataObject)[propertyName] = messageBody[propertyName][0];
+						}
 					}
 				}
-			}
 
-			newEmails.push(newEmail);
+				if (downloadAttachments) {
+					// Get attachments and add them if any get found
+					attachments = await getAttachment(imapConnection, parts, message);
+					if (attachments.length) {
+						newEmail.binary = {};
+						for (let i = 0; i < attachments.length; i++) {
+							newEmail.binary[`${dataPropertyAttachmentsPrefixName}${i}`] = attachments[i];
+						}
+					}
+				}
+
+				newEmails.push(newEmail);
+			}
+		} else if (format === 'raw') {
+			for (const message of results) {
+				const lastMessageUid = this.getWorkflowStaticData('node').lastMessageUid as number;
+				if (lastMessageUid !== undefined && message.attributes.uid <= lastMessageUid) {
+					continue;
+				}
+
+				// Track the maximum UID to update staticData later
+				if (message.attributes.uid > maxUid) {
+					maxUid = message.attributes.uid;
+				}
+				const part = find(message.parts, { which: 'TEXT' });
+
+				if (part === undefined) {
+					throw new NodeOperationError(this.getNode(), 'Email part could not be parsed.');
+				}
+				// Return base64 string
+				newEmail = {
+					json: {
+						raw: part.body as string,
+					},
+				};
+
+				newEmails.push(newEmail);
+			}
 		}
-	} else if (format === 'raw') {
-		for (const message of results) {
-			if (
-				staticData.lastMessageUid !== undefined &&
-				message.attributes.uid <= (staticData.lastMessageUid as number)
-			) {
-				continue;
-			}
-			if (
-				staticData.lastMessageUid === undefined ||
-				(staticData.lastMessageUid as number) < message.attributes.uid
-			) {
-				staticData.lastMessageUid = message.attributes.uid;
-			}
-			const part = find(message.parts, { which: 'TEXT' });
 
-			if (part === undefined) {
-				throw new NodeOperationError(this.getNode(), 'Email part could not be parsed.');
+		// only mark messages as seen once processing has finished
+		if (postProcessAction === 'read') {
+			const uidList = results.map((e) => e.attributes.uid);
+			if (uidList.length > 0) {
+				await imapConnection.addFlags(uidList, '\\SEEN');
 			}
-			// Return base64 string
-			newEmail = {
-				json: {
-					raw: part.body as string,
-				},
-			};
-
-			newEmails.push(newEmail);
 		}
+
+		await onEmailBatch(newEmails);
+	} while (results.length >= EMAIL_BATCH_SIZE);
+
+	// Update lastMessageUid after processing all messages
+	if (maxUid > ((staticData.lastMessageUid as number) ?? 0)) {
+		this.getWorkflowStaticData('node').lastMessageUid = maxUid;
 	}
-
-	// only mark messages as seen once processing has finished
-	if (postProcessAction === 'read') {
-		const uidList = results.map((e) => e.attributes.uid);
-		if (uidList.length > 0) {
-			await imapConnection.addFlags(uidList, '\\SEEN');
-		}
-	}
-	return newEmails;
 }

packages/workflow/vitest.config.ts

@@ -1,8 +1,3 @@
-/* eslint-disable import-x/no-default-export */
-export default async () => {
-	const { createVitestConfig } = await import('@n8n/vitest-config/node');
+import { createVitestConfig } from '@n8n/vitest-config/node';
 
-	return createVitestConfig({
-		include: ['test/**/*.test.ts'],
-	});
-};
+export default createVitestConfig({ include: ['test/**/*.test.ts'] });

pnpm-lock.yaml

@@ -830,6 +830,9 @@ importers:
       '@n8n/typescript-config':
         specifier: workspace:*
         version: link:../typescript-config
+      '@n8n/vitest-config':
+        specifier: workspace:*
+        version: link:../vitest-config
       '@types/imap':
         specifier: ^0.8.40
         version: 0.8.40
@@ -842,6 +845,9 @@ importers:
       '@types/uuencode':
         specifier: ^0.0.3
         version: 0.0.3(patch_hash=083a73709a54db57b092d986b43d27ddda3cb8008f9510e98bc9e6da0e1cbb62)
+      vitest-mock-extended:
+        specifier: 'catalog:'
+        version: 3.1.0(typescript@5.8.3)(vitest@3.1.3(@types/debug@4.1.12)(@types/node@20.19.1)(jiti@1.21.7)(jsdom@23.0.1)(sass@1.64.1)(terser@5.16.1)(tsx@4.19.3))
 
   packages/@n8n/json-schema-to-zod:
     devDependencies:
@@ -6865,8 +6871,8 @@ packages:
   '@types/docker-modem@3.0.6':
     resolution: {integrity: sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==}
 
-  '@types/dockerode@3.3.41':
-    resolution: {integrity: sha512-5kOi6bcnEjqfJ68ZNV/bBvSMLNIucc0XbRmBO4hg5OoFCoP99eSRcbMysjkzV7ZxQEmmc/zMnv4A7odwuKFzDA==}
+  '@types/dockerode@3.3.42':
+    resolution: {integrity: sha512-U1jqHMShibMEWHdxYhj3rCMNCiLx5f35i4e3CEUuW+JSSszc/tVqc6WCAPdhwBymG5R/vgbcceagK0St7Cq6Eg==}
 
   '@types/eslint@9.6.1':
     resolution: {integrity: sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==}
@@ -11521,10 +11527,6 @@ packages:
     resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==}
     engines: {node: '>=10'}
 
-  lilconfig@3.1.2:
-    resolution: {integrity: sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==}
-    engines: {node: '>=14'}
-
   lilconfig@3.1.3:
     resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==}
     engines: {node: '>=14'}
@@ -18986,7 +18988,7 @@ snapshots:
   '@n8n/localtunnel@3.0.0':
     dependencies:
       axios: 1.10.0(debug@4.3.6)
-      debug: 4.3.6(supports-color@8.1.1)
+      debug: 4.3.6
     transitivePeerDependencies:
       - supports-color
 
@@ -19172,13 +19174,13 @@ snapshots:
       ansis: 3.2.0
       clean-stack: 3.0.1
       cli-spinners: 2.9.2
-      debug: 4.3.6(supports-color@8.1.1)
+      debug: 4.4.1(supports-color@8.1.1)
       ejs: 3.1.10
       get-package-type: 0.1.0
       globby: 11.1.0
       indent-string: 4.0.0
       is-wsl: 2.2.0
-      lilconfig: 3.1.2
+      lilconfig: 3.1.3
       minimatch: 9.0.5
       string-width: 4.2.3
       supports-color: 8.1.1
@@ -20858,7 +20860,7 @@ snapshots:
       '@types/node': 20.19.1
       '@types/ssh2': 1.11.6
 
-  '@types/dockerode@3.3.41':
+  '@types/dockerode@3.3.42':
     dependencies:
       '@types/docker-modem': 3.0.6
       '@types/node': 20.19.1
@@ -23489,11 +23491,9 @@ snapshots:
     dependencies:
       ms: 2.1.2
 
-  debug@4.3.6(supports-color@8.1.1):
+  debug@4.3.6:
     dependencies:
       ms: 2.1.2
-    optionalDependencies:
-      supports-color: 8.1.1
 
   debug@4.4.0:
     dependencies:
@@ -24690,7 +24690,7 @@ snapshots:
 
   follow-redirects@1.15.9(debug@4.3.6):
     optionalDependencies:
-      debug: 4.3.6(supports-color@8.1.1)
+      debug: 4.3.6
 
   follow-redirects@1.15.9(debug@4.4.0):
     optionalDependencies:
@@ -26736,8 +26736,6 @@ snapshots:
 
   lilconfig@2.1.0: {}
 
-  lilconfig@3.1.2: {}
-
   lilconfig@3.1.3: {}
 
   lines-and-columns@1.2.4: {}
@@ -28397,7 +28395,7 @@ snapshots:
 
   postcss-load-config@4.0.2(postcss@8.4.49)(ts-node@10.9.2(@types/node@20.19.1)(typescript@5.8.3)):
     dependencies:
-      lilconfig: 3.1.2
+      lilconfig: 3.1.3
       yaml: 2.3.4
     optionalDependencies:
       postcss: 8.4.49
@@ -30126,7 +30124,7 @@ snapshots:
   testcontainers@11.0.3:
     dependencies:
       '@balena/dockerignore': 1.0.2
-      '@types/dockerode': 3.3.41
+      '@types/dockerode': 3.3.42
       archiver: 7.0.1
       async-lock: 1.4.1
       byline: 5.0.0