diff --git a/packages/@n8n/eslint-config/src/configs/base.ts b/packages/@n8n/eslint-config/src/configs/base.ts index aa8084de57..fda6f5f53b 100644 --- a/packages/@n8n/eslint-config/src/configs/base.ts +++ b/packages/@n8n/eslint-config/src/configs/base.ts @@ -19,6 +19,8 @@ export const baseConfig = tseslint.config( 'tsup.config.ts', 'jest.config.js', 'cypress.config.js', + 'vite.config.ts', + 'vitest.config.ts', ]), eslint.configs.recommended, tseslint.configs.recommended, @@ -409,6 +411,7 @@ export const baseConfig = tseslint.config( files: ['test/**/*.ts', '**/__tests__/*.ts', '**/*.test.ts', '**/*.cy.ts'], rules: { 'n8n-local-rules/no-plain-errors': 'off', + '@typescript-eslint/unbound-method': 'off', 'n8n-local-rules/no-skipped-tests': process.env.NODE_ENV === 'development' ? 'warn' : 'error', }, }, diff --git a/packages/@n8n/imap/jest.config.js b/packages/@n8n/imap/jest.config.js deleted file mode 100644 index d6c48554a7..0000000000 --- a/packages/@n8n/imap/jest.config.js +++ /dev/null @@ -1,2 +0,0 @@ -/** @type {import('jest').Config} */ -module.exports = require('../../../jest.config'); diff --git a/packages/@n8n/imap/package.json b/packages/@n8n/imap/package.json index e38fa270ed..532735afed 100644 --- a/packages/@n8n/imap/package.json +++ b/packages/@n8n/imap/package.json @@ -11,7 +11,8 @@ "lint": "eslint . --quiet", "lintfix": "eslint . --fix", "watch": "tsc -p tsconfig.build.json --watch", - "test": "jest" + "test": "vitest run", + "test:dev": "vitest --silent=false" }, "main": "dist/index.js", "module": "src/index.ts", @@ -28,9 +29,11 @@ }, "devDependencies": { "@n8n/typescript-config": "workspace:*", + "@n8n/vitest-config": "workspace:*", "@types/imap": "^0.8.40", "@types/quoted-printable": "^1.0.2", "@types/utf8": "^3.0.3", - "@types/uuencode": "^0.0.3" + "@types/uuencode": "^0.0.3", + "vitest-mock-extended": "catalog:" } } diff --git a/packages/@n8n/imap/src/imap-simple.test.ts b/packages/@n8n/imap/src/imap-simple.test.ts new file mode 100644 index 0000000000..896c09568a --- /dev/null +++ b/packages/@n8n/imap/src/imap-simple.test.ts @@ -0,0 +1,266 @@ +import { EventEmitter } from 'events'; +import Imap, { type Box, type MailBoxes } from 'imap'; +import { Readable } from 'stream'; +import type { Mocked } from 'vitest'; +import { mock } from 'vitest-mock-extended'; + +import { ImapSimple } from './imap-simple'; +import { PartData } from './part-data'; + +type MockImap = EventEmitter & { + connect: Mocked<() => unknown>; + fetch: Mocked<() => unknown>; + end: Mocked<() => unknown>; + search: Mocked<(...args: Parameters) => unknown>; + sort: Mocked<(...args: Parameters) => unknown>; + openBox: Mocked< + (boxName: string, onOpen: (error: Error | null, box?: Box) => unknown) => unknown + >; + closeBox: Mocked<(...args: Parameters) => unknown>; + getBoxes: Mocked<(onBoxes: (error: Error | null, boxes?: MailBoxes) => unknown) => unknown>; + addFlags: Mocked<(...args: Parameters) => unknown>; +}; + +vi.mock('imap', () => { + return { + default: class InlineMockImap extends EventEmitter implements MockImap { + connect = vi.fn(); + fetch = vi.fn(); + end = vi.fn(); + search = vi.fn(); + sort = vi.fn(); + openBox = vi.fn(); + closeBox = vi.fn(); + addFlags = vi.fn(); + getBoxes = vi.fn(); + }, + }; +}); + +vi.mock('./part-data', () => ({ + // eslint-disable-next-line @typescript-eslint/naming-convention + PartData: { fromData: vi.fn(() => 'decoded') }, +})); + +describe('ImapSimple', () => { + function createImap() { + const imap = new Imap({ user: 'testuser', password: 'testpass' }); + return { imapSimple: new ImapSimple(imap), mockImap: imap as unknown as MockImap }; + } + + describe('constructor', () => { + it('should forward nonerror events', () => { + const { imapSimple, mockImap } = createImap(); + const onMail = vi.fn(); + imapSimple.on('mail', onMail); + mockImap.emit('mail', 3); + expect(onMail).toHaveBeenCalledWith(3); + }); + + it('should suppress ECONNRESET errors if ending', () => { + const { imapSimple, mockImap } = createImap(); + const onError = vi.fn(); + imapSimple.on('error', onError); + imapSimple.end(); + + mockImap.emit('error', { message: 'reset', code: 'ECONNRESET' }); + expect(onError).not.toHaveBeenCalled(); + }); + + it('should forward ECONNRESET errors if not ending', () => { + const { imapSimple, mockImap } = createImap(); + const onError = vi.fn(); + imapSimple.on('error', onError); + + const error = { message: 'reset', code: 'ECONNRESET' }; + mockImap.emit('error', error); + expect(onError).toHaveBeenCalledWith(error); + }); + }); + + describe('search', () => { + it('should resolve with messages returned from fetch', async () => { + const { imapSimple, mockImap } = createImap(); + + const fetchEmitter = new EventEmitter(); + const mockMessages = [{ uid: 1 }, { uid: 2 }, { uid: 3 }]; + vi.mocked(mockImap.search).mockImplementation((_criteria, onResult) => + onResult( + null as unknown as Error, + mockMessages.map((m) => m.uid), + ), + ); + mockImap.fetch = vi.fn(() => fetchEmitter); + + const searchPromise = imapSimple.search(['UNSEEN', ['FROM', 'test@n8n.io']], { + bodies: ['BODY'], + }); + expect(mockImap.search).toHaveBeenCalledWith( + ['UNSEEN', ['FROM', 'test@n8n.io']], + expect.any(Function), + ); + + for (const message of mockMessages) { + const messageEmitter = new EventEmitter(); + const body = 'body' + message.uid; + const bodyStream = Readable.from(body); + fetchEmitter.emit('message', messageEmitter, message.uid); + messageEmitter.emit('body', bodyStream, { which: 'TEXT', size: Buffer.byteLength(body) }); + messageEmitter.emit('attributes', { uid: message.uid }); + await new Promise((resolve) => { + bodyStream.on('end', resolve); + }); + messageEmitter.emit('end'); + } + + fetchEmitter.emit('end'); + + const messages = await searchPromise; + + expect(messages).toEqual([ + { + attributes: { uid: 1 }, + parts: [{ body: 'body1', size: 5, which: 'TEXT' }], + seqNo: 1, + }, + { + attributes: { uid: 2 }, + parts: [{ body: 'body2', size: 5, which: 'TEXT' }], + seqNo: 2, + }, + { + attributes: { uid: 3 }, + parts: [{ body: 'body3', size: 5, which: 'TEXT' }], + seqNo: 3, + }, + ]); + }); + }); + + describe('getPartData', () => { + it('should return decoded part data', async () => { + const { imapSimple, mockImap } = createImap(); + + const fetchEmitter = new EventEmitter(); + mockImap.fetch = vi.fn(() => fetchEmitter); + + const message = { attributes: { uid: 123 } }; + const part = { partID: '1.2', encoding: 'BASE64' }; + + const partDataPromise = imapSimple.getPartData(mock(message), mock(part)); + + const body = 'encoded-body'; + const messageEmitter = new EventEmitter(); + const bodyStream = Readable.from(body); + + fetchEmitter.emit('message', messageEmitter); + + messageEmitter.emit('body', bodyStream, { + which: part.partID, + size: Buffer.byteLength(body), + }); + messageEmitter.emit('attributes', {}); + await new Promise((resolve) => bodyStream.on('end', resolve)); + messageEmitter.emit('end'); + + fetchEmitter.emit('end'); + + const result = await partDataPromise; + expect(PartData.fromData).toHaveBeenCalledWith('encoded-body', 'BASE64'); + expect(result).toBe('decoded'); + }); + }); + + describe('openBox', () => { + it('should open the mailbox', async () => { + const { imapSimple, mockImap } = createImap(); + const box = mock({ name: 'INBOX' }); + vi.mocked(mockImap.openBox).mockImplementation((_boxName, onOpen) => + onOpen(null as unknown as Error, box), + ); + await expect(imapSimple.openBox('INBOX')).resolves.toEqual(box); + }); + + it('should reject on error', async () => { + const { imapSimple, mockImap } = createImap(); + vi.mocked(mockImap.openBox).mockImplementation((_boxName, onOpen) => + onOpen(new Error('nope')), + ); + await expect(imapSimple.openBox('INBOX')).rejects.toThrow('nope'); + }); + }); + + describe('closeBox', () => { + it('should close the mailbox with default autoExpunge=true', async () => { + const { imapSimple, mockImap } = createImap(); + vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) => + onClose(null as unknown as Error), + ); + await expect(imapSimple.closeBox()).resolves.toBeUndefined(); + expect(mockImap.closeBox).toHaveBeenCalledWith(true, expect.any(Function)); + }); + + it('should close the mailbox with autoExpunge=false', async () => { + const { imapSimple, mockImap } = createImap(); + vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) => + onClose(null as unknown as Error), + ); + await expect(imapSimple.closeBox(false)).resolves.toBeUndefined(); + expect(mockImap.closeBox).toHaveBeenCalledWith(false, expect.any(Function)); + }); + + it('should reject on error', async () => { + const { imapSimple, mockImap } = createImap(); + vi.mocked(mockImap.closeBox).mockImplementation((_expunge, onClose) => + onClose(new Error('fail')), + ); + await expect(imapSimple.closeBox()).rejects.toThrow('fail'); + }); + }); + + describe('addFlags', () => { + it('should add flags to messages and resolve', async () => { + const { imapSimple, mockImap } = createImap(); + vi.mocked(mockImap.addFlags).mockImplementation((_uids, _flags, onAdd) => + onAdd(null as unknown as Error), + ); + + await expect(imapSimple.addFlags([1, 2], ['\\Seen'])).resolves.toBeUndefined(); + expect(mockImap.addFlags).toHaveBeenCalledWith([1, 2], ['\\Seen'], expect.any(Function)); + }); + + it('should reject on error', async () => { + const { imapSimple, mockImap } = createImap(); + vi.mocked(mockImap.addFlags).mockImplementation((_uids, _flags, onAdd) => + onAdd(new Error('add flags failed')), + ); + + await expect(imapSimple.addFlags([1], '\\Seen')).rejects.toThrow('add flags failed'); + }); + }); + + describe('getBoxes', () => { + it('should resolve with list of mailboxes', async () => { + const { imapSimple, mockImap } = createImap(); + // eslint-disable-next-line @typescript-eslint/naming-convention + const boxes = mock({ INBOX: {}, Archive: {} }); + + vi.mocked(mockImap.getBoxes).mockImplementation((onBoxes) => + onBoxes(null as unknown as Error, boxes), + ); + + await expect(imapSimple.getBoxes()).resolves.toEqual(boxes); + expect(mockImap.getBoxes).toHaveBeenCalledWith(expect.any(Function)); + }); + + it('should reject on error', async () => { + const { imapSimple, mockImap } = createImap(); + + vi.mocked(mockImap.getBoxes).mockImplementation((onBoxes) => + onBoxes(new Error('getBoxes failed')), + ); + + await expect(imapSimple.getBoxes()).rejects.toThrow('getBoxes failed'); + }); + }); +}); diff --git a/packages/@n8n/imap/src/imap-simple.ts b/packages/@n8n/imap/src/imap-simple.ts index 78918b0841..93bb9e052a 100644 --- a/packages/@n8n/imap/src/imap-simple.ts +++ b/packages/@n8n/imap/src/imap-simple.ts @@ -4,7 +4,7 @@ import { type ImapMessage } from 'imap'; import { getMessage } from './helpers/get-message'; import { PartData } from './part-data'; -import type { Message, MessagePart } from './types'; +import type { Message, MessagePart, SearchCriteria } from './types'; const IMAP_EVENTS = ['alert', 'mail', 'expunge', 'uidvalidity', 'update', 'close', 'end'] as const; @@ -63,10 +63,11 @@ export class ImapSimple extends EventEmitter { */ async search( /** Criteria to use to search. Passed to node-imap's .search() 1:1 */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - searchCriteria: any[], + searchCriteria: SearchCriteria[], /** Criteria to use to fetch the search results. Passed to node-imap's .fetch() 1:1 */ fetchOptions: Imap.FetchOptions, + /** Optional limit to restrict the number of messages fetched */ + limit?: number, ) { return await new Promise((resolve, reject) => { this.imap.search(searchCriteria, (e, uids) => { @@ -80,17 +81,23 @@ export class ImapSimple extends EventEmitter { return; } - const fetch = this.imap.fetch(uids, fetchOptions); + // If limit is specified, take only the first N UIDs + let uidsToFetch = uids; + if (limit && limit > 0 && uids.length > limit) { + uidsToFetch = uids.slice(0, limit); + } + + const fetch = this.imap.fetch(uidsToFetch, fetchOptions); let messagesRetrieved = 0; const messages: Message[] = []; const fetchOnMessage = async (message: Imap.ImapMessage, seqNo: number) => { const msg: Message = await getMessage(message); msg.seqNo = seqNo; - messages[seqNo] = msg; + messages.push(msg); messagesRetrieved++; - if (messagesRetrieved === uids.length) { + if (messagesRetrieved === uidsToFetch.length) { resolve(messages.filter((m) => !!m)); } }; diff --git a/packages/@n8n/imap/test/part-data.test.ts b/packages/@n8n/imap/src/part-data.test.ts similarity index 100% rename from packages/@n8n/imap/test/part-data.test.ts rename to packages/@n8n/imap/src/part-data.test.ts diff --git a/packages/@n8n/imap/src/types.ts b/packages/@n8n/imap/src/types.ts index 6fa533dd9d..de0964dea7 100644 --- a/packages/@n8n/imap/src/types.ts +++ b/packages/@n8n/imap/src/types.ts @@ -39,3 +39,5 @@ export interface Message { parts: MessageBodyPart[]; seqNo?: number; } + +export type SearchCriteria = string | [string, string]; diff --git a/packages/@n8n/imap/tsconfig.build.json b/packages/@n8n/imap/tsconfig.build.json index 0794319028..8f1e58acad 100644 --- a/packages/@n8n/imap/tsconfig.build.json +++ b/packages/@n8n/imap/tsconfig.build.json @@ -7,5 +7,5 @@ "tsBuildInfoFile": "dist/build.tsbuildinfo" }, "include": ["src/**/*.ts"], - "exclude": ["test/**"] + "exclude": ["src/**/*.test.ts"] } diff --git a/packages/@n8n/imap/tsconfig.json b/packages/@n8n/imap/tsconfig.json index 94d5721691..ce0179d943 100644 --- a/packages/@n8n/imap/tsconfig.json +++ b/packages/@n8n/imap/tsconfig.json @@ -2,9 +2,9 @@ "extends": "@n8n/typescript-config/tsconfig.common.json", "compilerOptions": { "rootDir": ".", - "types": ["node", "jest"], + "types": ["node", "vite/client", "vitest/globals"], "baseUrl": "src", "tsBuildInfoFile": "dist/typecheck.tsbuildinfo" }, - "include": ["src/**/*.ts", "test/**/*.ts"] + "include": ["src/**/*.ts"] } diff --git a/packages/@n8n/imap/vite.config.ts b/packages/@n8n/imap/vite.config.ts new file mode 100644 index 0000000000..0d1ecf4c75 --- /dev/null +++ b/packages/@n8n/imap/vite.config.ts @@ -0,0 +1,3 @@ +import { vitestConfig } from '@n8n/vitest-config/node'; + +export default vitestConfig; diff --git a/packages/@n8n/utils/vite.config.ts b/packages/@n8n/utils/vite.config.ts index 784f3fb497..e0d123a7b3 100644 --- a/packages/@n8n/utils/vite.config.ts +++ b/packages/@n8n/utils/vite.config.ts @@ -1,4 +1,3 @@ -import { defineConfig, mergeConfig } from 'vite'; import { vitestConfig } from '@n8n/vitest-config/frontend'; -export default mergeConfig(defineConfig({}), vitestConfig); +export default vitestConfig; diff --git a/packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.ts b/packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.ts index e8eabf9282..4329cccdff 100644 --- a/packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.ts +++ b/packages/nodes-base/nodes/EmailReadImap/EmailReadImap.node.ts @@ -12,12 +12,13 @@ export class EmailReadImap extends VersionedNodeType { icon: 'fa:inbox', group: ['trigger'], description: 'Triggers the workflow when a new email is received', - defaultVersion: 2, + defaultVersion: 2.1, }; const nodeVersions: IVersionedNodeType['nodeVersions'] = { 1: new EmailReadImapV1(baseDescription), 2: new EmailReadImapV2(baseDescription), + 2.1: new EmailReadImapV2(baseDescription), }; super(nodeVersions, baseDescription); diff --git a/packages/nodes-base/nodes/EmailReadImap/test/v2/utils.test.ts b/packages/nodes-base/nodes/EmailReadImap/test/v2/utils.test.ts index afc2fe048f..1f14ae77d4 100644 --- a/packages/nodes-base/nodes/EmailReadImap/test/v2/utils.test.ts +++ b/packages/nodes-base/nodes/EmailReadImap/test/v2/utils.test.ts @@ -1,7 +1,7 @@ import { type ImapSimple } from '@n8n/imap'; -import { mock } from 'jest-mock-extended'; +import { mock, mockDeep } from 'jest-mock-extended'; import { returnJsonArray } from 'n8n-core'; -import { type IDataObject, type ITriggerFunctions } from 'n8n-workflow'; +import type { INode, ITriggerFunctions } from 'n8n-workflow'; import { getNewEmails } from '../../v2/utils'; @@ -9,7 +9,7 @@ describe('Test IMap V2 utils', () => { afterEach(() => jest.resetAllMocks()); describe('getNewEmails', () => { - const triggerFunctions = mock({ + const triggerFunctions = mockDeep({ helpers: { returnJsonArray }, }); @@ -73,27 +73,27 @@ describe('Test IMap V2 utils', () => { ]; expectedResults.forEach(async (expectedResult) => { - // use new staticData for each iteration - const staticData: IDataObject = {}; - + triggerFunctions.getNode.mockReturnValue(mock({ typeVersion: 2.1 })); triggerFunctions.getNodeParameter .calledWith('format') .mockReturnValue(expectedResult.format); triggerFunctions.getNodeParameter .calledWith('dataPropertyAttachmentsPrefixName') .mockReturnValue('resolved'); + triggerFunctions.getWorkflowStaticData.mockReturnValue({}); - const result = getNewEmails.call( - triggerFunctions, + const onEmailBatch = jest.fn(); + await getNewEmails.call(triggerFunctions, { imapConnection, - [], - staticData, - '', + searchCriteria: [], + postProcessAction: '', getText, getAttachment, - ); + onEmailBatch, + }); - await expect(result).resolves.toEqual([expectedResult.expected]); + expect(onEmailBatch).toHaveBeenCalledTimes(1); + expect(onEmailBatch).toHaveBeenCalledWith([expectedResult.expected]); }); }); }); diff --git a/packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts b/packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts index 8c9fcf9051..76cc1e34f0 100644 --- a/packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts +++ b/packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts @@ -1,4 +1,4 @@ -import type { ImapSimple, ImapSimpleOptions, Message } from '@n8n/imap'; +import type { ImapSimple, ImapSimpleOptions, Message, SearchCriteria } from '@n8n/imap'; import { connect as imapConnect, getParts } from '@n8n/imap'; import find from 'lodash/find'; import isEmpty from 'lodash/isEmpty'; @@ -343,7 +343,7 @@ export class EmailReadImapV1 implements INodeType { // Returns all the new unseen messages const getNewEmails = async ( imapConnection: ImapSimple, - searchCriteria: Array, + searchCriteria: SearchCriteria[], ): Promise => { const format = this.getNodeParameter('format', 0) as string; @@ -508,7 +508,7 @@ export class EmailReadImapV1 implements INodeType { const returnedPromise = this.helpers.createDeferredPromise(); const establishConnection = async (): Promise => { - let searchCriteria = ['UNSEEN'] as Array; + let searchCriteria: SearchCriteria[] = ['UNSEEN']; if (options.customEmailConfig !== undefined) { try { searchCriteria = JSON.parse(options.customEmailConfig as string); diff --git a/packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts b/packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts index 643bd37ec0..5b5f9dcba0 100644 --- a/packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts +++ b/packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts @@ -1,6 +1,15 @@ -import type { ImapSimple, ImapSimpleOptions, Message, MessagePart } from '@n8n/imap'; +import type { ICredentialsDataImap } from '@credentials/Imap.credentials'; +import { isCredentialsDataImap } from '@credentials/Imap.credentials'; +import type { + ImapSimple, + ImapSimpleOptions, + Message, + MessagePart, + SearchCriteria, +} from '@n8n/imap'; import { connect as imapConnect } from '@n8n/imap'; import isEmpty from 'lodash/isEmpty'; +import { DateTime } from 'luxon'; import type { ITriggerFunctions, IBinaryData, @@ -13,13 +22,11 @@ import type { INodeTypeDescription, ITriggerResponse, JsonObject, + INodeExecutionData, } from 'n8n-workflow'; import { NodeConnectionTypes, NodeOperationError, TriggerCloseError } from 'n8n-workflow'; import rfc2047 from 'rfc2047'; -import type { ICredentialsDataImap } from '@credentials/Imap.credentials'; -import { isCredentialsDataImap } from '@credentials/Imap.credentials'; - import { getNewEmails } from './utils'; const versionDescription: INodeTypeDescription = { @@ -28,7 +35,7 @@ const versionDescription: INodeTypeDescription = { icon: 'fa:inbox', iconColor: 'green', group: ['trigger'], - version: 2, + version: [2, 2.1], description: 'Triggers the workflow when a new email is received', eventTriggerDescription: 'Waiting for you to receive an email', defaults: { @@ -248,6 +255,7 @@ export class EmailReadImapV2 implements INodeType { const mailbox = this.getNodeParameter('mailbox') as string; const postProcessAction = this.getNodeParameter('postProcessAction') as string; const options = this.getNodeParameter('options', {}) as IDataObject; + const activatedAt = DateTime.now(); const staticData = this.getWorkflowStaticData('node'); this.logger.debug('Loaded static data for node "EmailReadImap"', { staticData }); @@ -333,12 +341,10 @@ export class EmailReadImapV2 implements INodeType { const returnedPromise = this.helpers.createDeferredPromise(); const establishConnection = async (): Promise => { - let searchCriteria = ['UNSEEN'] as Array; + let searchCriteria: SearchCriteria[] = ['UNSEEN']; if (options.customEmailConfig !== undefined) { try { - searchCriteria = JSON.parse(options.customEmailConfig as string) as Array< - string | string[] - >; + searchCriteria = JSON.parse(options.customEmailConfig as string) as SearchCriteria[]; } catch (error) { throw new NodeOperationError(this.getNode(), 'Custom email config is not valid JSON.'); } @@ -353,10 +359,21 @@ export class EmailReadImapV2 implements INodeType { tls: credentials.secure, authTimeout: 20000, }, - onMail: async () => { + onMail: async (numEmails) => { + this.logger.debug('New emails received in node "EmailReadImap"', { + numEmails, + }); + if (connection) { + /** + * Only process new emails: + * - If we've seen emails before (lastMessageUid is set), fetch messages higher UID. + * - Otherwise, fetch emails received since the workflow activation date. + * + * Note: IMAP 'SINCE' only filters by date (not time), + * so it may include emails from earlier on the activation day. + */ if (staticData.lastMessageUid !== undefined) { - searchCriteria.push(['UID', `${staticData.lastMessageUid as number}:*`]); /** * A short explanation about UIDs and how they work * can be found here: https://dev.to/kehers/imap-new-messages-since-last-check-44gm @@ -369,24 +386,28 @@ export class EmailReadImapV2 implements INodeType { * - You can check if UIDs changed in the above example * by checking UIDValidity. */ - this.logger.debug('Querying for new messages on node "EmailReadImap"', { - searchCriteria, - }); + searchCriteria.push(['UID', `${staticData.lastMessageUid as number}:*`]); + } else { + searchCriteria.push(['SINCE', activatedAt.toFormat('dd-LLL-yyyy')]); } + this.logger.debug('Querying for new messages on node "EmailReadImap"', { + searchCriteria, + }); + try { - const returnData = await getNewEmails.call( - this, - connection, + await getNewEmails.call(this, { + imapConnection: connection, searchCriteria, - staticData, postProcessAction, getText, getAttachment, - ); - if (returnData.length) { - this.emit([returnData]); - } + onEmailBatch: async (returnData: INodeExecutionData[]) => { + if (returnData.length) { + this.emit([returnData]); + } + }, + }); } catch (error) { this.logger.error('Email Read Imap node encountered an error fetching new emails', { error: error as Error, @@ -399,7 +420,7 @@ export class EmailReadImapV2 implements INodeType { } } }, - onUpdate: async (seqNo: number, info) => { + onUpdate: (seqNo: number, info) => { this.logger.debug(`Email Read Imap:update ${seqNo}`, info); }, }; @@ -420,8 +441,8 @@ export class EmailReadImapV2 implements INodeType { // Connect to the IMAP server and open the mailbox // that we get informed whenever a new email arrives - return await imapConnect(config).then(async (conn) => { - conn.on('close', async (_hadError: boolean) => { + return await imapConnect(config).then((conn) => { + conn.on('close', (_hadError: boolean) => { if (isCurrentlyReconnecting) { this.logger.debug('Email Read Imap: Connected closed for forced reconnecting'); } else if (closeFunctionWasCalled) { @@ -431,7 +452,7 @@ export class EmailReadImapV2 implements INodeType { this.emitError(new Error('Imap connection closed unexpectedly')); } }); - conn.on('error', async (error) => { + conn.on('error', (error) => { const errorCode = ((error as JsonObject).code as string).toUpperCase(); this.logger.debug(`IMAP connection experienced an error: (${errorCode})`, { error: error as Error, diff --git a/packages/nodes-base/nodes/EmailReadImap/v2/utils.ts b/packages/nodes-base/nodes/EmailReadImap/v2/utils.ts index 92edb9e1da..1d79f31d19 100644 --- a/packages/nodes-base/nodes/EmailReadImap/v2/utils.ts +++ b/packages/nodes-base/nodes/EmailReadImap/v2/utils.ts @@ -1,4 +1,10 @@ -import { getParts, type ImapSimple, type Message, type MessagePart } from '@n8n/imap'; +import { + getParts, + type ImapSimple, + type Message, + type MessagePart, + type SearchCriteria, +} from '@n8n/imap'; import find from 'lodash/find'; import { simpleParser, type Source as ParserSource } from 'mailparser'; import { @@ -46,19 +52,30 @@ async function parseRawEmail( } as INodeExecutionData; } +const EMAIL_BATCH_SIZE = 20; + export async function getNewEmails( this: ITriggerFunctions, - imapConnection: ImapSimple, - searchCriteria: Array, - staticData: IDataObject, - postProcessAction: string, - getText: (parts: MessagePart[], message: Message, subtype: string) => Promise, - getAttachment: ( - imapConnection: ImapSimple, - parts: MessagePart[], - message: Message, - ) => Promise, -): Promise { + { + getAttachment, + getText, + onEmailBatch, + imapConnection, + postProcessAction, + searchCriteria, + }: { + imapConnection: ImapSimple; + searchCriteria: SearchCriteria[]; + postProcessAction: string; + getText: (parts: MessagePart[], message: Message, subtype: string) => Promise; + getAttachment: ( + imapConnection: ImapSimple, + parts: MessagePart[], + message: Message, + ) => Promise; + onEmailBatch: (data: INodeExecutionData[]) => Promise; + }, +) { const format = this.getNodeParameter('format', 0) as string; let fetchOptions = {}; @@ -77,150 +94,167 @@ export async function getNewEmails( }; } - const results = await imapConnection.search(searchCriteria, fetchOptions); + let results: Message[] = []; + let maxUid = 0; - const newEmails: INodeExecutionData[] = []; - let newEmail: INodeExecutionData; - let attachments: IBinaryData[]; - let propertyName: string; + const staticData = this.getWorkflowStaticData('node'); + const limit = this.getNode().typeVersion >= 2.1 ? EMAIL_BATCH_SIZE : undefined; - // All properties get by default moved to metadata except the ones - // which are defined here which get set on the top level. - const topLevelProperties = ['cc', 'date', 'from', 'subject', 'to']; + do { + if (maxUid) { + searchCriteria = searchCriteria.filter((criteria: SearchCriteria) => { + if (Array.isArray(criteria)) { + return !['UID', 'SINCE'].includes(criteria[0]); + } + return true; + }); - if (format === 'resolved') { - const dataPropertyAttachmentsPrefixName = this.getNodeParameter( - 'dataPropertyAttachmentsPrefixName', - ) as string; - - for (const message of results) { - if ( - staticData.lastMessageUid !== undefined && - message.attributes.uid <= (staticData.lastMessageUid as number) - ) { - continue; - } - if ( - staticData.lastMessageUid === undefined || - (staticData.lastMessageUid as number) < message.attributes.uid - ) { - staticData.lastMessageUid = message.attributes.uid; - } - const part = find(message.parts, { which: '' }); - - if (part === undefined) { - throw new NodeOperationError(this.getNode(), 'Email part could not be parsed.'); - } - const parsedEmail = await parseRawEmail.call( - this, - part.body as Buffer, - dataPropertyAttachmentsPrefixName, - ); - - parsedEmail.json.attributes = { - uid: message.attributes.uid, - }; - - newEmails.push(parsedEmail); + searchCriteria.push(['UID', `${maxUid}:*`]); } - } else if (format === 'simple') { - const downloadAttachments = this.getNodeParameter('downloadAttachments') as boolean; + results = await imapConnection.search(searchCriteria, fetchOptions, limit); - let dataPropertyAttachmentsPrefixName = ''; - if (downloadAttachments) { - dataPropertyAttachmentsPrefixName = this.getNodeParameter( + this.logger.debug(`Process ${results.length} new emails in node "EmailReadImap"`); + + const newEmails: INodeExecutionData[] = []; + let newEmail: INodeExecutionData; + let attachments: IBinaryData[]; + let propertyName: string; + + // All properties get by default moved to metadata except the ones + // which are defined here which get set on the top level. + const topLevelProperties = ['cc', 'date', 'from', 'subject', 'to']; + + if (format === 'resolved') { + const dataPropertyAttachmentsPrefixName = this.getNodeParameter( 'dataPropertyAttachmentsPrefixName', ) as string; - } - for (const message of results) { - if ( - staticData.lastMessageUid !== undefined && - message.attributes.uid <= (staticData.lastMessageUid as number) - ) { - continue; - } - if ( - staticData.lastMessageUid === undefined || - (staticData.lastMessageUid as number) < message.attributes.uid - ) { - staticData.lastMessageUid = message.attributes.uid; - } - const parts = getParts(message.attributes.struct as IDataObject[]); - - newEmail = { - json: { - textHtml: await getText(parts, message, 'html'), - textPlain: await getText(parts, message, 'plain'), - metadata: {} as IDataObject, - attributes: { - uid: message.attributes.uid, - } as IDataObject, - }, - }; - - const messageHeader = message.parts.filter((part) => part.which === 'HEADER'); - - const messageBody = messageHeader[0].body as Record; - for (propertyName of Object.keys(messageBody)) { - if (messageBody[propertyName].length) { - if (topLevelProperties.includes(propertyName)) { - newEmail.json[propertyName] = messageBody[propertyName][0]; - } else { - (newEmail.json.metadata as IDataObject)[propertyName] = messageBody[propertyName][0]; - } + for (const message of results) { + const lastMessageUid = this.getWorkflowStaticData('node').lastMessageUid as number; + if (lastMessageUid !== undefined && message.attributes.uid <= lastMessageUid) { + continue; } - } + // Track the maximum UID to update staticData later + if (message.attributes.uid > maxUid) { + maxUid = message.attributes.uid; + } + const part = find(message.parts, { which: '' }); + + if (part === undefined) { + throw new NodeOperationError(this.getNode(), 'Email part could not be parsed.'); + } + const parsedEmail = await parseRawEmail.call( + this, + part.body as Buffer, + dataPropertyAttachmentsPrefixName, + ); + + parsedEmail.json.attributes = { + uid: message.attributes.uid, + }; + + newEmails.push(parsedEmail); + } + } else if (format === 'simple') { + const downloadAttachments = this.getNodeParameter('downloadAttachments') as boolean; + + let dataPropertyAttachmentsPrefixName = ''; if (downloadAttachments) { - // Get attachments and add them if any get found - attachments = await getAttachment(imapConnection, parts, message); - if (attachments.length) { - newEmail.binary = {}; - for (let i = 0; i < attachments.length; i++) { - newEmail.binary[`${dataPropertyAttachmentsPrefixName}${i}`] = attachments[i]; + dataPropertyAttachmentsPrefixName = this.getNodeParameter( + 'dataPropertyAttachmentsPrefixName', + ) as string; + } + + for (const message of results) { + const lastMessageUid = this.getWorkflowStaticData('node').lastMessageUid as number; + if (lastMessageUid !== undefined && message.attributes.uid <= lastMessageUid) { + continue; + } + + // Track the maximum UID to update staticData later + if (message.attributes.uid > maxUid) { + maxUid = message.attributes.uid; + } + const parts = getParts(message.attributes.struct as IDataObject[]); + + newEmail = { + json: { + textHtml: await getText(parts, message, 'html'), + textPlain: await getText(parts, message, 'plain'), + metadata: {} as IDataObject, + attributes: { + uid: message.attributes.uid, + } as IDataObject, + }, + }; + + const messageHeader = message.parts.filter((part) => part.which === 'HEADER'); + + const messageBody = messageHeader[0].body as Record; + for (propertyName of Object.keys(messageBody)) { + if (messageBody[propertyName].length) { + if (topLevelProperties.includes(propertyName)) { + newEmail.json[propertyName] = messageBody[propertyName][0]; + } else { + (newEmail.json.metadata as IDataObject)[propertyName] = messageBody[propertyName][0]; + } } } - } - newEmails.push(newEmail); + if (downloadAttachments) { + // Get attachments and add them if any get found + attachments = await getAttachment(imapConnection, parts, message); + if (attachments.length) { + newEmail.binary = {}; + for (let i = 0; i < attachments.length; i++) { + newEmail.binary[`${dataPropertyAttachmentsPrefixName}${i}`] = attachments[i]; + } + } + } + + newEmails.push(newEmail); + } + } else if (format === 'raw') { + for (const message of results) { + const lastMessageUid = this.getWorkflowStaticData('node').lastMessageUid as number; + if (lastMessageUid !== undefined && message.attributes.uid <= lastMessageUid) { + continue; + } + + // Track the maximum UID to update staticData later + if (message.attributes.uid > maxUid) { + maxUid = message.attributes.uid; + } + const part = find(message.parts, { which: 'TEXT' }); + + if (part === undefined) { + throw new NodeOperationError(this.getNode(), 'Email part could not be parsed.'); + } + // Return base64 string + newEmail = { + json: { + raw: part.body as string, + }, + }; + + newEmails.push(newEmail); + } } - } else if (format === 'raw') { - for (const message of results) { - if ( - staticData.lastMessageUid !== undefined && - message.attributes.uid <= (staticData.lastMessageUid as number) - ) { - continue; - } - if ( - staticData.lastMessageUid === undefined || - (staticData.lastMessageUid as number) < message.attributes.uid - ) { - staticData.lastMessageUid = message.attributes.uid; - } - const part = find(message.parts, { which: 'TEXT' }); - if (part === undefined) { - throw new NodeOperationError(this.getNode(), 'Email part could not be parsed.'); + // only mark messages as seen once processing has finished + if (postProcessAction === 'read') { + const uidList = results.map((e) => e.attributes.uid); + if (uidList.length > 0) { + await imapConnection.addFlags(uidList, '\\SEEN'); } - // Return base64 string - newEmail = { - json: { - raw: part.body as string, - }, - }; - - newEmails.push(newEmail); } + + await onEmailBatch(newEmails); + } while (results.length >= EMAIL_BATCH_SIZE); + + // Update lastMessageUid after processing all messages + if (maxUid > ((staticData.lastMessageUid as number) ?? 0)) { + this.getWorkflowStaticData('node').lastMessageUid = maxUid; } - - // only mark messages as seen once processing has finished - if (postProcessAction === 'read') { - const uidList = results.map((e) => e.attributes.uid); - if (uidList.length > 0) { - await imapConnection.addFlags(uidList, '\\SEEN'); - } - } - return newEmails; } diff --git a/packages/workflow/vitest.config.ts b/packages/workflow/vitest.config.ts index 5853b4209d..7042490e9a 100644 --- a/packages/workflow/vitest.config.ts +++ b/packages/workflow/vitest.config.ts @@ -1,8 +1,3 @@ -/* eslint-disable import-x/no-default-export */ -export default async () => { - const { createVitestConfig } = await import('@n8n/vitest-config/node'); +import { createVitestConfig } from '@n8n/vitest-config/node'; - return createVitestConfig({ - include: ['test/**/*.test.ts'], - }); -}; +export default createVitestConfig({ include: ['test/**/*.test.ts'] }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 72c487ce72..9a9283007b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -830,6 +830,9 @@ importers: '@n8n/typescript-config': specifier: workspace:* version: link:../typescript-config + '@n8n/vitest-config': + specifier: workspace:* + version: link:../vitest-config '@types/imap': specifier: ^0.8.40 version: 0.8.40 @@ -842,6 +845,9 @@ importers: '@types/uuencode': specifier: ^0.0.3 version: 0.0.3(patch_hash=083a73709a54db57b092d986b43d27ddda3cb8008f9510e98bc9e6da0e1cbb62) + vitest-mock-extended: + specifier: 'catalog:' + version: 3.1.0(typescript@5.8.3)(vitest@3.1.3(@types/debug@4.1.12)(@types/node@20.19.1)(jiti@1.21.7)(jsdom@23.0.1)(sass@1.64.1)(terser@5.16.1)(tsx@4.19.3)) packages/@n8n/json-schema-to-zod: devDependencies: @@ -6865,8 +6871,8 @@ packages: '@types/docker-modem@3.0.6': resolution: {integrity: sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==} - '@types/dockerode@3.3.41': - resolution: {integrity: sha512-5kOi6bcnEjqfJ68ZNV/bBvSMLNIucc0XbRmBO4hg5OoFCoP99eSRcbMysjkzV7ZxQEmmc/zMnv4A7odwuKFzDA==} + '@types/dockerode@3.3.42': + resolution: {integrity: sha512-U1jqHMShibMEWHdxYhj3rCMNCiLx5f35i4e3CEUuW+JSSszc/tVqc6WCAPdhwBymG5R/vgbcceagK0St7Cq6Eg==} '@types/eslint@9.6.1': resolution: {integrity: sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==} @@ -11521,10 +11527,6 @@ packages: resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} engines: {node: '>=10'} - lilconfig@3.1.2: - resolution: {integrity: sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==} - engines: {node: '>=14'} - lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -18986,7 +18988,7 @@ snapshots: '@n8n/localtunnel@3.0.0': dependencies: axios: 1.10.0(debug@4.3.6) - debug: 4.3.6(supports-color@8.1.1) + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -19172,13 +19174,13 @@ snapshots: ansis: 3.2.0 clean-stack: 3.0.1 cli-spinners: 2.9.2 - debug: 4.3.6(supports-color@8.1.1) + debug: 4.4.1(supports-color@8.1.1) ejs: 3.1.10 get-package-type: 0.1.0 globby: 11.1.0 indent-string: 4.0.0 is-wsl: 2.2.0 - lilconfig: 3.1.2 + lilconfig: 3.1.3 minimatch: 9.0.5 string-width: 4.2.3 supports-color: 8.1.1 @@ -20858,7 +20860,7 @@ snapshots: '@types/node': 20.19.1 '@types/ssh2': 1.11.6 - '@types/dockerode@3.3.41': + '@types/dockerode@3.3.42': dependencies: '@types/docker-modem': 3.0.6 '@types/node': 20.19.1 @@ -23489,11 +23491,9 @@ snapshots: dependencies: ms: 2.1.2 - debug@4.3.6(supports-color@8.1.1): + debug@4.3.6: dependencies: ms: 2.1.2 - optionalDependencies: - supports-color: 8.1.1 debug@4.4.0: dependencies: @@ -24690,7 +24690,7 @@ snapshots: follow-redirects@1.15.9(debug@4.3.6): optionalDependencies: - debug: 4.3.6(supports-color@8.1.1) + debug: 4.3.6 follow-redirects@1.15.9(debug@4.4.0): optionalDependencies: @@ -26736,8 +26736,6 @@ snapshots: lilconfig@2.1.0: {} - lilconfig@3.1.2: {} - lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} @@ -28397,7 +28395,7 @@ snapshots: postcss-load-config@4.0.2(postcss@8.4.49)(ts-node@10.9.2(@types/node@20.19.1)(typescript@5.8.3)): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 yaml: 2.3.4 optionalDependencies: postcss: 8.4.49 @@ -30126,7 +30124,7 @@ snapshots: testcontainers@11.0.3: dependencies: '@balena/dockerignore': 1.0.2 - '@types/dockerode': 3.3.41 + '@types/dockerode': 3.3.42 archiver: 7.0.1 async-lock: 1.4.1 byline: 5.0.0