fix(Postgres Node): Re-use connection pool across executions (#12346)

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>

packages/nodes-base/nodes/Postgres/v2/actions/router.ts

@@ -35,25 +35,21 @@ export async function router(this: IExecuteFunctions): Promise<INodeExecutionDat
 		operation,
 	} as PostgresType;
 
-	try {
-		switch (postgresNodeData.resource) {
-			case 'database':
-				returnData = await database[postgresNodeData.operation].execute.call(
-					this,
-					runQueries,
-					items,
-					options,
-					db,
-				);
-				break;
-			default:
-				throw new NodeOperationError(
-					this.getNode(),
-					`The operation "${operation}" is not supported!`,
-				);
-		}
-	} finally {
-		if (!db.$pool.ending) await db.$pool.end();
+	switch (postgresNodeData.resource) {
+		case 'database':
+			returnData = await database[postgresNodeData.operation].execute.call(
+				this,
+				runQueries,
+				items,
+				options,
+				db,
+			);
+			break;
+		default:
+			throw new NodeOperationError(
+				this.getNode(),
+				`The operation "${operation}" is not supported!`,
+			);
 	}
 
 	if (operation === 'select' && items.length > 1 && !node.executeOnce) {

packages/nodes-base/nodes/Postgres/v2/methods/listSearch.ts

@@ -9,18 +9,14 @@ export async function schemaSearch(this: ILoadOptionsFunctions): Promise<INodeLi
 
 	const { db } = await configurePostgres.call(this, credentials, options);
 
-	try {
-		const response = await db.any('SELECT schema_name FROM information_schema.schemata');
+	const response = await db.any('SELECT schema_name FROM information_schema.schemata');
 
-		return {
-			results: response.map((schema) => ({
-				name: schema.schema_name as string,
-				value: schema.schema_name as string,
-			})),
-		};
-	} finally {
-		if (!db.$pool.ending) await db.$pool.end();
-	}
+	return {
+		results: response.map((schema) => ({
+			name: schema.schema_name as string,
+			value: schema.schema_name as string,
+		})),
+	};
 }
 export async function tableSearch(this: ILoadOptionsFunctions): Promise<INodeListSearchResult> {
 	const credentials = await this.getCredentials<PostgresNodeCredentials>('postgres');
@@ -32,19 +28,15 @@ export async function tableSearch(this: ILoadOptionsFunctions): Promise<INodeLis
 		extractValue: true,
 	}) as string;
 
-	try {
-		const response = await db.any(
-			'SELECT table_name FROM information_schema.tables WHERE table_schema=$1',
-			[schema],
-		);
+	const response = await db.any(
+		'SELECT table_name FROM information_schema.tables WHERE table_schema=$1',
+		[schema],
+	);
 
-		return {
-			results: response.map((table) => ({
-				name: table.table_name as string,
-				value: table.table_name as string,
-			})),
-		};
-	} finally {
-		if (!db.$pool.ending) await db.$pool.end();
-	}
+	return {
+		results: response.map((table) => ({
+			name: table.table_name as string,
+			value: table.table_name as string,
+		})),
+	};
 }

packages/nodes-base/nodes/Postgres/v2/methods/loadOptions.ts

@@ -18,17 +18,13 @@ export async function getColumns(this: ILoadOptionsFunctions): Promise<INodeProp
 		extractValue: true,
 	}) as string;
 
-	try {
-		const columns = await getTableSchema(db, schema, table);
+	const columns = await getTableSchema(db, schema, table);
 
-		return columns.map((column) => ({
-			name: column.column_name,
-			value: column.column_name,
-			description: `Type: ${column.data_type.toUpperCase()}, Nullable: ${column.is_nullable}`,
-		}));
-	} finally {
-		if (!db.$pool.ending) await db.$pool.end();
-	}
+	return columns.map((column) => ({
+		name: column.column_name,
+		value: column.column_name,
+		description: `Type: ${column.data_type.toUpperCase()}, Nullable: ${column.is_nullable}`,
+	}));
 }
 
 export async function getColumnsMultiOptions(

packages/nodes-base/nodes/Postgres/v2/methods/resourceMapping.ts

@@ -63,34 +63,30 @@ export async function getMappingColumns(
 		extractValue: true,
 	}) as string;
 
-	try {
-		const columns = await getTableSchema(db, schema, table, { getColumnsForResourceMapper: true });
-		const unique = operation === 'upsert' ? await uniqueColumns(db, table, schema) : [];
-		const enumInfo = await getEnums(db);
-		const fields = await Promise.all(
-			columns.map(async (col) => {
-				const canBeUsedToMatch =
-					operation === 'upsert' ? unique.some((u) => u.attname === col.column_name) : true;
-				const type = mapPostgresType(col.data_type);
-				const options =
-					type === 'options' ? getEnumValues(enumInfo, col.udt_name as string) : undefined;
-				const hasDefault = Boolean(col.column_default);
-				const isGenerated = col.is_generated === 'ALWAYS' || col.identity_generation === 'ALWAYS';
-				const nullable = col.is_nullable === 'YES';
-				return {
-					id: col.column_name,
-					displayName: col.column_name,
-					required: !nullable && !hasDefault && !isGenerated,
-					defaultMatch: (col.column_name === 'id' && canBeUsedToMatch) || false,
-					display: true,
-					type,
-					canBeUsedToMatch,
-					options,
-				};
-			}),
-		);
-		return { fields };
-	} finally {
-		if (!db.$pool.ending) await db.$pool.end();
-	}
+	const columns = await getTableSchema(db, schema, table, { getColumnsForResourceMapper: true });
+	const unique = operation === 'upsert' ? await uniqueColumns(db, table, schema) : [];
+	const enumInfo = await getEnums(db);
+	const fields = await Promise.all(
+		columns.map(async (col) => {
+			const canBeUsedToMatch =
+				operation === 'upsert' ? unique.some((u) => u.attname === col.column_name) : true;
+			const type = mapPostgresType(col.data_type);
+			const options =
+				type === 'options' ? getEnumValues(enumInfo, col.udt_name as string) : undefined;
+			const hasDefault = Boolean(col.column_default);
+			const isGenerated = col.is_generated === 'ALWAYS' || col.identity_generation === 'ALWAYS';
+			const nullable = col.is_nullable === 'YES';
+			return {
+				id: col.column_name,
+				displayName: col.column_name,
+				required: !nullable && !hasDefault && !isGenerated,
+				defaultMatch: (col.column_name === 'id' && canBeUsedToMatch) || false,
+				display: true,
+				type,
+				canBeUsedToMatch,
+				options,
+			};
+		}),
+	);
+	return { fields };
 }

packages/nodes-base/nodes/Postgres/v2/transport/index.ts

@@ -7,6 +7,7 @@ import type {
 import { createServer, type AddressInfo } from 'node:net';
 import pgPromise from 'pg-promise';
 
+import { ConnectionPoolManager } from '@utils/connection-pool-manager';
 import { LOCALHOST } from '@utils/constants';
 import { formatPrivateKey } from '@utils/utilities';
 
@@ -56,120 +57,135 @@ export async function configurePostgres(
 	credentials: PostgresNodeCredentials,
 	options: PostgresNodeOptions = {},
 ): Promise<ConnectionsData> {
-	const pgp = pgPromise({
-		// prevent spam in console "WARNING: Creating a duplicate database object for the same connection."
-		// duplicate connections created when auto loading parameters, they are closed immediately after, but several could be open at the same time
-		noWarnings: true,
-	});
+	const poolManager = ConnectionPoolManager.getInstance();
 
-	if (typeof options.nodeVersion === 'number' && options.nodeVersion >= 2.1) {
-		// Always return dates as ISO strings
-		[pgp.pg.types.builtins.TIMESTAMP, pgp.pg.types.builtins.TIMESTAMPTZ].forEach((type) => {
-			pgp.pg.types.setTypeParser(type, (value: string) => {
-				const parsedDate = new Date(value);
+	const fallBackHandler = async () => {
+		const pgp = pgPromise({
+			// prevent spam in console "WARNING: Creating a duplicate database object for the same connection."
+			// duplicate connections created when auto loading parameters, they are closed immediately after, but several could be open at the same time
+			noWarnings: true,
+		});
 
-				if (isNaN(parsedDate.getTime())) {
-					return value;
+		if (typeof options.nodeVersion === 'number' && options.nodeVersion >= 2.1) {
+			// Always return dates as ISO strings
+			[pgp.pg.types.builtins.TIMESTAMP, pgp.pg.types.builtins.TIMESTAMPTZ].forEach((type) => {
+				pgp.pg.types.setTypeParser(type, (value: string) => {
+					const parsedDate = new Date(value);
+
+					if (isNaN(parsedDate.getTime())) {
+						return value;
+					}
+
+					return parsedDate.toISOString();
+				});
+			});
+		}
+
+		if (options.largeNumbersOutput === 'numbers') {
+			pgp.pg.types.setTypeParser(20, (value: string) => {
+				return parseInt(value, 10);
+			});
+			pgp.pg.types.setTypeParser(1700, (value: string) => {
+				return parseFloat(value);
+			});
+		}
+
+		const dbConfig = getPostgresConfig(credentials, options);
+
+		if (!credentials.sshTunnel) {
+			const db = pgp(dbConfig);
+
+			return { db, pgp };
+		} else {
+			if (credentials.sshAuthenticateWith === 'privateKey' && credentials.privateKey) {
+				credentials.privateKey = formatPrivateKey(credentials.privateKey);
+			}
+			const sshClient = await this.helpers.getSSHClient(credentials);
+
+			// Create a TCP proxy listening on a random available port
+			const proxy = createServer();
+			const proxyPort = await new Promise<number>((resolve) => {
+				proxy.listen(0, LOCALHOST, () => {
+					resolve((proxy.address() as AddressInfo).port);
+				});
+			});
+
+			const close = () => {
+				proxy.close();
+				sshClient.off('end', close);
+				sshClient.off('error', close);
+			};
+			sshClient.on('end', close);
+			sshClient.on('error', close);
+
+			await new Promise<void>((resolve, reject) => {
+				proxy.on('error', (err) => reject(err));
+				proxy.on('connection', (localSocket) => {
+					sshClient.forwardOut(
+						LOCALHOST,
+						localSocket.remotePort!,
+						credentials.host,
+						credentials.port,
+						(err, clientChannel) => {
+							if (err) {
+								proxy.close();
+								localSocket.destroy();
+							} else {
+								localSocket.pipe(clientChannel);
+								clientChannel.pipe(localSocket);
+							}
+						},
+					);
+				});
+				resolve();
+			}).catch((err) => {
+				proxy.close();
+
+				let message = err.message;
+				let description = err.description;
+
+				if (err.message.includes('ECONNREFUSED')) {
+					message = 'Connection refused';
+					try {
+						description = err.message.split('ECONNREFUSED ')[1].trim();
+					} catch (e) {}
 				}
 
-				return parsedDate.toISOString();
+				if (err.message.includes('ENOTFOUND')) {
+					message = 'Host not found';
+					try {
+						description = err.message.split('ENOTFOUND ')[1].trim();
+					} catch (e) {}
+				}
+
+				if (err.message.includes('ETIMEDOUT')) {
+					message = 'Connection timed out';
+					try {
+						description = err.message.split('ETIMEDOUT ')[1].trim();
+					} catch (e) {}
+				}
+
+				err.message = message;
+				err.description = description;
+				throw err;
 			});
-		});
-	}
 
-	if (options.largeNumbersOutput === 'numbers') {
-		pgp.pg.types.setTypeParser(20, (value: string) => {
-			return parseInt(value, 10);
-		});
-		pgp.pg.types.setTypeParser(1700, (value: string) => {
-			return parseFloat(value);
-		});
-	}
-
-	const dbConfig = getPostgresConfig(credentials, options);
-
-	if (!credentials.sshTunnel) {
-		const db = pgp(dbConfig);
-		return { db, pgp };
-	} else {
-		if (credentials.sshAuthenticateWith === 'privateKey' && credentials.privateKey) {
-			credentials.privateKey = formatPrivateKey(credentials.privateKey);
+			const db = pgp({
+				...dbConfig,
+				port: proxyPort,
+				host: LOCALHOST,
+			});
+			return { db, pgp };
 		}
-		const sshClient = await this.helpers.getSSHClient(credentials);
+	};
 
-		// Create a TCP proxy listening on a random available port
-		const proxy = createServer();
-		const proxyPort = await new Promise<number>((resolve) => {
-			proxy.listen(0, LOCALHOST, () => {
-				resolve((proxy.address() as AddressInfo).port);
-			});
-		});
-
-		const close = () => {
-			proxy.close();
-			sshClient.off('end', close);
-			sshClient.off('error', close);
-		};
-		sshClient.on('end', close);
-		sshClient.on('error', close);
-
-		await new Promise<void>((resolve, reject) => {
-			proxy.on('error', (err) => reject(err));
-			proxy.on('connection', (localSocket) => {
-				sshClient.forwardOut(
-					LOCALHOST,
-					localSocket.remotePort!,
-					credentials.host,
-					credentials.port,
-					(err, clientChannel) => {
-						if (err) {
-							proxy.close();
-							localSocket.destroy();
-						} else {
-							localSocket.pipe(clientChannel);
-							clientChannel.pipe(localSocket);
-						}
-					},
-				);
-			});
-			resolve();
-		}).catch((err) => {
-			proxy.close();
-
-			let message = err.message;
-			let description = err.description;
-
-			if (err.message.includes('ECONNREFUSED')) {
-				message = 'Connection refused';
-				try {
-					description = err.message.split('ECONNREFUSED ')[1].trim();
-				} catch (e) {}
-			}
-
-			if (err.message.includes('ENOTFOUND')) {
-				message = 'Host not found';
-				try {
-					description = err.message.split('ENOTFOUND ')[1].trim();
-				} catch (e) {}
-			}
-
-			if (err.message.includes('ETIMEDOUT')) {
-				message = 'Connection timed out';
-				try {
-					description = err.message.split('ETIMEDOUT ')[1].trim();
-				} catch (e) {}
-			}
-
-			err.message = message;
-			err.description = description;
-			throw err;
-		});
-
-		const db = pgp({
-			...dbConfig,
-			port: proxyPort,
-			host: LOCALHOST,
-		});
-		return { db, pgp };
-	}
+	return await poolManager.getConnection({
+		credentials,
+		nodeType: 'postgres',
+		nodeVersion: options.nodeVersion as unknown as string,
+		fallBackHandler,
+		cleanUpHandler: async ({ db }) => {
+			await db.$pool.end();
+		},
+	});
 }