feat(Google Sheets Node): Overhaul of node

packages/nodes-base/nodes/Google/Sheet/v2/helpers/GoogleSheet.ts

@@ -0,0 +1,657 @@
+import { IDataObject, NodeOperationError } from 'n8n-workflow';
+import { IExecuteFunctions, ILoadOptionsFunctions } from 'n8n-core';
+import { apiRequest } from '../transport';
+import { utils as xlsxUtils } from 'xlsx';
+import { get } from 'lodash';
+import {
+	ILookupValues,
+	ISheetUpdateData,
+	SheetCellDecoded,
+	SheetRangeData,
+	SheetRangeDecoded,
+	ValueInputOption,
+	ValueRenderOption,
+} from './GoogleSheets.types';
+import { removeEmptyColumns } from './GoogleSheets.utils';
+
+export class GoogleSheet {
+	id: string;
+	executeFunctions: IExecuteFunctions | ILoadOptionsFunctions;
+
+	constructor(spreadsheetId: string, executeFunctions: IExecuteFunctions | ILoadOptionsFunctions) {
+		this.executeFunctions = executeFunctions;
+		this.id = spreadsheetId;
+	}
+
+	/**
+	 * Encodes the range that also none latin character work
+	 *
+	 * @param {string} range
+	 * @returns {string}
+	 * @memberof GoogleSheet
+	 */
+	private encodeRange(range: string): string {
+		if (range.includes('!')) {
+			const [sheet, ranges] = range.split('!');
+			return `${encodeURIComponent(sheet)}!${ranges}`;
+		}
+		return encodeURIComponent(range);
+	}
+
+	/**
+	 * Clears values from a sheet
+	 *
+	 * @param {string} range
+	 * @returns {Promise<object>}
+	 * @memberof GoogleSheet
+	 */
+	async clearData(range: string): Promise<object> {
+		const body = {
+			spreadsheetId: this.id,
+			range,
+		};
+
+		const response = await apiRequest.call(
+			this.executeFunctions,
+			'POST',
+			`/v4/spreadsheets/${this.id}/values/${this.encodeRange(range)}:clear`,
+			body,
+		);
+
+		return response;
+	}
+
+	/**
+	 * Returns the cell values
+	 */
+	async getData(range: string, valueRenderMode: ValueRenderOption, dateTimeRenderOption?: string) {
+		const query: IDataObject = {
+			valueRenderOption: valueRenderMode,
+			dateTimeRenderOption: 'FORMATTED_STRING',
+		};
+
+		if (dateTimeRenderOption) {
+			query.dateTimeRenderOption = dateTimeRenderOption;
+		}
+
+		const response = await apiRequest.call(
+			this.executeFunctions,
+			'GET',
+			`/v4/spreadsheets/${this.id}/values/${this.encodeRange(range)}`,
+			{},
+			query,
+		);
+
+		return response.values as string[][] | undefined;
+	}
+
+	/**
+	 * Returns the sheets in a Spreadsheet
+	 */
+	async spreadsheetGetSheets() {
+		const query = {
+			fields: 'sheets.properties',
+		};
+
+		const response = await apiRequest.call(
+			this.executeFunctions,
+			'GET',
+			`/v4/spreadsheets/${this.id}`,
+			{},
+			query,
+		);
+
+		return response;
+	}
+
+	/**
+	 *  Returns the name of a sheet from a sheet id
+	 */
+	async spreadsheetGetSheetNameById(sheetId: string) {
+		const query = {
+			fields: 'sheets.properties',
+		};
+
+		const response = await apiRequest.call(
+			this.executeFunctions,
+			'GET',
+			`/v4/spreadsheets/${this.id}`,
+			{},
+			query,
+		);
+
+		const foundItem = response.sheets.find(
+			(item: { properties: { sheetId: number } }) => item.properties.sheetId === +sheetId,
+		);
+		if (!foundItem || !foundItem.properties || !foundItem.properties.title) {
+			throw new Error(`Sheet with id ${sheetId} not found`);
+		}
+		return foundItem.properties.title;
+	}
+
+	/**
+	 *  Returns the grid properties of a sheet
+	 */
+	async getDataRange(sheetId: string) {
+		const query = {
+			fields: 'sheets.properties',
+		};
+
+		const response = await apiRequest.call(
+			this.executeFunctions,
+			'GET',
+			`/v4/spreadsheets/${this.id}`,
+			{},
+			query,
+		);
+		const foundItem = response.sheets.find(
+			(item: { properties: { sheetId: string } }) => item.properties.sheetId === sheetId,
+		);
+		return foundItem.properties.gridProperties;
+	}
+
+	/**
+	 * Sets values in one or more ranges of a spreadsheet.
+	 */
+	async spreadsheetBatchUpdate(requests: IDataObject[]) {
+		const body = {
+			requests,
+		};
+
+		const response = await apiRequest.call(
+			this.executeFunctions,
+			'POST',
+			`/v4/spreadsheets/${this.id}:batchUpdate`,
+			body,
+		);
+
+		return response;
+	}
+
+	/**
+	 * Sets the cell values
+	 */
+	async batchUpdate(updateData: ISheetUpdateData[], valueInputMode: ValueInputOption) {
+		const body = {
+			data: updateData,
+			valueInputOption: valueInputMode,
+		};
+
+		const response = await apiRequest.call(
+			this.executeFunctions,
+			'POST',
+			`/v4/spreadsheets/${this.id}/values:batchUpdate`,
+			body,
+		);
+
+		return response;
+	}
+
+	/**
+	 * Appends the cell values
+	 */
+	async appendData(
+		range: string,
+		data: string[][],
+		valueInputMode: ValueInputOption,
+		lastRow?: number,
+	) {
+		// const body = {
+		// 	range,
+		// 	values: data,
+		// };
+
+		// const query = {
+		// 	valueInputOption: valueInputMode,
+		// };
+
+		// const response = await apiRequest.call(
+		// 	this.executeFunctions,
+		// 	'POST',
+		// 	`/v4/spreadsheets/${this.id}/values/${this.encodeRange(range)}:append`,
+		// 	body,
+		// 	query,
+		// );
+
+		const lastRowWithData =
+			lastRow ||
+			(((await this.getData(range, 'UNFORMATTED_VALUE')) as string[][]) || []).length + 1;
+
+		const response = await this.updateRows(
+			range,
+			data,
+			valueInputMode,
+			lastRowWithData,
+			data.length,
+		);
+
+		return response;
+	}
+
+	async updateRows(
+		sheetName: string,
+		data: string[][],
+		valueInputMode: ValueInputOption,
+		row: number,
+		rowsLength?: number,
+	) {
+		const [name, _sheetRange] = sheetName.split('!');
+		const range = `${name}!${row}:${rowsLength ? row + rowsLength - 1 : row}`;
+
+		const body = {
+			range,
+			values: data,
+		};
+
+		const query = {
+			valueInputOption: valueInputMode,
+		};
+
+		const response = await apiRequest.call(
+			this.executeFunctions,
+			'PUT',
+			`/v4/spreadsheets/${this.id}/values/${this.encodeRange(range)}`,
+			body,
+			query,
+		);
+
+		return response;
+	}
+
+	/**
+	 * Returns the given sheet data in a structured way
+	 */
+	convertSheetDataArrayToObjectArray(
+		data: SheetRangeData,
+		startRow: number,
+		columnKeys: string[],
+		addEmpty?: boolean,
+	): IDataObject[] {
+		const returnData = [];
+
+		for (let rowIndex = startRow; rowIndex < data.length; rowIndex++) {
+			const item: IDataObject = {};
+			for (let columnIndex = 0; columnIndex < data[rowIndex].length; columnIndex++) {
+				const key = columnKeys[columnIndex];
+				if (key) {
+					item[key] = data[rowIndex][columnIndex];
+				}
+			}
+			if (Object.keys(item).length || addEmpty === true) {
+				returnData.push(item);
+			}
+		}
+
+		return returnData;
+	}
+
+	/**
+	 * Returns the given sheet data in a structured way using
+	 * the startRow as the one with the name of the key
+	 */
+	structureArrayDataByColumn(
+		inputData: string[][],
+		keyRow: number,
+		dataStartRow: number,
+	): IDataObject[] {
+		const keys: string[] = [];
+
+		if (keyRow < 0 || dataStartRow < keyRow || keyRow >= inputData.length) {
+			// The key row does not exist so it is not possible to structure data
+			return [];
+		}
+
+		const longestRow = inputData.reduce((a, b) => (a.length > b.length ? a : b), []).length;
+		for (let columnIndex = 0; columnIndex < longestRow; columnIndex++) {
+			keys.push(inputData[keyRow][columnIndex] || `col_${columnIndex}`);
+		}
+
+		return this.convertSheetDataArrayToObjectArray(inputData, dataStartRow, keys);
+	}
+
+	testFilter(inputData: string[][], keyRow: number, dataStartRow: number): string[] {
+		const keys: string[] = [];
+		//const returnData = [];
+
+		if (keyRow < 0 || dataStartRow < keyRow || keyRow >= inputData.length) {
+			// The key row does not exist so it is not possible to structure data
+			return [];
+		}
+
+		// Create the keys array
+		for (let columnIndex = 0; columnIndex < inputData[keyRow].length; columnIndex++) {
+			keys.push(inputData[keyRow][columnIndex]);
+		}
+
+		return keys;
+	}
+
+	async appendSheetData(
+		inputData: IDataObject[],
+		range: string,
+		keyRowIndex: number,
+		valueInputMode: ValueInputOption,
+		usePathForKeyRow: boolean,
+		columnNamesList?: string[][],
+		lastRow?: number,
+	): Promise<string[][]> {
+		const data = await this.convertObjectArrayToSheetDataArray(
+			inputData,
+			range,
+			keyRowIndex,
+			usePathForKeyRow,
+			columnNamesList,
+		);
+		return this.appendData(range, data, valueInputMode, lastRow);
+	}
+
+	getColumnWithOffset(startColumn: string, offset: number): string {
+		const columnIndex = xlsxUtils.decode_col(startColumn) + offset;
+		return xlsxUtils.encode_col(columnIndex);
+	}
+
+	async getColumnValues(
+		range: string,
+		keyIndex: number,
+		dataStartRowIndex: number,
+		valueRenderMode: ValueRenderOption,
+		sheetData?: string[][],
+	): Promise<string[]> {
+		let columnValuesList;
+		if (sheetData) {
+			columnValuesList = sheetData.slice(dataStartRowIndex - 1).map((row) => row[keyIndex]);
+		} else {
+			const decodedRange = this.getDecodedSheetRange(range);
+			const startRowIndex = decodedRange.start?.row || dataStartRowIndex;
+			const endRowIndex = decodedRange.end?.row || '';
+
+			const keyColumn = this.getColumnWithOffset(decodedRange.start?.column || 'A', keyIndex);
+			const keyColumnRange = `${decodedRange.name}!${keyColumn}${startRowIndex}:${keyColumn}${endRowIndex}`;
+			columnValuesList = await this.getData(keyColumnRange, valueRenderMode);
+		}
+
+		if (columnValuesList === undefined) {
+			throw new NodeOperationError(
+				this.executeFunctions.getNode(),
+				'Could not retrieve the data from key column',
+			);
+		}
+
+		//Remove the first row which contains the key and flaten the array
+		return columnValuesList.splice(1).flatMap((value) => value);
+	}
+
+	/**
+	 * Updates data in a sheet
+	 *
+	 * @param {IDataObject[]} inputData Data to update Sheet with
+	 * @param {string} indexKey The name of the key which gets used to know which rows to update
+	 * @param {string} range The range to look for data
+	 * @param {number} keyRowIndex Index of the row which contains the keys
+	 * @param {number} dataStartRowIndex Index of the first row which contains data
+	 * @returns {Promise<string[][]>}
+	 * @memberof GoogleSheet
+	 */
+	async prepareDataForUpdateOrUpsert(
+		inputData: IDataObject[],
+		indexKey: string,
+		range: string,
+		keyRowIndex: number,
+		dataStartRowIndex: number,
+		valueRenderMode: ValueRenderOption,
+		upsert = false,
+		columnNamesList?: string[][],
+		columnValuesList?: string[],
+	) {
+		const decodedRange = this.getDecodedSheetRange(range);
+		// prettier-ignore
+		const	keyRowRange = `${decodedRange.name}!${decodedRange.start?.column || ''}${keyRowIndex + 1}:${decodedRange.end?.column || ''}${keyRowIndex + 1}`;
+
+		const sheetDatakeyRow = columnNamesList || (await this.getData(keyRowRange, valueRenderMode));
+
+		if (sheetDatakeyRow === undefined) {
+			throw new NodeOperationError(
+				this.executeFunctions.getNode(),
+				'Could not retrieve the key row',
+			);
+		}
+
+		const columnNames = sheetDatakeyRow[0];
+
+		const keyIndex = columnNames.indexOf(indexKey);
+
+		if (keyIndex === -1) {
+			throw new NodeOperationError(
+				this.executeFunctions.getNode(),
+				`Could not find column for key "${indexKey}"`,
+			);
+		}
+
+		const columnValues =
+			columnValuesList ||
+			(await this.getColumnValues(range, keyIndex, dataStartRowIndex, valueRenderMode));
+
+		const updateData: ISheetUpdateData[] = [];
+		const appendData: IDataObject[] = [];
+
+		for (const item of inputData) {
+			const inputIndexKey = item[indexKey] as string;
+			if (inputIndexKey === undefined || inputIndexKey === null) {
+				// Item does not have the indexKey so we can ignore it or append it if upsert true
+				if (upsert) {
+					appendData.push(item);
+				}
+				continue;
+			}
+
+			// Item does have the key so check if it exists in Sheet
+			const indexOfIndexKeyInSheet = columnValues.indexOf(inputIndexKey);
+			if (indexOfIndexKeyInSheet === -1) {
+				// Key does not exist in the Sheet so it can not be updated so skip it or append it if upsert true
+				if (upsert) {
+					appendData.push(item);
+				}
+				continue;
+			}
+
+			// Get the row index in which the data should be updated
+			const updateRowIndex = indexOfIndexKeyInSheet + dataStartRowIndex + 1;
+
+			// Check all the properties in the sheet and check which ones exist on the
+			// item and should be updated
+			for (const name of columnNames) {
+				if (name === indexKey) {
+					// Ignore the key itself as that does not get changed it gets
+					// only used to find the correct row to update
+					continue;
+				}
+				if (item[name] === undefined || item[name] === null) {
+					// Property does not exist so skip it
+					continue;
+				}
+
+				// Property exists so add it to the data to update
+				// Get the column name in which the property data can be found
+				const columnToUpdate = this.getColumnWithOffset(
+					decodedRange.start?.column || 'A',
+					columnNames.indexOf(name),
+				);
+
+				updateData.push({
+					range: `${decodedRange.name}!${columnToUpdate}${updateRowIndex}`,
+					values: [[item[name] as string]],
+				});
+			}
+		}
+
+		return { updateData, appendData };
+	}
+
+	/**
+	 * Looks for a specific value in a column and if it gets found it returns the whole row
+	 *
+	 * @param {string[][]} inputData Data to to check for lookup value in
+	 * @param {number} keyRowIndex Index of the row which contains the keys
+	 * @param {number} dataStartRowIndex Index of the first row which contains data
+	 * @param {ILookupValues[]} lookupValues The lookup values which decide what data to return
+	 * @param {boolean} [returnAllMatches] Returns all the found matches instead of only the first one
+	 * @returns {Promise<IDataObject[]>}
+	 * @memberof GoogleSheet
+	 */
+	async lookupValues(
+		inputData: string[][],
+		keyRowIndex: number,
+		dataStartRowIndex: number,
+		lookupValues: ILookupValues[],
+		returnAllMatches?: boolean,
+	): Promise<IDataObject[]> {
+		const keys: string[] = [];
+
+		if (keyRowIndex < 0 || dataStartRowIndex < keyRowIndex || keyRowIndex >= inputData.length) {
+			// The key row does not exist so it is not possible to look up the data
+			throw new NodeOperationError(this.executeFunctions.getNode(), `The key row does not exist`);
+		}
+
+		// Create the keys array
+		for (let columnIndex = 0; columnIndex < inputData[keyRowIndex].length; columnIndex++) {
+			keys.push(inputData[keyRowIndex][columnIndex] || `col_${columnIndex}`);
+		}
+
+		// Standardise values array, if rows is [[]], map it to [['']] (Keep the columns into consideration)
+		for (let rowIndex = 0; rowIndex < inputData?.length; rowIndex++) {
+			if (inputData[rowIndex].length === 0) {
+				for (let i = 0; i < keys.length; i++) {
+					inputData[rowIndex][i] = '';
+				}
+			} else if (inputData[rowIndex].length < keys.length) {
+				for (let i = 0; i < keys.length; i++) {
+					if (inputData[rowIndex][i] === undefined) {
+						inputData[rowIndex].push('');
+					}
+				}
+			}
+		}
+		// Loop over all the lookup values and try to find a row to return
+		let rowIndex: number;
+		let returnColumnIndex: number;
+		const addedRows: number[] = [];
+
+		// const returnData = [inputData[keyRowIndex]];
+		const returnData = [keys];
+
+		lookupLoop: for (const lookupValue of lookupValues) {
+			returnColumnIndex = keys.indexOf(lookupValue.lookupColumn);
+
+			if (returnColumnIndex === -1) {
+				throw new NodeOperationError(
+					this.executeFunctions.getNode(),
+					`The column "${lookupValue.lookupColumn}" could not be found`,
+				);
+			}
+
+			// Loop over all the items and find the one with the matching value
+			for (rowIndex = dataStartRowIndex; rowIndex < inputData.length; rowIndex++) {
+				if (
+					inputData[rowIndex][returnColumnIndex]?.toString() === lookupValue.lookupValue.toString()
+				) {
+					if (addedRows.indexOf(rowIndex) === -1) {
+						returnData.push(inputData[rowIndex]);
+						addedRows.push(rowIndex);
+					}
+
+					if (returnAllMatches !== true) {
+						continue lookupLoop;
+					}
+				}
+			}
+		}
+
+		return this.convertSheetDataArrayToObjectArray(removeEmptyColumns(returnData), 1, keys, true);
+	}
+
+	private async convertObjectArrayToSheetDataArray(
+		inputData: IDataObject[],
+		range: string,
+		keyRowIndex: number,
+		usePathForKeyRow: boolean,
+		columnNamesList?: string[][],
+	): Promise<string[][]> {
+		const decodedRange = this.getDecodedSheetRange(range);
+
+		const columnNamesRow =
+			columnNamesList ||
+			(await this.getData(
+				`${decodedRange.name}!${keyRowIndex}:${keyRowIndex}`,
+				'UNFORMATTED_VALUE',
+			));
+
+		if (columnNamesRow === undefined) {
+			throw new NodeOperationError(
+				this.executeFunctions.getNode(),
+				'Could not retrieve the column data',
+			);
+		}
+
+		const columnNames = columnNamesRow ? columnNamesRow[0] : [];
+		const setData: string[][] = [];
+
+		inputData.forEach((item) => {
+			const rowData: string[] = [];
+			columnNames.forEach((key) => {
+				let value;
+				if (usePathForKeyRow) {
+					value = get(item, key) as string;
+				} else {
+					value = item[key] as string;
+				}
+				if (value === undefined || value === null) {
+					rowData.push('');
+					return;
+				}
+				if (typeof value === 'object') {
+					rowData.push(JSON.stringify(value));
+				} else {
+					rowData.push(value);
+				}
+			});
+			setData.push(rowData);
+		});
+		return setData;
+	}
+
+	private getDecodedSheetRange(stringToDecode: string): SheetRangeDecoded {
+		const decodedRange: IDataObject = {};
+		const [name, range] = stringToDecode.split('!');
+
+		decodedRange.nameWithRange = stringToDecode;
+		decodedRange.name = name;
+		decodedRange.range = range || '';
+		decodedRange.start = {};
+		decodedRange.end = {};
+
+		if (range) {
+			const [startCell, endCell] = range.split(':');
+			if (startCell) {
+				decodedRange.start = this.splitCellRange(startCell, range);
+			}
+			if (endCell) {
+				decodedRange.end = this.splitCellRange(endCell, range);
+			}
+		}
+
+		return decodedRange as SheetRangeDecoded;
+	}
+
+	private splitCellRange(cell: string, range: string): SheetCellDecoded {
+		const cellData = cell.match(/([a-zA-Z]{1,10})([0-9]{0,10})/) || [];
+
+		if (cellData === null || cellData.length !== 3) {
+			throw new NodeOperationError(
+				this.executeFunctions.getNode(),
+				`The range "${range}" is not valid`,
+			);
+		}
+
+		return { cell: cellData[0], column: cellData[1], row: +cellData[2] };
+	}
+}