Merge branch 'master' into feature/wordpress-node

packages/nodes-base/nodes/Function.node.ts

@@ -63,9 +63,8 @@ export class Function implements INodeType {
 			console: 'inherit',
 			sandbox,
 			require: {
-				external: false,
+				external: false as boolean | { modules: string[] },
 				builtin: [] as string[],
-				root: './',
 			}
 		};
 
@@ -73,6 +72,11 @@ export class Function implements INodeType {
 			options.require.builtin = process.env.NODE_FUNCTION_ALLOW_BUILTIN.split(',');
 		}
 
+		if (process.env.NODE_FUNCTION_ALLOW_EXTERNAL) {
+			options.require.external = { modules: process.env.NODE_FUNCTION_ALLOW_EXTERNAL.split(',') };
+		}
+
+
 		const vm = new NodeVM(options);
 
 		// Get the code to execute
@@ -80,7 +84,7 @@ export class Function implements INodeType {
 
 		try {
 			// Execute the function code
-			items = (await vm.run(`module.exports = async function() {${functionCode}}()`));
+			items = (await vm.run(`module.exports = async function() {${functionCode}}()`, './'));
 		} catch (e) {
 			return Promise.reject(e);
 		}

packages/nodes-base/nodes/Google/GoogleSheet.ts

@@ -93,7 +93,7 @@ export class GoogleSheet {
 			}
 		);
 
-		return response.data.values;
+		return response.data.values as string[][] | undefined;
 	}
 
 
@@ -141,9 +141,9 @@ export class GoogleSheet {
 	async batchUpdate(updateData: ISheetUpdateData[], valueInputMode: ValueInputOption) {
 		const client = await this.getAuthenticationClient();
 
+		// @ts-ignore
 		const response = await Sheets.spreadsheets.values.batchUpdate(
 			{
-				// @ts-ignore
 				auth: client,
 				spreadsheetId: this.id,
 				valueInputOption: valueInputMode,
@@ -163,6 +163,7 @@ export class GoogleSheet {
 	async setData(range: string, data: string[][], valueInputMode: ValueInputOption) {
 		const client = await this.getAuthenticationClient();
 
+		// @ts-ignore
 		const response = await Sheets.spreadsheets.values.update(
 			{
 				// @ts-ignore
@@ -186,9 +187,9 @@ export class GoogleSheet {
 	async appendData(range: string, data: string[][], valueInputMode: ValueInputOption) {
 		const client = await this.getAuthenticationClient();
 
+		// @ts-ignore
 		const response = await Sheets.spreadsheets.values.append(
 			{
-				// @ts-ignore
 				auth: client,
 				spreadsheetId: this.id,
 				range,

packages/nodes-base/nodes/HtmlExtract/HtmlExtract.node.ts

@@ -0,0 +1,277 @@
+import * as cheerio from 'cheerio';
+import { IExecuteFunctions } from 'n8n-core';
+import {
+	INodeExecutionData,
+	INodeType,
+	INodeTypeDescription,
+	IDataObject,
+} from 'n8n-workflow';
+
+interface IValueData {
+	attribute?: string;
+	cssSelector: string;
+	returnValue: string;
+	key: string;
+	returnArray: boolean;
+}
+
+
+// The extraction functions
+const extractFunctions: {
+	[key: string]: ($: Cheerio, valueData: IValueData) => string | undefined;
+} = {
+	attribute: ($: Cheerio, valueData: IValueData): string | undefined => $.attr(valueData.attribute!),
+	html: ($: Cheerio, valueData: IValueData): string | undefined => $.html() || undefined,
+	text: ($: Cheerio, valueData: IValueData): string | undefined => $.text(),
+	value: ($: Cheerio, valueData: IValueData): string | undefined => $.val(),
+};
+
+
+/**
+ * Simple helper function which applies options
+ */
+function getValue($: Cheerio, valueData: IValueData, options: IDataObject) {
+	const value = extractFunctions[valueData.returnValue]($, valueData);
+	if (options.trimValues === false || value === undefined) {
+		return value;
+	}
+
+	return value.trim();
+}
+
+
+export class HtmlExtract implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'HTML Extract',
+		name: 'htmlExtract',
+		icon: 'fa:cut',
+		group: ['transform'],
+		version: 1,
+		subtitle: '={{$parameter["sourceData"] + ": " + $parameter["dataPropertyName"]}}',
+		description: 'Extracts data from HTML',
+		defaults: {
+			name: 'HTML Extract',
+			color: '#333377',
+		},
+		inputs: ['main'],
+		outputs: ['main'],
+		properties: [
+			{
+				displayName: 'Source Data',
+				name: 'sourceData',
+				type: 'options',
+				options: [
+					{
+						name: 'Binary',
+						value: 'binary',
+					},
+					{
+						name: 'JSON',
+						value: 'json',
+					},
+				],
+				default: 'json',
+				description: 'If HTML should be read from binary or json data.',
+			},
+			{
+				displayName: 'Binary Property',
+				name: 'dataPropertyName',
+				type: 'string',
+				displayOptions: {
+					show: {
+						sourceData: [
+							'binary',
+						],
+					},
+				},
+				default: 'data',
+				required: true,
+				description: 'Name of the binary property in which the HTML to extract the data from can be found.',
+			},
+			{
+				displayName: 'JSON Property',
+				name: 'dataPropertyName',
+				type: 'string',
+				displayOptions: {
+					show: {
+						sourceData: [
+							'json',
+						],
+					},
+				},
+				default: 'data',
+				required: true,
+				description: 'Name of the json property in which the HTML to extract the data from can be found.<br />The property can either contain a string or an array of strings.',
+			},
+			{
+				displayName: 'Extraction Values',
+				name: 'extractionValues',
+				placeholder: 'Add Value',
+				type: 'fixedCollection',
+				typeOptions: {
+					multipleValues: true,
+				},
+				description: 'The extraction values.',
+				default: {},
+				options: [
+					{
+						name: 'values',
+						displayName: 'Values',
+						values: [
+							{
+								displayName: 'Key',
+								name: 'key',
+								type: 'string',
+								default: '',
+								description: 'The key under which the extracted value should be saved.',
+							},
+							{
+								displayName: 'CSS Selector',
+								name: 'cssSelector',
+								type: 'string',
+								default: '',
+								placeholder: '.price',
+								description: 'The CSS selector to use.',
+							},
+							{
+								displayName: 'Return Value',
+								name: 'returnValue',
+								type: 'options',
+								options: [
+									{
+										name: 'Attribute',
+										value: 'attribute',
+										description: 'Get an attribute value like "class" from an element.',
+									},
+									{
+										name: 'HTML',
+										value: 'html',
+										description: 'Get the HTML the element contains.',
+									},
+									{
+										name: 'Text',
+										value: 'text',
+										description: 'Get only the text content of the element.',
+									},
+									{
+										name: 'Value',
+										value: 'value',
+										description: 'Get value of an input, select or textarea.',
+									},
+								],
+								default: 'text',
+								description: 'What kind of data should be returned.',
+							},
+							{
+								displayName: 'Attribute',
+								name: 'attribute',
+								type: 'string',
+								displayOptions: {
+									show: {
+										returnValue: [
+											'attribute',
+										],
+									},
+								},
+								default: '',
+								placeholder: 'class',
+								description: 'The name of the attribute to return the value off.',
+							},
+							{
+								displayName: 'Return Array',
+								name: 'returnArray',
+								type: 'boolean',
+								default: false,
+								description: 'Returns the values as an array so if multiple ones get found they also get<br />returned separately.If not set all will be returned as a single string.',
+							},
+						],
+					},
+				],
+			},
+
+			{
+				displayName: 'Options',
+				name: 'options',
+				type: 'collection',
+				placeholder: 'Add Option',
+				default: {},
+				options: [
+					{
+						displayName: 'Trim Values',
+						name: 'trimValues',
+						type: 'boolean',
+						default: true,
+						description: 'Removes automatically all spaces and newlines from<br />the beginning and end of the values.',
+					},
+				],
+			}
+		]
+	};
+
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		const items = this.getInputData();
+
+		const returnData: INodeExecutionData[] = [];
+
+		let item: INodeExecutionData;
+		for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+			const dataPropertyName = this.getNodeParameter('dataPropertyName', itemIndex) as string;
+			const extractionValues = this.getNodeParameter('extractionValues', itemIndex) as IDataObject;
+			const options = this.getNodeParameter('options', itemIndex, {}) as IDataObject;
+			const sourceData = this.getNodeParameter('sourceData', itemIndex) as string;
+
+			item = items[itemIndex];
+
+			let htmlArray: string[] | string = [];
+			if (sourceData === 'json') {
+				if (item.json[dataPropertyName] === undefined) {
+					throw new Error(`No property named "${dataPropertyName}" exists!`);
+				}
+				htmlArray = item.json[dataPropertyName] as string;
+			} else {
+				if (item.binary === undefined) {
+					throw new Error(`No item does not contain binary data!`);
+				}
+				if (item.binary[dataPropertyName] === undefined) {
+					throw new Error(`No property named "${dataPropertyName}" exists!`);
+				}
+				htmlArray = Buffer.from(item.binary[dataPropertyName].data, 'base64').toString('utf8');
+			}
+
+			// Convert it always to array that it works with a string or an array of strings
+			if (!Array.isArray(htmlArray)) {
+				htmlArray = [htmlArray];
+			}
+
+			for (const html of htmlArray as string[]) {
+				const $ = cheerio.load(html);
+
+				const newItem: INodeExecutionData = {
+					json: {},
+				};
+
+				// Itterate over all the defined values which should be extracted
+				let htmlElement;
+				for (const valueData of extractionValues.values as IValueData[]) {
+					htmlElement = $(valueData.cssSelector);
+
+					if (valueData.returnArray === true) {
+						// An array should be returned so itterate over one
+						// value at a time
+						newItem.json[valueData.key as string] = [];
+						htmlElement.each((i, el) => {
+							(newItem.json[valueData.key as string] as Array<string | undefined>).push(getValue($(el), valueData, options));
+						});
+					} else {
+						// One single value should be returned
+						newItem.json[valueData.key as string] = getValue(htmlElement, valueData, options);
+					}
+				}
+				returnData.push(newItem);
+			}
+		}
+
+		return this.prepareOutputData(returnData);
+	}
+}

packages/nodes-base/nodes/HttpRequest.node.ts

@@ -25,6 +25,7 @@ export class HttpRequest implements INodeType {
 		icon: 'fa:at',
 		group: ['input'],
 		version: 1,
+		subtitle: '={{$parameter["requestMethod"] + ": " + $parameter["url"]}}',
 		description: 'Makes a HTTP request and returns the received data',
 		defaults: {
 			name: 'HTTP Request',

packages/nodes-base/nodes/SpreadsheetFile.node.ts

@@ -109,27 +109,27 @@ export class SpreadsheetFile implements INodeType {
 				type: 'options',
 				options: [
 					{
-						name: 'csv',
+						name: 'CSV',
 						value: 'csv',
 						description: 'Comma-separated values',
 					},
 					{
-						name: 'ods',
-						value: 'ods',
-						description: 'OpenDocument Spreadsheet',
-					},
-					{
-						name: 'rtf',
-						value: 'rtf',
-						description: 'Rich Text Format',
-					},
-					{
-						name: 'html',
+						name: 'HTML',
 						value: 'html',
 						description: 'HTML Table',
 					},
 					{
-						name: 'xls',
+						name: 'ODS',
+						value: 'ods',
+						description: 'OpenDocument Spreadsheet',
+					},
+					{
+						name: 'RTF',
+						value: 'rtf',
+						description: 'Rich Text Format',
+					},
+					{
+						name: 'XLS',
 						value: 'xls',
 						description: 'Excel',
 					},
@@ -166,22 +166,68 @@ export class SpreadsheetFile implements INodeType {
 				name: 'options',
 				type: 'collection',
 				placeholder: 'Add Option',
-				displayOptions: {
-					show: {
-						operation: [
-							'toFile',
-						],
-					},
-				},
 				default: {},
 				options: [
 					{
 						displayName: 'File Name',
 						name: 'fileName',
 						type: 'string',
+						displayOptions: {
+							show: {
+								'/operation': [
+									'toFile',
+								],
+							},
+						},
 						default: '',
 						description: 'File name to set in binary data. By default will "spreadsheet.<fileFormat>" be used.',
 					},
+					{
+						displayName: 'RAW Data',
+						name: 'rawData',
+						type: 'boolean',
+						displayOptions: {
+							show: {
+								'/operation': [
+									'fromFile'
+								],
+							},
+						},
+						default: false,
+						description: 'If the data should be returned RAW instead of parsed.',
+					},
+					{
+						displayName: 'Sheet Name',
+						name: 'sheetName',
+						type: 'string',
+						displayOptions: {
+							show: {
+								'/operation': [
+									'fromFile',
+								],
+							},
+						},
+						default: 'Sheet',
+						description: 'Name of the sheet to read from in the spreadsheet (if supported). If not set, the first one gets chosen.',
+					},
+					{
+						displayName: 'Sheet Name',
+						name: 'sheetName',
+						type: 'string',
+						displayOptions: {
+							show: {
+								'/operation': [
+									'toFile',
+								],
+								'/fileFormat': [
+									'ods',
+									'xls',
+								],
+							},
+						},
+						default: 'Sheet',
+						description: 'Name of the sheet to create in the spreadsheet.',
+					},
 				],
 			},
 		]
@@ -203,7 +249,8 @@ export class SpreadsheetFile implements INodeType {
 			for (let i = 0; i < items.length; i++) {
 				item = items[i];
 
-				const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string;
+				const binaryPropertyName = this.getNodeParameter('binaryPropertyName', i) as string;
+				const options = this.getNodeParameter('options', i, {}) as IDataObject;
 
 				if (item.binary === undefined || item.binary[binaryPropertyName] === undefined) {
 					// Property did not get found on item
@@ -212,14 +259,22 @@ export class SpreadsheetFile implements INodeType {
 
 				// Read the binary spreadsheet data
 				const binaryData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
-				const workbook = xlsxRead(binaryData);
+				const workbook = xlsxRead(binaryData, { raw: options.rawData as boolean });
 
 				if (workbook.SheetNames.length === 0) {
-					throw new Error('File does not have any sheets!');
+					throw new Error('Spreadsheet does not have any sheets!');
+				}
+
+				let sheetName = workbook.SheetNames[0];
+				if (options.sheetName) {
+					if (!workbook.SheetNames.includes(options.sheetName as string)) {
+						throw new Error(`Spreadsheet does not contain sheet called "${options.sheetName}"!`);
+					}
+					sheetName = options.sheetName as string;
 				}
 
 				// Convert it to json
-				const sheetJson = xlsxUtils.sheet_to_json(workbook.Sheets[workbook.SheetNames[0]]);
+				const sheetJson = xlsxUtils.sheet_to_json(workbook.Sheets[sheetName]);
 
 				// Check if data could be found in file
 				if (sheetJson.length === 0) {
@@ -267,7 +322,7 @@ export class SpreadsheetFile implements INodeType {
 			}
 
 			// Convert the data in the correct format
-			const sheetName = 'Sheet';
+			const sheetName = options.sheetName as string || 'Sheet';
 			const wb: WorkBook = {
 				SheetNames: [sheetName],
 				Sheets: {