feat(KoBoToolbox Node): Add KoBoToolbox Regular and Trigger Node (#2765)

* First version

* Added hooks

* Added Credentials test

* Add support for downloading attachments

* Slight restructure of downloaded binaries

* Added Trigger node

* Some linting

* Reverting package-lock changes

* Minor GeoJSON parsing fixes

* KoboToolbox: improve GeoJSON format

* Kobo: Support for get/set validation status

* Remove some logs

* [kobo] Fix default attachment options

* Proper debug logging

* Support for hook log status filter

* Kobo: Review fixes

* [kobo]: Add Get All Forms + lookup Form ID

* [kobo] Lookup Form ID in Trigger node

* [kobo] Update branded spelling

* [kobo] Support pagination

* ⚡ fix linting issue

* ⚡ Improvements to #2510

* ⚡ Download files using n8n helper

* ⚡ Improvements

* ⚡ Improvements

* 🐛 Fix filenames

* ⚡ Fix some issues

Co-authored-by: Yann Jouanique <yann.jouanique@oneacrefund.org>
Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>

packages/nodes-base/nodes/KoBoToolbox/FormDescription.ts

@@ -0,0 +1,202 @@
+import {
+	INodeProperties,
+} from 'n8n-workflow';
+
+export const formOperations: INodeProperties[] = [
+	{
+		displayName: 'Operation',
+		name: 'operation',
+		type: 'options',
+		displayOptions: {
+			show: {
+				resource: [
+					'form',
+				],
+			},
+		},
+		options: [
+			{
+				name: 'Get',
+				value: 'get',
+				description: 'Get a form',
+			},
+			{
+				name: 'Get All',
+				value: 'getAll',
+				description: 'Get all forms',
+			},
+		],
+		default: 'get',
+	},
+];
+
+export const formFields: INodeProperties[] = [
+
+	/* -------------------------------------------------------------------------- */
+	/*                                form:get                                    */
+	/* -------------------------------------------------------------------------- */
+	{
+		displayName: 'Form ID',
+		name: 'formId',
+		type: 'string',
+		required: true,
+		default: '',
+		displayOptions: {
+			show: {
+				resource: [
+					'form',
+				],
+				operation: [
+					'get',
+				],
+			},
+		},
+		description: 'Form ID (e.g. aSAvYreNzVEkrWg5Gdcvg)',
+	},
+	/* -------------------------------------------------------------------------- */
+	/*                                form:getAll                                 */
+	/* -------------------------------------------------------------------------- */
+	{
+		displayName: 'Return All',
+		name: 'returnAll',
+		type: 'boolean',
+		required: true,
+		default: false,
+		displayOptions: {
+			show: {
+				resource: [
+					'form',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+		description: 'Whether to return all results',
+	},
+	{
+		displayName: 'Limit',
+		name: 'limit',
+		type: 'number',
+		required: false,
+		typeOptions: {
+			maxValue: 3000,
+		},
+		displayOptions: {
+			show: {
+				resource: [
+					'form',
+				],
+				operation: [
+					'getAll',
+				],
+				returnAll: [
+					false,
+				],
+			},
+		},
+		default: 1000,
+		description: 'The number of results to return',
+	},
+	{
+		displayName: 'Options',
+		name: 'options',
+		placeholder: 'Add Option',
+		type: 'collection',
+		default: {},
+		displayOptions: {
+			show: {
+				resource: [
+					'form',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+		options: [
+			{
+				displayName: 'Sort',
+				name: 'sort',
+				type: 'fixedCollection',
+				typeOptions: {
+					multipleValues: false,
+				},
+				default: '',
+				placeholder: 'Add Sort',
+				options: [
+					{
+						displayName: 'Sort',
+						name: 'value',
+						values: [
+							{
+								displayName: 'Descending',
+								name: 'descending',
+								type: 'boolean',
+								default: true,
+								description: 'Sort by descending order',
+							},
+							{
+								displayName: 'Order By',
+								name: 'ordering',
+								type: 'options',
+								required: false,
+								default: 'date_modified',
+								options: [
+									{
+										name: 'Asset Type',
+										value: 'asset_type',
+									},
+									{
+										name: 'Date Modified',
+										value: 'date_modified',
+									},
+									{
+										name: 'Name',
+										value: 'name',
+									},
+									{
+										name: 'Owner Username',
+										value: 'owner__username',
+									},
+									{
+										name: 'Subscribers Count',
+										value: 'subscribers_count',
+									},
+								],
+								description: 'Field to order by',
+							},
+						],
+					},
+				],
+			},
+		],
+	},
+	{
+		displayName: 'Filters',
+		name: 'filters',
+		type: 'collection',
+		placeholder: 'Add Filter',
+		default: {},
+		displayOptions: {
+			show: {
+				resource: [
+					'form',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+		options: [
+			{
+				displayName: 'Filter',
+				name: 'filter',
+				type: 'string',
+				default: 'asset_type:survey',
+				required: false,
+				description: 'A text search query based on form data - e.g. "owner__username:meg AND name__icontains:quixotic" - see <a href="https://github.com/kobotoolbox/kpi#searching" target="_blank">docs</a> for more details',
+			},
+		],
+	},
+];

packages/nodes-base/nodes/KoBoToolbox/GenericFunctions.ts

@@ -0,0 +1,238 @@
+import {
+	IExecuteFunctions,
+	ILoadOptionsFunctions,
+} from 'n8n-core';
+
+import {
+	IDataObject,
+	IHookFunctions,
+	IHttpRequestOptions,
+	INodeExecutionData,
+	INodePropertyOptions,
+	IWebhookFunctions,
+} from 'n8n-workflow';
+
+import * as _ from 'lodash';
+
+export async function koBoToolboxApiRequest(this: IExecuteFunctions | IWebhookFunctions | IHookFunctions | ILoadOptionsFunctions, option: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any
+	const credentials = await this.getCredentials('koBoToolboxApi') as IDataObject;
+
+	// Set up pagination / scrolling
+	const returnAll = !!option.returnAll;
+	if (returnAll) {
+		// Override manual pagination options
+		_.set(option, 'qs.limit', 3000);
+		// Don't pass this custom param to helpers.httpRequest
+		delete option.returnAll;
+	}
+
+	const options: IHttpRequestOptions = {
+		url: '',
+		headers: {
+			'Accept': 'application/json',
+			'Authorization': `Token ${credentials.token}`,
+		},
+		json: true,
+	};
+	if (Object.keys(option)) {
+		Object.assign(options, option);
+	}
+	if (options.url && !/^http(s)?:/.test(options.url)) {
+		options.url = credentials.URL + options.url;
+	}
+
+	let results = null;
+	let keepLooking = true;
+	while (keepLooking) {
+		const response = await this.helpers.httpRequest(options);
+		// Append or set results
+		results = response.results ? _.concat(results || [], response.results) : response;
+		if (returnAll && response.next) {
+			options.url = response.next;
+			continue;
+		}
+		else {
+			keepLooking = false;
+		}
+	}
+
+	return results;
+}
+
+function parseGeoPoint(geoPoint: string): null | number[] {
+	// Check if it looks like a "lat lon z precision" flat string e.g. "-1.931161 30.079811 0 0" (lat, lon, elevation, precision)
+	const coordinates = _.split(geoPoint, ' ');
+	if (coordinates.length >= 2 && _.every(coordinates, coord => coord && /^-?\d+(?:\.\d+)?$/.test(_.toString(coord)))) {
+		// NOTE: GeoJSON uses lon, lat, while most common systems use lat, lon order!
+		return _.concat([
+			_.toNumber(coordinates[1]),
+			_.toNumber(coordinates[0]),
+		], _.toNumber(coordinates[2]) ? _.toNumber(coordinates[2]) : []);
+	}
+	return null;
+}
+
+export function parseStringList(value: string): string[] {
+	return _.split(_.toString(value), /[\s,]+/);
+}
+
+const matchWildcard = (value: string, pattern: string): boolean => {
+	const regex = new RegExp(`^${_.escapeRegExp(pattern).replace('\\*', '.*')}$`);
+	return regex.test(value);
+};
+
+const formatValue = (value: any, format: string): any => { //tslint:disable-line:no-any
+	if (_.isString(value)) {
+		// Sanitize value
+		value = _.toString(value);
+
+		// Parse geoPoints
+		const geoPoint = parseGeoPoint(value);
+		if (geoPoint) {
+			return {
+				type: 'Point',
+				coordinates: geoPoint,
+			};
+		}
+
+		// Check if it's a closed polygon geo-shape: -1.954117 30.085159 0 0;-1.955005 30.084622 0 0;-1.956057 30.08506 0 0;-1.956393 30.086229 0 0;-1.955853 30.087143 0 0;-1.954609 30.08725 0 0;-1.953966 30.086735 0 0;-1.953805 30.085897 0 0;-1.954117 30.085159 0 0
+		const points = value.split(';');
+		if (points.length >= 2 && /^[-\d\.\s;]+$/.test(value)) {
+			// Using the GeoJSON format as per https://geojson.org/
+			const coordinates = _.compact(points.map(parseGeoPoint));
+			// Only return if all values are properly parsed
+			if (coordinates.length === points.length) {
+				return {
+					type: _.first(points) === _.last(points) ? 'Polygon' : 'LineString',  // check if shape is closed or open
+					coordinates,
+				};
+			}
+		}
+
+		// Parse numbers
+		if ('number' === format) {
+			return _.toNumber(value);
+		}
+
+		// Split multi-select
+		if ('multiSelect' === format) {
+			return _.split(_.toString(value), ' ');
+		}
+	}
+
+	return value;
+};
+
+export function formatSubmission(submission: IDataObject, selectMasks: string[] = [], numberMasks: string[] = []): IDataObject {
+	// Create a shallow copy of the submission
+	const response = {} as IDataObject;
+
+	for (const key of Object.keys(submission)) {
+		let value = _.clone(submission[key]);
+		// Sanitize key names: split by group, trim _
+		const sanitizedKey = key.split('/').map(k => _.trim(k, ' _')).join('.');
+		const leafKey = sanitizedKey.split('.').pop() || '';
+		let format = 'string';
+		if (_.some(numberMasks, mask => matchWildcard(leafKey, mask))) {
+			format = 'number';
+		}
+		if (_.some(selectMasks, mask => matchWildcard(leafKey, mask))) {
+			format = 'multiSelect';
+		}
+
+		value = formatValue(value, format);
+
+		_.set(response, sanitizedKey, value);
+	}
+
+	// Reformat _geolocation
+	if (_.isArray(response.geolocation) && response.geolocation.length === 2 && response.geolocation[0] && response.geolocation[1]) {
+		response.geolocation = {
+			type: 'Point',
+			coordinates: [response.geolocation[1], response.geolocation[0]],
+		};
+	}
+
+	return response;
+}
+
+export async function downloadAttachments(this: IExecuteFunctions | IWebhookFunctions, submission: IDataObject, options: IDataObject): Promise<INodeExecutionData> {
+	// Initialize return object with the original submission JSON content
+	const binaryItem: INodeExecutionData = {
+		json: {
+			...submission,
+		},
+		binary: {},
+	};
+
+	const credentials = await this.getCredentials('koBoToolboxApi') as IDataObject;
+
+	// Look for attachment links - there can be more than one
+	const attachmentList = (submission['_attachments'] || submission['attachments']) as any[];  // tslint:disable-line:no-any
+	if (attachmentList && attachmentList.length) {
+		for (const [index, attachment] of attachmentList.entries()) {
+			// look for the question name linked to this attachment
+			const filename = attachment.filename;
+			Object.keys(submission).forEach(question => {
+				if (filename.endsWith('/' + _.toString(submission[question]).replace(/\s/g, '_'))) {
+				}
+			});
+
+			// Download attachment
+			// NOTE: this needs to follow redirects (possibly across domains), while keeping Authorization headers
+			// The Axios client will not propagate the Authorization header on redirects (see https://github.com/axios/axios/issues/3607), so we need to follow ourselves...
+			let response = null;
+			const attachmentUrl = attachment[options.version as string] || attachment.download_url as string;
+			let final = false, redir = 0;
+
+			const axiosOptions: IHttpRequestOptions = {
+				url: attachmentUrl,
+				method: 'GET',
+				headers: {
+					'Authorization': `Token ${credentials.token}`,
+				},
+				ignoreHttpStatusErrors: true,
+				returnFullResponse: true,
+				disableFollowRedirect: true,
+				encoding: 'arraybuffer',
+			};
+
+			while (!final && redir < 5) {
+				response = await this.helpers.httpRequest(axiosOptions);
+
+				if (response && response.headers.location) {
+					// Follow redirect
+					axiosOptions.url = response.headers.location;
+					redir++;
+				} else {
+					final = true;
+				}
+			}
+
+			const dataPropertyAttachmentsPrefixName = options.dataPropertyAttachmentsPrefixName || 'attachment_';
+			const fileName = filename.split('/').pop();
+
+			if (response && response.body) {
+				binaryItem.binary![`${dataPropertyAttachmentsPrefixName}${index}`] = await this.helpers.prepareBinaryData(response.body, fileName);
+			}
+		}
+	} else {
+		delete binaryItem.binary;
+	}
+
+	// Add item to final output - even if there's no attachment retrieved
+	return binaryItem;
+}
+
+export async function loadForms(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
+	const responseData = await koBoToolboxApiRequest.call(this, {
+		url: '/api/v2/assets/',
+		qs: {
+			q: 'asset_type:survey',
+			ordering: 'name',
+		},
+		scroll: true,
+	});
+
+	return responseData?.map((survey: any) => ({ name: survey.name, value: survey.uid })) || [];  // tslint:disable-line:no-any
+}

packages/nodes-base/nodes/KoBoToolbox/HookDescription.ts

@@ -0,0 +1,184 @@
+import {
+	INodeProperties,
+} from 'n8n-workflow';
+
+export const hookOperations: INodeProperties[] = [
+	{
+		displayName: 'Operation',
+		name: 'operation',
+		type: 'options',
+		displayOptions: {
+			show: {
+				resource: [
+					'hook',
+				],
+			},
+		},
+		options: [
+			{
+				name: 'Get',
+				value: 'get',
+				description: 'Get a single hook definition',
+			},
+			{
+				name: 'Get All',
+				value: 'getAll',
+				description: 'List all hooks on a form',
+			},
+			{
+				name: 'Logs',
+				value: 'getLogs',
+				description: 'Get hook logs',
+			},
+			{
+				name: 'Retry All',
+				value: 'retryAll',
+				description: 'Retry all failed attempts for a given hook',
+			},
+			{
+				name: 'Retry One',
+				value: 'retryOne',
+				description: 'Retry a specific hook',
+			},
+		],
+		default: 'getAll',
+	},
+];
+
+export const hookFields: INodeProperties[] = [
+
+	/* -------------------------------------------------------------------------- */
+	/*                                hook:get                                    */
+	/* -------------------------------------------------------------------------- */
+	{
+		displayName: 'Form ID',
+		name: 'formId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'loadForms',
+		},
+		required: true,
+		default: '',
+		displayOptions: {
+			show: {
+				resource: [
+					'hook',
+				],
+				operation: [
+					'get',
+					'retryOne',
+					'retryAll',
+					'getLogs',
+				],
+			},
+		},
+		description: 'Form ID (e.g. aSAvYreNzVEkrWg5Gdcvg)',
+	},
+	{
+		displayName: 'Hook ID',
+		name: 'hookId',
+		type: 'string',
+		required: true,
+		displayOptions: {
+			show: {
+				resource: [
+					'hook',
+				],
+				operation: [
+					'get',
+					'retryOne',
+					'retryAll',
+					'getLogs',
+				],
+			},
+		},
+		default: '',
+		description: 'Hook ID (starts with h, e.g. hVehywQ2oXPYGHJHKtqth4)',
+	},
+	/* -------------------------------------------------------------------------- */
+	/*                                hook:getAll                                 */
+	/* -------------------------------------------------------------------------- */
+	{
+		displayName: 'Form ID',
+		name: 'formId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'loadForms',
+		},
+		required: true,
+		default: '',
+		displayOptions: {
+			show: {
+				resource: [
+					'hook',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+		description: 'Form ID (e.g. aSAvYreNzVEkrWg5Gdcvg)',
+	},
+	{
+		displayName: 'Hook Log ID',
+		name: 'logId',
+		type: 'string',
+		required: true,
+		displayOptions: {
+			show: {
+				resource: [
+					'hook',
+				],
+				operation: [
+					'retryOne',
+				],
+			},
+		},
+		default: '',
+		description: 'Hook log ID (starts with hl, e.g. hlSbGKaUKzTVNoWEVMYbLHe)',
+	},
+	{
+		displayName: 'Return All',
+		name: 'returnAll',
+		type: 'boolean',
+		required: true,
+		default: false,
+		displayOptions: {
+			show: {
+				resource: [
+					'hook',
+				],
+				operation: [
+					'getAll',
+					'getLogs',
+				],
+			},
+		},
+		description: 'Whether to return all results',
+	},
+	{
+		displayName: 'Limit',
+		name: 'limit',
+		type: 'number',
+		required: false,
+		typeOptions: {
+			maxValue: 3000,
+		},
+		displayOptions: {
+			show: {
+				resource: [
+					'hook',
+				],
+				operation: [
+					'getAll',
+					'getLogs',
+				],
+				returnAll: [
+					false,
+				],
+			},
+		},
+		default: 1000,
+		description: 'The number of results to return',
+	},
+];

packages/nodes-base/nodes/KoBoToolbox/KoBoToolbox.node.ts

@@ -0,0 +1,371 @@
+import {
+	IExecuteFunctions,
+} from 'n8n-core';
+
+import {
+	ICredentialsDecrypted,
+	ICredentialTestFunctions,
+	IDataObject,
+	INodeCredentialTestResult,
+	INodeExecutionData,
+	INodeType,
+	INodeTypeDescription,
+	JsonObject,
+} from 'n8n-workflow';
+
+import {
+	downloadAttachments,
+	formatSubmission,
+	koBoToolboxApiRequest,
+	loadForms,
+	parseStringList,
+} from './GenericFunctions';
+
+import {
+	formFields,
+	formOperations
+} from './FormDescription';
+
+import {
+	submissionFields,
+	submissionOperations,
+} from './SubmissionDescription';
+
+import {
+	hookFields,
+	hookOperations,
+} from './HookDescription';
+
+export class KoBoToolbox implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'KoBoToolbox',
+		name: 'koBoToolbox',
+		icon: 'file:koBoToolbox.svg',
+		group: ['transform'],
+		version: 1,
+		description: 'Work with KoBoToolbox forms and submissions',
+		subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
+		defaults: {
+			name: 'KoBoToolbox',
+			color: '#64C0FF',
+		},
+		inputs: ['main'],
+		outputs: ['main'],
+		credentials: [
+			{
+				name: 'koBoToolboxApi',
+				required: true,
+				testedBy: 'koBoToolboxApiCredentialTest',
+			},
+		],
+		properties: [
+			{
+				displayName: 'Resource',
+				name: 'resource',
+				type: 'options',
+				options: [
+					{
+						name: 'Form',
+						value: 'form',
+					},
+					{
+						name: 'Hook',
+						value: 'hook',
+					},
+					{
+						name: 'Submission',
+						value: 'submission',
+					},
+				],
+				default: 'submission',
+				required: true,
+			},
+			...formOperations,
+			...formFields,
+			...hookOperations,
+			...hookFields,
+			...submissionOperations,
+			...submissionFields,
+		],
+	};
+
+	methods = {
+		credentialTest: {
+			async koBoToolboxApiCredentialTest(this: ICredentialTestFunctions, credential: ICredentialsDecrypted): Promise<INodeCredentialTestResult> {
+				const credentials = credential.data;
+				try {
+					const response = await this.helpers.request({
+						url: `${credentials!.URL}/api/v2/assets/hash`,
+						headers: {
+							'Accept': 'application/json',
+							'Authorization': `Token ${credentials!.token}`,
+						},
+						json: true,
+					});
+
+					if (response.hash) {
+						return {
+							status: 'OK',
+							message: 'Connection successful!',
+						};
+					}
+					else {
+						return {
+							status: 'Error',
+							message: `Credentials are not valid. Response: ${response.detail}`,
+						};
+					}
+				}
+				catch (err) {
+					return {
+						status: 'Error',
+						message: `Credentials validation failed: ${(err as JsonObject).message}`,
+					};
+				}
+			},
+		},
+
+		loadOptions: {
+			loadForms,
+		},
+	};
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		// tslint:disable-next-line:no-any
+		let responseData: any;
+		// tslint:disable-next-line:no-any
+		let returnData: any[] = [];
+		const binaryItems: INodeExecutionData[] = [];
+		const items = this.getInputData();
+		const resource = this.getNodeParameter('resource', 0) as string;
+		const operation = this.getNodeParameter('operation', 0) as string;
+
+		for (let i = 0; i < items.length; i++) {
+
+			if (resource === 'form') {
+				// *********************************************************************
+				//                             Form
+				// *********************************************************************
+
+				if (operation === 'get') {
+					// ----------------------------------
+					//          Form: get
+					// ----------------------------------
+					const formId = this.getNodeParameter('formId', i) as string;
+					responseData = [await koBoToolboxApiRequest.call(this, {
+						url: `/api/v2/assets/${formId}`,
+					})];
+				}
+
+				if (operation === 'getAll') {
+					// ----------------------------------
+					//          Form: getAll
+					// ----------------------------------
+					const formQueryOptions = this.getNodeParameter('options', i) as {
+						sort: {
+							value: {
+								descending: boolean,
+								ordering: string,
+							}
+						}
+					};
+					const formFilterOptions = this.getNodeParameter('filters', i) as IDataObject;
+
+					responseData = await koBoToolboxApiRequest.call(this, {
+						url: '/api/v2/assets/',
+						qs: {
+							limit: this.getNodeParameter('limit', i, 1000) as number,
+							...(formFilterOptions.filter && { q: formFilterOptions.filter }),
+							...(formQueryOptions?.sort?.value?.ordering && { ordering: (formQueryOptions?.sort?.value?.descending ? '-' : '') + formQueryOptions?.sort?.value?.ordering }),
+						},
+						scroll: this.getNodeParameter('returnAll', i) as boolean,
+					});
+				}
+			}
+			if (resource === 'submission') {
+				// *********************************************************************
+				//                             Submissions
+				// *********************************************************************
+				const formId = this.getNodeParameter('formId', i) as string;
+
+				if (operation === 'getAll') {
+					// ----------------------------------
+					//          Submissions: getAll
+					// ----------------------------------
+
+					const submissionQueryOptions = this.getNodeParameter('options', i) as IDataObject;
+
+					responseData = await koBoToolboxApiRequest.call(this, {
+						url: `/api/v2/assets/${formId}/data/`,
+						qs: {
+							limit: this.getNodeParameter('limit', i, 1000) as number,
+							...(submissionQueryOptions.query && { query: submissionQueryOptions.query }),
+							//...(submissionQueryOptions.sort && { sort: submissionQueryOptions.sort }),
+							...(submissionQueryOptions.fields && { fields: JSON.stringify(parseStringList(submissionQueryOptions.fields as string)) }),
+						},
+						scroll: this.getNodeParameter('returnAll', i) as boolean,
+					});
+
+					if (submissionQueryOptions.reformat) {
+						responseData = responseData.map((submission: IDataObject) => {
+							return formatSubmission(submission, parseStringList(submissionQueryOptions.selectMask as string), parseStringList(submissionQueryOptions.numberMask as string));
+						});
+					}
+
+					if (submissionQueryOptions.download) {
+						// Download related attachments
+						for (const submission of responseData) {
+							binaryItems.push(await downloadAttachments.call(this, submission, submissionQueryOptions));
+						}
+					}
+				}
+
+				if (operation === 'get') {
+					// ----------------------------------
+					//          Submissions: get
+					// ----------------------------------
+					const submissionId = this.getNodeParameter('submissionId', i) as string;
+					const options = this.getNodeParameter('options', i) as IDataObject;
+
+					responseData = [await koBoToolboxApiRequest.call(this, {
+						url: `/api/v2/assets/${formId}/data/${submissionId}`,
+						qs: {
+							...(options.fields && { fields: JSON.stringify(parseStringList(options.fields as string)) }),
+						},
+					})];
+
+					if (options.reformat) {
+						responseData = responseData.map((submission: IDataObject) => {
+							return formatSubmission(submission, parseStringList(options.selectMask as string), parseStringList(options.numberMask as string));
+						});
+					}
+
+					if (options.download) {
+						// Download related attachments
+						for (const submission of responseData) {
+							binaryItems.push(await downloadAttachments.call(this, submission, options));
+						}
+					}
+				}
+
+				if (operation === 'delete') {
+					// ----------------------------------
+					//          Submissions: delete
+					// ----------------------------------
+					const id = this.getNodeParameter('submissionId', i) as string;
+
+					await koBoToolboxApiRequest.call(this, {
+						method: 'DELETE',
+						url: `/api/v2/assets/${formId}/data/${id}`,
+					});
+
+					responseData = [{
+						success: true,
+					}];
+				}
+
+				if (operation === 'getValidation') {
+					// ----------------------------------
+					//          Submissions: getValidation
+					// ----------------------------------
+					const submissionId = this.getNodeParameter('submissionId', i) as string;
+
+					responseData = [await koBoToolboxApiRequest.call(this, {
+						url: `/api/v2/assets/${formId}/data/${submissionId}/validation_status/`,
+					})];
+				}
+
+				if (operation === 'setValidation') {
+					// ----------------------------------
+					//          Submissions: setValidation
+					// ----------------------------------
+					const submissionId = this.getNodeParameter('submissionId', i) as string;
+					const status = this.getNodeParameter('validationStatus', i) as string;
+
+					responseData = [await koBoToolboxApiRequest.call(this, {
+						method: 'PATCH',
+						url: `/api/v2/assets/${formId}/data/${submissionId}/validation_status/`,
+						body: {
+							'validation_status.uid': status,
+						},
+					})];
+				}
+			}
+
+			if (resource === 'hook') {
+				const formId = this.getNodeParameter('formId', i) as string;
+				// *********************************************************************
+				//                             Hook
+				// *********************************************************************
+
+				if (operation === 'getAll') {
+					// ----------------------------------
+					//          Hook: getAll
+					// ----------------------------------
+					responseData = await koBoToolboxApiRequest.call(this, {
+						url: `/api/v2/assets/${formId}/hooks/`,
+						qs: {
+							limit: this.getNodeParameter('limit', i, 1000) as number,
+						},
+						scroll: this.getNodeParameter('returnAll', i) as boolean,
+					});
+				}
+
+				if (operation === 'get') {
+					// ----------------------------------
+					//          Hook: get
+					// ----------------------------------
+					const hookId = this.getNodeParameter('hookId', i) as string;
+					responseData = [await koBoToolboxApiRequest.call(this, {
+						url: `/api/v2/assets/${formId}/hooks/${hookId}`,
+					})];
+				}
+
+				if (operation === 'retryAll') {
+					// ----------------------------------
+					//          Hook: retryAll
+					// ----------------------------------
+					const hookId = this.getNodeParameter('hookId', i) as string;
+					responseData = [await koBoToolboxApiRequest.call(this, {
+						method: 'PATCH',
+						url: `/api/v2/assets/${formId}/hooks/${hookId}/retry/`,
+					})];
+				}
+
+				if (operation === 'getLogs') {
+					// ----------------------------------
+					//          Hook: getLogs
+					// ----------------------------------
+					const hookId = this.getNodeParameter('hookId', i) as string;
+					responseData = await koBoToolboxApiRequest.call(this, {
+						url: `/api/v2/assets/${formId}/hooks/${hookId}/logs/`,
+						qs: {
+							start: this.getNodeParameter('start', i, 0) as number,
+							limit: this.getNodeParameter('limit', i, 1000) as number,
+						},
+						scroll: this.getNodeParameter('returnAll', i) as boolean,
+					});
+				}
+
+				if (operation === 'retryOne') {
+					// ----------------------------------
+					//          Hook: retryOne
+					// ----------------------------------
+					const hookId = this.getNodeParameter('hookId', i) as string;
+					const logId = this.getNodeParameter('logId', i) as string;
+
+					responseData = [await koBoToolboxApiRequest.call(this, {
+						url: `/api/v2/assets/${formId}/hooks/${hookId}/logs/${logId}/retry/`,
+					})];
+				}
+			}
+
+			returnData = returnData.concat(responseData);
+		}
+
+		// Map data to n8n data
+		return binaryItems.length > 0
+			? [binaryItems]
+			: [this.helpers.returnJsonArray(returnData)];
+	}
+}

packages/nodes-base/nodes/KoBoToolbox/KoBoToolboxTrigger.node.ts

@@ -0,0 +1,168 @@
+import {
+	IDataObject,
+	IHookFunctions,
+	INodeType,
+	INodeTypeDescription,
+	IWebhookFunctions,
+	IWebhookResponseData,
+} from 'n8n-workflow';
+
+import {
+	downloadAttachments,
+	formatSubmission,
+	koBoToolboxApiRequest,
+	loadForms,
+	parseStringList
+} from './GenericFunctions';
+
+import {
+	 options,
+} from './Options';
+
+export class KoBoToolboxTrigger implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'KoBoToolbox Trigger',
+		name: 'koBoToolboxTrigger',
+		icon: 'file:koBoToolbox.svg',
+		group: ['trigger'],
+		version: 1,
+		description: 'Process KoBoToolbox submissions',
+		defaults: {
+			name: 'KoBoToolbox Trigger',
+			color: '#64C0FF',
+		},
+		inputs: [],
+		outputs: ['main'],
+		credentials: [
+			{
+				name: 'koBoToolboxApi',
+				required: true,
+			},
+		],
+		webhooks: [
+			{
+				name: 'default',
+				httpMethod: 'POST',
+				responseMode: 'onReceived',
+				path: 'webhook',
+			},
+		],
+		properties: [
+			{
+				displayName: 'Form Name/ID',
+				name: 'formId',
+				type: 'options',
+				typeOptions: {
+					loadOptionsMethod: 'loadForms',
+				},
+				required: true,
+				default: '',
+				description: 'Form ID (e.g. aSAvYreNzVEkrWg5Gdcvg)',
+			},
+			{
+				displayName: 'Trigger On',
+				name: 'triggerOn',
+				type: 'options',
+				required: true,
+				default: 'formSubmission',
+				options: [
+					{
+						name: 'On Form Submission',
+						value: 'formSubmission',
+					},
+				],
+			},
+			{ ...options },
+		],
+	};
+
+	// @ts-ignore
+	webhookMethods = {
+		default: {
+			async checkExists(this: IHookFunctions): Promise<boolean> {
+				const webhookData = this.getWorkflowStaticData('node');
+				const webhookUrl = this.getNodeWebhookUrl('default');
+				const formId = this.getNodeParameter('formId') as string; //tslint:disable-line:variable-name
+				const webhooks = await koBoToolboxApiRequest.call(this, {
+					url: `/api/v2/assets/${formId}/hooks/`,
+				});
+				for (const webhook of webhooks || []) {
+					if (webhook.endpoint === webhookUrl && webhook.active === true) {
+						webhookData.webhookId = webhook.uid;
+						return true;
+					}
+				}
+				return false;
+			},
+
+			async create(this: IHookFunctions): Promise<boolean> {
+				const webhookData = this.getWorkflowStaticData('node');
+				const webhookUrl = this.getNodeWebhookUrl('default');
+				const formId = this.getNodeParameter('formId') as string; //tslint:disable-line:variable-name
+
+				const response = await koBoToolboxApiRequest.call(this, {
+					method: 'POST',
+					url: `/api/v2/assets/${formId}/hooks/`,
+					body: {
+						name: `n8n-webhook:${webhookUrl}`,
+						endpoint: webhookUrl,
+						email_notification: true,
+					},
+				});
+
+				if (response.uid) {
+					webhookData.webhookId = response.uid;
+					return true;
+				}
+
+				return false;
+			},
+
+			async delete(this: IHookFunctions): Promise<boolean> {
+				const webhookData = this.getWorkflowStaticData('node');
+				const formId = this.getNodeParameter('formId') as string; //tslint:disable-line:variable-name
+				try {
+					await koBoToolboxApiRequest.call(this, {
+						method: 'DELETE',
+						url: `/api/v2/assets/${formId}/hooks/${webhookData.webhookId}`,
+					});
+				} catch (error) {
+					return false;
+				}
+				delete webhookData.webhookId;
+				return true;
+			},
+		},
+	};
+
+	methods = {
+		loadOptions: {
+			loadForms,
+		},
+	};
+
+	async webhook(this: IWebhookFunctions): Promise<IWebhookResponseData> {
+		const req = this.getRequestObject();
+		const formatOptions = this.getNodeParameter('formatOptions') as IDataObject;
+
+		const responseData = formatOptions.reformat
+			? formatSubmission(req.body, parseStringList(formatOptions.selectMask as string), parseStringList(formatOptions.numberMask as string))
+			: req.body;
+
+		if (formatOptions.download) {
+			// Download related attachments
+			return {
+				workflowData: [
+					[await downloadAttachments.call(this, responseData, formatOptions)],
+				],
+			};
+		}
+		else {
+			return {
+				workflowData: [
+					this.helpers.returnJsonArray([responseData]),
+				],
+			};
+		}
+	}
+}

packages/nodes-base/nodes/KoBoToolbox/Options.ts

@@ -0,0 +1,87 @@
+import {
+	INodeProperties,
+} from 'n8n-workflow';
+
+export const options = {
+	displayName: 'Options',
+	placeholder: 'Add Option',
+	name: 'formatOptions',
+	type: 'collection',
+	default: {},
+	options: [
+		{
+			displayName: 'Attachments Prefix',
+			name: 'dataPropertyAttachmentsPrefixName',
+			type: 'string',
+			displayOptions: {
+				show: {
+					download: [
+						true,
+					],
+				},
+			},
+			default: 'attachment_',
+			description: 'Prefix for name of the binary property to which to write the attachments. An index starting with 0 will be added. So if name is "attachment_" the first attachment is saved to "attachment_0"',
+		},
+		{
+			displayName: 'Download Attachments',
+			name: 'download',
+			type: 'boolean',
+			default: false,
+			description: 'Download submitted attachments',
+		},
+		{
+			displayName: 'File Size',
+			name: 'version',
+			type: 'options',
+			displayOptions: {
+				show: {
+					download: [
+						true,
+					],
+				},
+			},
+			default: 'download_url',
+			description: 'Attachment size to retrieve, if multiple versions are available',
+			options: [
+				{
+					name: 'Original',
+					value: 'download_url',
+				},
+				{
+					name: 'Small',
+					value: 'download_small_url',
+				},
+				{
+					name: 'Medium',
+					value: 'download_medium_url',
+				},
+				{
+					name: 'Large',
+					value: 'download_large_url',
+				},
+			],
+		},
+		{
+			displayName: 'Multiselect Mask',
+			name: 'selectMask',
+			type: 'string',
+			default: 'select_*',
+			description: 'Comma-separated list of wildcard-style selectors for fields that should be treated as multiselect fields, i.e. parsed as arrays',
+		},
+		{
+			displayName: 'Number Mask',
+			name: 'numberMask',
+			type: 'string',
+			default: 'n_*, f_*',
+			description: 'Comma-separated list of wildcard-style selectors for fields that should be treated as numbers',
+		},
+		{
+			displayName: 'Reformat',
+			name: 'reformat',
+			type: 'boolean',
+			default: false,
+			description: 'Apply some reformatting to the submission data, such as parsing GeoJSON coordinates',
+		},
+	],
+} as INodeProperties;

packages/nodes-base/nodes/KoBoToolbox/SubmissionDescription.ts

@@ -0,0 +1,304 @@
+import {
+	INodeProperties,
+} from 'n8n-workflow';
+
+export const submissionOperations: INodeProperties[] = [
+	{
+		displayName: 'Operation',
+		name: 'operation',
+		type: 'options',
+		displayOptions: {
+			show: {
+				resource: [
+					'submission',
+				],
+			},
+		},
+		options: [
+			{
+				name: 'Delete',
+				value: 'delete',
+				description: 'Delete a single submission',
+			},
+			{
+				name: 'Get',
+				value: 'get',
+				description: 'Get a single submission',
+			},
+			{
+				name: 'Get All',
+				value: 'getAll',
+				description: 'Get all submissions',
+			},
+			{
+				name: 'Get Validation Status',
+				value: 'getValidation',
+				description: 'Get the validation status for the submission',
+			},
+			{
+				name: 'Update Validation Status',
+				value: 'setValidation',
+				description: 'Set the validation status of the submission',
+			},
+		],
+		default: 'getAll',
+	},
+];
+
+export const submissionFields: INodeProperties[] = [
+
+	/* -------------------------------------------------------------------------- */
+	/*                                submission:get                              */
+	/* -------------------------------------------------------------------------- */
+	{
+		displayName: 'Form ID',
+		name: 'formId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'loadForms',
+		},
+		required: true,
+		default: '',
+		displayOptions: {
+			show: {
+				resource: [
+					'submission',
+				],
+				operation: [
+					'get',
+					'delete',
+					'getValidation',
+					'setValidation',
+				],
+			},
+		},
+		description: 'Form ID (e.g. aSAvYreNzVEkrWg5Gdcvg)',
+	},
+	{
+		displayName: 'Submission ID',
+		name: 'submissionId',
+		type: 'string',
+		required: true,
+		default: '',
+		displayOptions: {
+			show: {
+				resource: [
+					'submission',
+				],
+				operation: [
+					'get',
+					'delete',
+					'getValidation',
+					'setValidation',
+				],
+			},
+		},
+		description: 'Submission ID (number, e.g. 245128)',
+	},
+	{
+		displayName: 'Validation Status',
+		name: 'validationStatus',
+		type: 'options',
+		required: true,
+		displayOptions: {
+			show: {
+				resource: [
+					'submission',
+				],
+				operation: [
+					'setValidation',
+				],
+			},
+		},
+		default: '',
+		options: [
+			{
+				name: 'Approved',
+				value: 'validation_status_approved',
+			},
+			{
+				name: 'Not Approved',
+				value: 'validation_status_not_approved',
+			},
+			{
+				name: 'On Hold',
+				value: 'validation_status_on_hold',
+			},
+		],
+		description: 'Desired Validation Status',
+	},
+	/* -------------------------------------------------------------------------- */
+	/*                                submission:getAll                           */
+	/* -------------------------------------------------------------------------- */
+	{
+		displayName: 'Form Name/ID',
+		name: 'formId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'loadForms',
+		},
+		required: true,
+		default: '',
+		displayOptions: {
+			show: {
+				resource: [
+					'submission',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+		description: 'Form ID (e.g. aSAvYreNzVEkrWg5Gdcvg)',
+	},
+	{
+		displayName: 'Return All',
+		name: 'returnAll',
+		type: 'boolean',
+		required: true,
+		default: false,
+		displayOptions: {
+			show: {
+				resource: [
+					'submission',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+		description: 'Whether to return all results',
+	},
+	{
+		displayName: 'Limit',
+		name: 'limit',
+		type: 'number',
+		required: false,
+		typeOptions: {
+			maxValue: 3000,
+		},
+		displayOptions: {
+			show: {
+				resource: [
+					'submission',
+				],
+				operation: [
+					'getAll',
+				],
+				returnAll: [
+					false,
+				],
+			},
+		},
+		default: 100,
+		description: 'The number of results to return',
+	},
+	{
+		displayName: 'Options',
+		name: 'options',
+		type: 'collection',
+		displayOptions: {
+			show: {
+				resource: [
+					'submission',
+				],
+				operation: [
+					'get',
+					'getAll',
+				],
+			},
+		},
+		default: {},
+		placeholder: 'Add Option',
+		options: [
+			{
+				displayName: 'Attachments Prefix',
+				name: 'dataPropertyAttachmentsPrefixName',
+				type: 'string',
+				displayOptions: {
+					show: {
+						download: [
+							true,
+						],
+					},
+				},
+				default: 'attachment_',
+				description: 'Prefix for name of the binary property to which to write the attachments. An index starting with 0 will be added. So if name is "attachment_" the first attachment is saved to "attachment_0"',
+			},
+			{
+				displayName: 'Download Attachments',
+				name: 'download',
+				type: 'boolean',
+				default: false,
+				description: 'Download submitted attachments',
+			},
+			{
+				displayName: 'Fields to Retrieve',
+				name: 'fields',
+				type: 'string',
+				default: '',
+				description: 'Comma-separated list of fields to retrieve (e.g. _submission_time,_submitted_by). If left blank, all fields are retrieved',
+			},
+			{
+				displayName: 'File Size',
+				name: 'version',
+				type: 'options',
+				displayOptions: {
+					show: {
+						download: [
+							true,
+						],
+					},
+				},
+				default: 'download_url',
+				description: 'Attachment size to retrieve, if multiple versions are available',
+				options: [
+					{
+						name: 'Original',
+						value: 'download_url',
+					},
+					{
+						name: 'Small',
+						value: 'download_small_url',
+					},
+					{
+						name: 'Medium',
+						value: 'download_medium_url',
+					},
+					{
+						name: 'Large',
+						value: 'download_large_url',
+					},
+				],
+			},
+			{
+				displayName: 'Multiselect Mask',
+				name: 'selectMask',
+				type: 'string',
+				default: 'select_*',
+				description: 'Comma-separated list of wildcard-style selectors for fields that should be treated as multiselect fields, i.e. parsed as arrays',
+			},
+			{
+				displayName: 'Number Mask',
+				name: 'numberMask',
+				type: 'string',
+				default: 'n_*, f_*',
+				description: 'Comma-separated list of wildcard-style selectors for fields that should be treated as numbers',
+			},
+			{
+				displayName: 'Reformat',
+				name: 'reformat',
+				type: 'boolean',
+				default: false,
+				description: 'Apply some reformatting to the submission data, such as parsing GeoJSON coordinates',
+			},
+			// {
+			// 	displayName: 'Sort',
+			// 	name: 'sort',
+			// 	type: 'json',
+			// 	default: '',
+			// 	description: 'Sort predicates, in Mongo JSON format (e.g. {"_submission_time":1})',
+			// },
+		],
+	},
+];

packages/nodes-base/nodes/KoBoToolbox/koBoToolbox.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="23.17 23.5 19.18 28.6"><style>.st79{fill:#64c0ff}</style><g id="Layer_4"><path class="st79" d="M38.6 42.8v2.8c0 1.6-1.3 2.8-2.8 2.8h-6.1c-1.6 0-2.8-1.3-2.8-2.8V30c0-1.6 1.3-2.8 2.8-2.8h6.1c1.6 0 2.8 1.3 2.8 2.8v.4c.5-.2 1.1-.3 1.7-.3.7 0 1.4.1 2 .4V30c0-3.6-2.9-6.5-6.5-6.5h-6.1c-3.6 0-6.5 2.9-6.5 6.5v15.6c0 3.6 2.9 6.5 6.5 6.5h6.1c3.6 0 6.5-2.9 6.5-6.5v-7.1l-3.7 4.3z"/><path class="st79" d="M35.6 41.9l6.6-7.6c.2-.2.2-.6-.1-.8-1.2-1-2.9-.9-3.9.3l-4.1 4.7c-.1.1-.3.1-.3 0l-1.5-1.9c-.3-.4-.8-.4-1.2 0-1 1-1 2.6-.1 3.7l1.2 1.5c.8 1.2 2.5 1.2 3.4.1z"/></g></svg>