diff --git a/packages/nodes-base/credentials/UrlScanIoApi.credentials.ts b/packages/nodes-base/credentials/UrlScanIoApi.credentials.ts new file mode 100644 index 0000000000..df29c66cf8 --- /dev/null +++ b/packages/nodes-base/credentials/UrlScanIoApi.credentials.ts @@ -0,0 +1,19 @@ +import { + ICredentialType, + INodeProperties, +} from 'n8n-workflow'; + +export class UrlScanIoApi implements ICredentialType { + name = 'urlScanIoApi'; + displayName = 'urlscan.io API'; + documentationUrl = 'urlScanIo'; + properties: INodeProperties[] = [ + { + displayName: 'API Key', + name: 'apiKey', + type: 'string', + default: '', + required: true, + }, + ]; +} diff --git a/packages/nodes-base/nodes/UrlScanIo/GenericFunctions.ts b/packages/nodes-base/nodes/UrlScanIo/GenericFunctions.ts new file mode 100644 index 0000000000..8a3d5deaf6 --- /dev/null +++ b/packages/nodes-base/nodes/UrlScanIo/GenericFunctions.ts @@ -0,0 +1,85 @@ +import { + OptionsWithUri, +} from 'request'; + +import { + IExecuteFunctions, +} from 'n8n-core'; + +import { + IDataObject, + NodeApiError, +} from 'n8n-workflow'; + +export async function urlScanIoApiRequest( + this: IExecuteFunctions, + method: 'GET' | 'POST', + endpoint: string, + body: IDataObject = {}, + qs: IDataObject = {}, +) { + const { apiKey } = await this.getCredentials('urlScanIoApi') as { apiKey: string }; + + const options: OptionsWithUri = { + headers: { + 'API-KEY': apiKey, + }, + method, + body, + qs, + uri: `https://urlscan.io/api/v1${endpoint}`, + json: true, + }; + + if (!Object.keys(body).length) { + delete options.body; + } + + if (!Object.keys(qs).length) { + delete options.qs; + } + + try { + return await this.helpers.request(options); + } catch (error) { + throw new NodeApiError(this.getNode(), error); + } +} + + +export async function handleListing( + this: IExecuteFunctions, + endpoint: string, + qs: IDataObject = {}, +): Promise { + const returnData: IDataObject[] = []; + let responseData; + + qs.size = 100; + + const returnAll = this.getNodeParameter('returnAll', 0, false) as boolean; + const limit = this.getNodeParameter('limit', 0, 0) as number; + + do { + responseData = await urlScanIoApiRequest.call(this, 'GET', endpoint, {}, qs); + returnData.push(...responseData.results); + + if (!returnAll && returnData.length > limit) { + return returnData.slice(0, limit); + } + + if (responseData.results.length) { + const lastResult = responseData.results[responseData.results.length -1]; + qs.search_after = lastResult.sort; + } + + } while (responseData.total > returnData.length); + + return returnData; +} + +export const normalizeId = ({ _id, uuid, ...rest }: IDataObject) => { + if (_id) return ({ scanId: _id, ...rest }); + if (uuid) return ({ scanId: uuid, ...rest }); + return rest; +}; diff --git a/packages/nodes-base/nodes/UrlScanIo/UrlScanIo.node.ts b/packages/nodes-base/nodes/UrlScanIo/UrlScanIo.node.ts new file mode 100644 index 0000000000..cc6d956e9f --- /dev/null +++ b/packages/nodes-base/nodes/UrlScanIo/UrlScanIo.node.ts @@ -0,0 +1,212 @@ +import { + IExecuteFunctions, +} from 'n8n-core'; + +import { + ICredentialsDecrypted, + ICredentialTestFunctions, + IDataObject, + INodeExecutionData, + INodeType, + INodeTypeDescription, + NodeCredentialTestResult, + NodeOperationError, +} from 'n8n-workflow'; + +import { + OptionsWithUri, +} from 'request'; + +import { + scanFields, + scanOperations, +} from './descriptions'; + +import { + handleListing, + normalizeId, + urlScanIoApiRequest, +} from './GenericFunctions'; + +export class UrlScanIo implements INodeType { + description: INodeTypeDescription = { + displayName: 'urlscan.io', + name: 'urlScanIo', + icon: 'file:urlScanIo.svg', + group: ['transform'], + version: 1, + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Consume the urlscan.io API', + defaults: { + name: 'urlscan.io', + color: '#f3d337', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'urlScanIoApi', + required: true, + testedBy: 'urlScanIoApiTest', + }, + ], + properties: [ + { + displayName: 'Resource', + name: 'resource', + noDataExpression: true, + type: 'options', + options: [ + { + name: 'Scan', + value: 'scan', + }, + ], + default: 'scan', + }, + ...scanOperations, + ...scanFields, + ], + }; + + methods = { + credentialTest: { + async urlScanIoApiTest( + this: ICredentialTestFunctions, + credentials: ICredentialsDecrypted, + ): Promise { + const { apiKey } = credentials.data as { apiKey: string }; + + const options: OptionsWithUri = { + headers: { + 'API-KEY': apiKey, + }, + method: 'GET', + uri: 'https://urlscan.io/user/quotas', + json: true, + }; + + try { + await this.helpers.request(options); + return { + status: 'OK', + message: 'Authentication successful', + }; + } catch (error) { + return { + status: 'Error', + message: error.message, + }; + } + }, + }, + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const returnData: IDataObject[] = []; + + const resource = this.getNodeParameter('resource', 0) as 'scan'; + const operation = this.getNodeParameter('operation', 0) as 'perform' | 'get' | 'getAll'; + + let responseData; + + for (let i = 0; i < items.length; i++) { + + try { + + if (resource === 'scan') { + + // ********************************************************************** + // scan + // ********************************************************************** + + if (operation === 'get') { + + // ---------------------------------------- + // scan: get + // ---------------------------------------- + + const scanId = this.getNodeParameter('scanId', i) as string; + responseData = await urlScanIoApiRequest.call(this, 'GET', `/result/${scanId}`); + + } else if (operation === 'getAll') { + + // ---------------------------------------- + // scan: getAll + // ---------------------------------------- + + // https://urlscan.io/docs/search + + const filters = this.getNodeParameter('filters', i) as { query?: string }; + + const qs: IDataObject = {}; + + if (filters?.query) { + qs.q = filters.query; + } + + responseData = await handleListing.call(this, '/search', qs); + responseData = responseData.map(normalizeId); + + } else if (operation === 'perform') { + + // ---------------------------------------- + // scan: perform + // ---------------------------------------- + + // https://urlscan.io/docs/search + + const { + tags: rawTags, + ...rest + } = this.getNodeParameter('additionalFields', i) as { + customAgent?: string; + visibility?: 'public' | 'private' | 'unlisted'; + tags?: string; + referer?: string; + overrideSafety: string; + }; + + const body: IDataObject = { + url: this.getNodeParameter('url', i) as string, + ...rest, + }; + + if (rawTags) { + const tags = rawTags.split(',').map(tag => tag.trim()); + + if (tags.length > 10) { + throw new NodeOperationError( + this.getNode(), + 'Please enter at most 10 tags', + ); + } + + body.tags = tags; + } + + responseData = await urlScanIoApiRequest.call(this, 'POST', '/scan', body); + responseData = normalizeId(responseData); + + } + + } + + Array.isArray(responseData) + ? returnData.push(...responseData) + : returnData.push(responseData); + + } catch (error) { + if (this.continueOnFail()) { + returnData.push({ error: error.message }); + continue; + } + throw error; + } + + } + + return [this.helpers.returnJsonArray(returnData)]; + } +} diff --git a/packages/nodes-base/nodes/UrlScanIo/descriptions/ScanDescription.ts b/packages/nodes-base/nodes/UrlScanIo/descriptions/ScanDescription.ts new file mode 100644 index 0000000000..e8dde789e1 --- /dev/null +++ b/packages/nodes-base/nodes/UrlScanIo/descriptions/ScanDescription.ts @@ -0,0 +1,218 @@ +import { + INodeProperties, +} from 'n8n-workflow'; + +export const scanOperations: INodeProperties[] = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + displayOptions: { + show: { + resource: [ + 'scan', + ], + }, + }, + options: [ + { + name: 'Get', + value: 'get', + }, + { + name: 'Get All', + value: 'getAll', + }, + { + name: 'Perform', + value: 'perform', + }, + ], + default: 'perform', + }, +]; + +export const scanFields: INodeProperties[] = [ + // ---------------------------------------- + // scan: get + // ---------------------------------------- + { + displayName: 'Scan ID', + name: 'scanId', + type: 'string', + default: '', + description: 'ID of the scan to retrieve', + displayOptions: { + show: { + resource: [ + 'scan', + ], + operation: [ + 'get', + ], + }, + }, + }, + + // ---------------------------------------- + // scan: getAll + // ---------------------------------------- + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + default: false, + description: 'Whether to return all results or only up to a given limit', + displayOptions: { + show: { + resource: [ + 'scan', + ], + operation: [ + 'getAll', + ], + }, + }, + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + default: 50, + description: 'Max number of results to return', + typeOptions: { + minValue: 1, + }, + displayOptions: { + show: { + resource: [ + 'scan', + ], + operation: [ + 'getAll', + ], + returnAll: [ + false, + ], + }, + }, + }, + { + displayName: 'Filters', + name: 'filters', + type: 'collection', + placeholder: 'Add Filter', + default: {}, + displayOptions: { + show: { + resource: [ + 'scan', + ], + operation: [ + 'getAll', + ], + }, + }, + options: [ + { + displayName: 'Query', + name: 'query', + type: 'string', + description: 'Query using the Elastic Search Query String syntax. See supported fields in the documentation.', + default: '', + placeholder: 'domain:n8n.io', + }, + ], + }, + + // ---------------------------------------- + // scan: perform + // ---------------------------------------- + { + displayName: 'URL', + name: 'url', + type: 'string', + default: '', + placeholder: 'https://n8n.io', + description: 'URL to scan', + displayOptions: { + show: { + resource: [ + 'scan', + ], + operation: [ + 'perform', + ], + }, + }, + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: [ + 'scan', + ], + operation: [ + 'perform', + ], + }, + }, + options: [ + { + displayName: 'Custom Agent', + name: 'customAgent', + description: 'User-Agent header to set for this scan. Defaults to n8n', + type: 'string', + default: '', + }, + { + displayName: 'Override Safety', + name: 'overrideSafety', + description: 'Disable reclassification of URLs with potential PII in them', + type: 'string', + default: '', + }, + { + displayName: 'Referer', + name: 'referer', + description: 'HTTP referer to set for this scan', + type: 'string', + default: '', + }, + { + displayName: 'Tags', + name: 'tags', + description: 'Comma-separated list of user-defined tags to add to this scan. Limited to 10 tags.', + placeholder: 'phishing, malicious', + type: 'string', + default: '', + }, + { + displayName: 'Visibility', + name: 'visibility', + type: 'options', + default: 'private', + options: [ + { + name: 'Private', + value: 'private', + }, + { + name: 'Public', + value: 'public', + }, + { + name: 'Unlisted', + value: 'unlisted', + }, + ], + }, + ], + }, +]; diff --git a/packages/nodes-base/nodes/UrlScanIo/descriptions/index.ts b/packages/nodes-base/nodes/UrlScanIo/descriptions/index.ts new file mode 100644 index 0000000000..ee9d97583b --- /dev/null +++ b/packages/nodes-base/nodes/UrlScanIo/descriptions/index.ts @@ -0,0 +1 @@ +export * from './ScanDescription'; diff --git a/packages/nodes-base/nodes/UrlScanIo/urlScanIo.svg b/packages/nodes-base/nodes/UrlScanIo/urlScanIo.svg new file mode 100644 index 0000000000..e361a718c5 --- /dev/null +++ b/packages/nodes-base/nodes/UrlScanIo/urlScanIo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index fc00fea912..ba3551c9c1 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -286,6 +286,7 @@ "dist/credentials/UpleadApi.credentials.js", "dist/credentials/UProcApi.credentials.js", "dist/credentials/UptimeRobotApi.credentials.js", + "dist/credentials/UrlScanIoApi.credentials.js", "dist/credentials/VeroApi.credentials.js", "dist/credentials/VonageApi.credentials.js", "dist/credentials/WebflowApi.credentials.js", @@ -606,6 +607,7 @@ "dist/nodes/Uplead/Uplead.node.js", "dist/nodes/UProc/UProc.node.js", "dist/nodes/UptimeRobot/UptimeRobot.node.js", + "dist/nodes/UrlScanIo/UrlScanIo.node.js", "dist/nodes/Vero/Vero.node.js", "dist/nodes/Vonage/Vonage.node.js", "dist/nodes/Wait.node.js",