feat(HTTP Request Node): Add pagination support (#5993)

Is still WIP and does not implement the correct UI yet.

Github issue / Community forum post (link here to close automatically):
https://community.n8n.io/t/pagination-included-into-http-node/15080

https://community.n8n.io/t/how-to-paginate-through-data-in-http-requests/28103
This commit is contained in:
Jan Oberhauser
2023-11-01 14:24:43 +01:00
committed by GitHub
parent 9bdb85c4ce
commit cc2bd2e19c
7 changed files with 2408 additions and 238 deletions

View File

@@ -9,6 +9,7 @@ import type {
INodeTypeBaseDescription,
INodeTypeDescription,
IRequestOptionsSimplified,
PaginationOptions,
JsonObject,
} from 'n8n-workflow';
@@ -927,6 +928,228 @@ export class HttpRequestV3 implements INodeType {
},
],
},
{
displayName: 'Pagination',
name: 'pagination',
placeholder: 'Add pagination',
type: 'fixedCollection',
typeOptions: {
multipleValues: false,
},
default: {
pagination: {},
},
options: [
{
displayName: 'Pagination',
name: 'pagination',
values: [
{
displayName: 'Pagination Mode',
name: 'paginationMode',
type: 'options',
typeOptions: {
noDataExpression: true,
},
options: [
{
name: 'Off',
value: 'off',
},
{
name: 'Update a Parameter in Each Request',
value: 'updateAParameterInEachRequest',
},
{
name: 'Response Contains Next URL',
value: 'responseContainsNextURL',
},
],
default: 'updateAParameterInEachRequest',
description: 'If pagination should be used',
},
{
displayName:
'Use the $response variables to access the data of the previous response. <a href="https://docs.n8n.io/code/builtin/http-node-variables/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.httpRequest" target="_blank">More info</a>',
name: 'webhookNotice',
displayOptions: {
hide: {
paginationMode: ['off'],
},
},
type: 'notice',
default: '',
},
{
displayName: 'Next URL',
name: 'nextURL',
type: 'string',
displayOptions: {
show: {
paginationMode: ['responseContainsNextURL'],
},
},
default: '',
description:
'Should evaluate to true when pagination is complete. More info.',
},
{
displayName: 'Parameters',
name: 'parameters',
type: 'fixedCollection',
displayOptions: {
show: {
paginationMode: ['updateAParameterInEachRequest'],
},
},
typeOptions: {
multipleValues: true,
noExpression: true,
},
placeholder: 'Add Parameter',
default: {
parameters: [
{
type: 'qs',
name: '',
value: '',
},
],
},
options: [
{
name: 'parameters',
displayName: 'Parameter',
values: [
{
displayName: 'Type',
name: 'type',
type: 'options',
options: [
{
name: 'Body',
value: 'body',
},
{
name: 'Header',
value: 'headers',
},
{
name: 'Query',
value: 'qs',
},
],
default: 'qs',
description: 'Where the parameter should be set',
},
{
displayName: 'Name',
name: 'name',
type: 'string',
default: '',
},
{
displayName: 'Value',
name: 'value',
type: 'string',
default: '',
},
],
},
],
},
{
displayName: 'Pagination Complete When',
name: 'paginationCompleteWhen',
type: 'options',
typeOptions: {
noDataExpression: true,
},
displayOptions: {
hide: {
paginationMode: ['off'],
},
},
options: [
{
name: 'Response Is Empty',
value: 'responseIsEmpty',
},
{
name: 'Receive Specific Status Code(s)',
value: 'receiveSpecificStatusCodes',
},
{
name: 'Other',
value: 'other',
},
],
default: 'responseIsEmpty',
description: 'When should no further requests be made?',
},
{
displayName: 'Status Code(s) when Complete',
name: 'statusCodesWhenComplete',
type: 'string',
typeOptions: {
noDataExpression: true,
},
displayOptions: {
show: {
paginationCompleteWhen: ['receiveSpecificStatusCodes'],
},
},
default: '',
description: 'Accepts comma-separated values',
},
{
displayName: 'Complete Expression',
name: 'completeExpression',
type: 'string',
displayOptions: {
show: {
paginationCompleteWhen: ['other'],
},
},
default: '',
description:
'Should evaluate to true when pagination is complete. More info.',
},
{
displayName: 'Limit Pages Fetched',
name: 'limitPagesFetched',
type: 'boolean',
typeOptions: {
noDataExpression: true,
},
displayOptions: {
hide: {
paginationMode: ['off'],
},
},
default: false,
noDataExpression: true,
description: 'Whether the number of requests should be limited',
},
{
displayName: 'Max Pages',
name: 'maxRequests',
type: 'number',
typeOptions: {
noDataExpression: true,
},
displayOptions: {
show: {
limitPagesFetched: [true],
},
},
default: 100,
description: 'Maximum amount of request to be make',
},
],
},
],
},
{
displayName: 'Proxy',
name: 'proxy',
@@ -1033,6 +1256,26 @@ export class HttpRequestV3 implements INodeType {
let autoDetectResponseFormat = false;
// Can not be defined on a per item level
const pagination = this.getNodeParameter('options.pagination.pagination', 0, null, {
rawExpressions: true,
}) as {
paginationMode: 'off' | 'updateAParameterInEachRequest' | 'responseContainsNextURL';
nextURL?: string;
parameters: {
parameters: Array<{
type: 'body' | 'headers' | 'qs';
name: string;
value: string;
}>;
};
paginationCompleteWhen: 'responseIsEmpty' | 'receiveSpecificStatusCodes' | 'other';
statusCodesWhenComplete: string;
completeExpression: string;
limitPagesFetched: boolean;
maxRequests: number;
};
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const requestMethod = this.getNodeParameter('method', itemIndex) as string;
@@ -1117,15 +1360,9 @@ export class HttpRequestV3 implements INodeType {
gzip: true,
rejectUnauthorized: !allowUnauthorizedCerts || false,
followRedirect: false,
resolveWithFullResponse: true,
};
// When response format is set to auto-detect,
// we need to access to response header content-type
// and the only way is using "resolveWithFullResponse"
if (autoDetectResponseFormat || fullResponse) {
requestOptions.resolveWithFullResponse = true;
}
if (requestOptions.method !== 'GET' && nodeVersion >= 4.1) {
requestOptions = { ...requestOptions, followAllRedirects: false };
}
@@ -1321,7 +1558,7 @@ export class HttpRequestV3 implements INodeType {
requestOptions.json = true;
}
// // Add Content Type if any are set
// Add Content Type if any are set
if (bodyContentType === 'raw') {
if (requestOptions.headers === undefined) {
requestOptions.headers = {};
@@ -1392,7 +1629,64 @@ export class HttpRequestV3 implements INodeType {
try {
this.sendMessageToUI(sanitizeUiMessage(requestOptions, authDataKeys));
} catch (e) {}
if (authentication === 'genericCredentialType' || authentication === 'none') {
if (pagination && pagination.paginationMode !== 'off') {
let continueExpression = '={{false}}';
if (pagination.paginationCompleteWhen === 'receiveSpecificStatusCodes') {
// Split out comma separated list of status codes into array
const statusCodesWhenCompleted = pagination.statusCodesWhenComplete
.split(',')
.map((item) => parseInt(item.trim()));
continueExpression = `={{ !${JSON.stringify(
statusCodesWhenCompleted,
)}.includes($response.statusCode) }}`;
} else if (pagination.paginationCompleteWhen === 'responseIsEmpty') {
continueExpression =
'={{ Array.isArray($response.body) ? $response.body.length : !!$response.body }}';
} else {
// Other
if (!pagination.completeExpression.length || pagination.completeExpression[0] !== '=') {
throw new NodeOperationError(this.getNode(), 'Invalid or empty Complete Expression');
}
continueExpression = `={{ !(${pagination.completeExpression.trim().slice(3, -2)}) }}`;
}
const paginationData: PaginationOptions = {
continue: continueExpression,
request: {},
};
if (pagination.paginationMode === 'updateAParameterInEachRequest') {
// Iterate over all parameters and add them to the request
paginationData.request = {};
pagination.parameters.parameters.forEach((parameter) => {
if (!paginationData.request[parameter.type]) {
paginationData.request[parameter.type] = {};
}
paginationData.request[parameter.type]![parameter.name] = parameter.value;
});
} else if (pagination.paginationMode === 'responseContainsNextURL') {
paginationData.request.url = pagination.nextURL;
}
if (pagination.limitPagesFetched) {
paginationData.maxRequests = pagination.maxRequests;
}
if (responseFormat === 'file') {
paginationData.binaryResult = true;
}
const requestPromise = this.helpers.requestWithAuthenticationPaginated.call(
this,
requestOptions,
itemIndex,
paginationData,
nodeCredentialType,
);
requestPromises.push(requestPromise);
} else if (authentication === 'genericCredentialType' || authentication === 'none') {
if (oAuth1Api) {
const requestOAuth1 = this.helpers.requestOAuth1.call(this, 'oAuth1Api', requestOptions);
requestOAuth1.catch(() => {});
@@ -1426,25 +1720,25 @@ export class HttpRequestV3 implements INodeType {
}
const promisesResponses = await Promise.allSettled(requestPromises);
let response: any;
let responseData: any;
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
response = promisesResponses.shift();
if (response!.status !== 'fulfilled') {
if (response.reason.statusCode === 429) {
response.reason.message =
responseData = promisesResponses.shift();
if (responseData!.status !== 'fulfilled') {
if (responseData.reason.statusCode === 429) {
responseData.reason.message =
"Try spacing your requests out using the batching settings under 'Options'";
}
if (!this.continueOnFail()) {
if (autoDetectResponseFormat && response.reason.error instanceof Buffer) {
response.reason.error = Buffer.from(response.reason.error as Buffer).toString();
if (autoDetectResponseFormat && responseData.reason.error instanceof Buffer) {
responseData.reason.error = Buffer.from(responseData.reason.error as Buffer).toString();
}
throw new NodeApiError(this.getNode(), response as JsonObject, { itemIndex });
throw new NodeApiError(this.getNode(), responseData as JsonObject, { itemIndex });
} else {
removeCircularRefs(response.reason as JsonObject);
removeCircularRefs(responseData.reason as JsonObject);
// Return the actual reason as error
returnItems.push({
json: {
error: response.reason,
error: responseData.reason,
},
pairedItem: {
item: itemIndex,
@@ -1454,7 +1748,12 @@ export class HttpRequestV3 implements INodeType {
}
}
response = response.value;
let responses: any[];
if (Array.isArray(responseData.value)) {
responses = responseData.value;
} else {
responses = [responseData.value];
}
let responseFormat = this.getNodeParameter(
'options.response.response.responseFormat',
@@ -1468,172 +1767,194 @@ export class HttpRequestV3 implements INodeType {
false,
) as boolean;
if (autoDetectResponseFormat) {
const responseContentType = response.headers['content-type'] ?? '';
if (responseContentType.includes('application/json')) {
responseFormat = 'json';
const neverError = this.getNodeParameter(
'options.response.response.neverError',
0,
false,
) as boolean;
const data = await this.helpers
.binaryToBuffer(response.body as Buffer | Readable)
.then((body) => body.toString());
response.body = jsonParse(data, {
...(neverError
? { fallbackValue: {} }
: { errorMessage: 'Invalid JSON in response body' }),
});
} else if (binaryContentTypes.some((e) => responseContentType.includes(e))) {
responseFormat = 'file';
} else {
responseFormat = 'text';
const data = await this.helpers
.binaryToBuffer(response.body as Buffer | Readable)
.then((body) => body.toString());
response.body = !data ? undefined : data;
}
}
if (autoDetectResponseFormat && !fullResponse) {
delete response.headers;
delete response.statusCode;
delete response.statusMessage;
response = response.body;
requestOptions.resolveWithFullResponse = false;
}
if (responseFormat === 'file') {
const outputPropertyName = this.getNodeParameter(
'options.response.response.outputPropertyName',
0,
'data',
) as string;
const newItem: INodeExecutionData = {
json: {},
binary: {},
pairedItem: {
item: itemIndex,
},
};
if (items[itemIndex].binary !== undefined) {
// Create a shallow copy of the binary data so that the old
// data references which do not get changed still stay behind
// but the incoming data does not get changed.
Object.assign(newItem.binary as IBinaryKeyData, items[itemIndex].binary);
}
let binaryData: Buffer | Readable;
if (fullResponse) {
const returnItem: IDataObject = {};
for (const property of fullResponseProperties) {
if (property === 'body') {
continue;
}
returnItem[property] = response![property];
}
newItem.json = returnItem;
binaryData = response!.body;
} else {
newItem.json = items[itemIndex].json;
binaryData = response;
}
newItem.binary![outputPropertyName] = await this.helpers.prepareBinaryData(binaryData);
returnItems.push(newItem);
} else if (responseFormat === 'text') {
const outputPropertyName = this.getNodeParameter(
'options.response.response.outputPropertyName',
0,
'data',
) as string;
if (fullResponse) {
const returnItem: IDataObject = {};
for (const property of fullResponseProperties) {
if (property === 'body') {
returnItem[outputPropertyName] = toText(response![property]);
continue;
}
returnItem[property] = response![property];
}
returnItems.push({
json: returnItem,
pairedItem: {
item: itemIndex,
},
});
} else {
returnItems.push({
json: {
[outputPropertyName]: toText(response),
},
pairedItem: {
item: itemIndex,
},
});
}
} else {
// responseFormat: 'json'
if (requestOptions.resolveWithFullResponse === true) {
const returnItem: IDataObject = {};
for (const property of fullResponseProperties) {
returnItem[property] = response![property];
}
if (responseFormat === 'json' && typeof returnItem.body === 'string') {
try {
returnItem.body = JSON.parse(returnItem.body);
} catch (error) {
throw new NodeOperationError(
this.getNode(),
'Response body is not valid JSON. Change "Response Format" to "Text"',
{ itemIndex },
);
}
}
returnItems.push({
json: returnItem,
pairedItem: {
item: itemIndex,
},
});
} else {
if (responseFormat === 'json' && typeof response === 'string') {
try {
response = JSON.parse(response);
} catch (error) {
throw new NodeOperationError(
this.getNode(),
'Response body is not valid JSON. Change "Response Format" to "Text"',
{ itemIndex },
);
}
}
if (Array.isArray(response)) {
// eslint-disable-next-line @typescript-eslint/no-loop-func
response.forEach((item) =>
returnItems.push({
json: item,
pairedItem: {
item: itemIndex,
},
}),
);
// eslint-disable-next-line prefer-const
for (let [index, response] of Object.entries(responses)) {
if (this.getMode() === 'manual' && index === '0') {
// For manual executions save the first response in the context
// so that we can use it in the frontend and so make it easier for
// the users to create the required pagination expressions
const nodeContext = this.getContext('node');
if (pagination && pagination.paginationMode !== 'off') {
nodeContext.response = responseData.value[0];
} else {
nodeContext.response = responseData.value;
}
}
if (autoDetectResponseFormat) {
const responseContentType = response.headers['content-type'] ?? '';
if (responseContentType.includes('application/json')) {
responseFormat = 'json';
if (!response.__bodyResolved) {
const neverError = this.getNodeParameter(
'options.response.response.neverError',
0,
false,
) as boolean;
const data = await this.helpers
.binaryToBuffer(response.body as Buffer | Readable)
.then((body) => body.toString());
response.body = jsonParse(data, {
...(neverError
? { fallbackValue: {} }
: { errorMessage: 'Invalid JSON in response body' }),
});
}
} else if (binaryContentTypes.some((e) => responseContentType.includes(e))) {
responseFormat = 'file';
} else {
responseFormat = 'text';
if (!response.__bodyResolved) {
const data = await this.helpers
.binaryToBuffer(response.body as Buffer | Readable)
.then((body) => body.toString());
response.body = !data ? undefined : data;
}
}
}
if (autoDetectResponseFormat && !fullResponse) {
delete response.headers;
delete response.statusCode;
delete response.statusMessage;
}
if (!fullResponse) {
response = response.body;
}
if (responseFormat === 'file') {
const outputPropertyName = this.getNodeParameter(
'options.response.response.outputPropertyName',
0,
'data',
) as string;
const newItem: INodeExecutionData = {
json: {},
binary: {},
pairedItem: {
item: itemIndex,
},
};
if (items[itemIndex].binary !== undefined) {
// Create a shallow copy of the binary data so that the old
// data references which do not get changed still stay behind
// but the incoming data does not get changed.
Object.assign(newItem.binary as IBinaryKeyData, items[itemIndex].binary);
}
let binaryData: Buffer | Readable;
if (fullResponse) {
const returnItem: IDataObject = {};
for (const property of fullResponseProperties) {
if (property === 'body') {
continue;
}
returnItem[property] = response[property];
}
newItem.json = returnItem;
binaryData = response?.body;
} else {
newItem.json = items[itemIndex].json;
binaryData = response;
}
newItem.binary![outputPropertyName] = await this.helpers.prepareBinaryData(binaryData);
returnItems.push(newItem);
} else if (responseFormat === 'text') {
const outputPropertyName = this.getNodeParameter(
'options.response.response.outputPropertyName',
0,
'data',
) as string;
if (fullResponse) {
const returnItem: IDataObject = {};
for (const property of fullResponseProperties) {
if (property === 'body') {
returnItem[outputPropertyName] = toText(response[property]);
continue;
}
returnItem[property] = response[property];
}
returnItems.push({
json: response,
json: returnItem,
pairedItem: {
item: itemIndex,
},
});
} else {
returnItems.push({
json: {
[outputPropertyName]: toText(response),
},
pairedItem: {
item: itemIndex,
},
});
}
} else {
// responseFormat: 'json'
if (fullResponse) {
const returnItem: IDataObject = {};
for (const property of fullResponseProperties) {
returnItem[property] = response[property];
}
if (responseFormat === 'json' && typeof returnItem.body === 'string') {
try {
returnItem.body = JSON.parse(returnItem.body);
} catch (error) {
throw new NodeOperationError(
this.getNode(),
'Response body is not valid JSON. Change "Response Format" to "Text"',
{ itemIndex },
);
}
}
returnItems.push({
json: returnItem,
pairedItem: {
item: itemIndex,
},
});
} else {
if (responseFormat === 'json' && typeof response === 'string') {
try {
if (typeof response !== 'object') {
response = JSON.parse(response);
}
} catch (error) {
throw new NodeOperationError(
this.getNode(),
'Response body is not valid JSON. Change "Response Format" to "Text"',
{ itemIndex },
);
}
}
if (Array.isArray(response)) {
// eslint-disable-next-line @typescript-eslint/no-loop-func
response.forEach((item) =>
returnItems.push({
json: item,
pairedItem: {
item: itemIndex,
},
}),
);
} else {
returnItems.push({
json: response,
pairedItem: {
item: itemIndex,
},
});
}
}
}
}