request-queue

Library for sending http(s)-requests over queue to limit simultaneous access to resources. Each queue has separate cookie container, so you can keep multiple sessions on single site.

Features

Version 4 now is completely awaitable. All requests are async.

Install

npm install @poccomaxa/request-queue

Using

Interface

/**
 * @param defaults {object} default options as options of http.ClientRequest
 * @param parallel {number=1} number of parallel requests
 * @param keepsession {boolean=false} true - new cookies will be saved, false - cookies will be ignored
 * @constructor
 */
const RequestQueue = require('@poccomaxa/request-queue');

/**
 * get-request without body
 * @param url {String} url
 * @param headers {object.<String, String>=} headers of single request
 * @param dontfollow {boolean=false} do not follow redirect (status == 302 || 301)
 * @return {Promise<Response>}
 * */
async RequestQueue.get(url, headers, dontfollow);

/**
 * post-request with header "Content-Type: application/x-www-form-urlencoded"
 * @param url {String} url
 * @param data {String=} requests body
 * @param headers {object.<String, String>=} headers of single request
 * @param dontfollow {boolean=false} do not follow redirect (status == 302 || 301)
 * @return {Promise<Response>}
 * */
async RequestQueue.post(url, data, headers, dontfollow);

/**
 * request with any method
 * @param method {String} http-method
 * @param url {String} url
 * @param data {String=} requests body
 * @param headers {object.<String, String>=} headers of single request
 * @param dontfollow {boolean=false} do not follow redirect (status == 302 || 301)
 * @return {Promise<Response>}
 * */
async RequestQueue.request(method, url, data, headers, dontfollow);

/**
 * adding or changing one header to defaults
 * @param name {String}
 * @param value {String}
 */
RequestQueue.setHeader(name, value);

/**
 * deleting header from defaults
 * @param name {String}
 */
RequestQueue.removeHeader(name);

/**
 * returns header if exists or undefined
 * @param name {String}
 * @returns {String|undefined}
 */
RequestQueue.getHeader(name);

/**
 * defaults object with custom headers and agent
 */
defaults = {
    headers: {
        "User-Agent": "Mozilla/5.0",
        "x-csrf-token": "0123456789"
    },
    agent: new SomeAgent() // child of http.Agent
};

/**
 * @class Response
 * @description Returned by any request of RequestQueue
 * @property {Buffer} buffer - raw response data
 * @property {Number} status - response http-status or -1 if error
 * @property {Object<String, String>} headers - response headers, ex. { "Set-Cookie": "a=1" }
 * @property {String} text - utf8 encoded response text
 * @property {Error} error - error during request
 */
Response


/**
 * raw http/https request, waiting all data and resolves Promise
 * Not recommended to use, use new RequestQueue.get(url) instead
 *
 * @param options {Object} see http.Options for more information
 * @param data {String}
 * @return {Promise<Response>} resolve (Response)
 */
static async RequestQueue.Request(options, data)

Cookies

/**
 * returns cookie cookieName value for host
 * if cookieName is null, then returns string for all cookies
 * if no cookie with name cookieName? then returns null
 * @param host {String}
 * @param cookieName {String || null}
 * @return {String || null}
 */
RequestQueue.cookies.get(host, cookieName);

/**
 * changes or adds cookie
 * @param host {String}
 * @param cookieString {String} in format "name=value;expires=1;max-age=1"
 */
RequestQueue.cookies.set(host, cookieString);

example

const RequestQueue = require('@poccomaxa/request-queue');
const parallelLimit = 5; // 5 parallel requests
const keepsession = false; // keeps cookies

// RequestQueue(defaults, parallel, secured)
let firstQueue = new RequestQueue(
    {
        headers: {
            "User-Agent": "Mozilla/5.0"
        }
    },
    parallelLimit,
    keepsession);
firstQueue.setHeader("x-csrf-token", "0123456789");
firstQueue.setHeader("x-requested-with", "XMLHttpRequest");
firstQueue.cookies.set("example.com", "mycookie=1;max-age=86400");
firstQueue.cookies.set("example.com", "mycookie2=1;max-age=86400");
console.log(firstQueue.cookies.get("example.com", "mycookie"));
// 1
console.log(firstQueue.cookies.get("example.com"));
// mycookie=1; mycookie2=1

let res = await firstQueue.post("http://some/path/to/resource", "param1=12&param2=2");
if(res.status === 200)
    console.log("result of request: %s", res.text);
else
    console.error(res);

res = await firstQueue.get("http://example.com", {"Referer": "https://example.com/ref"});
console.log(res);

Multiple queues

let secondQueue = new RequestQueue({ headers: { "User-Agent": "Mozilla/5.0" }});
secondQueue.get("http://some/url/?some=data")

KeepCookies of another queue

firstQueue.cookies = secondQueue.cookies;