request-manager

A NodeJS module to manage batch requests with the 'request' module. The timing and batch amount can be set and updated at any point in time.

Usage

Install from NPM:

npm install request-manager --save

Example usage:

const RequestManager = require('request-manager');

links = []; // Some full array of links


// Initialize a RequestManager instance
var reqm = new RequestManager({
    linkArray: links,       // required. 
    numberConcurrent: 3,    // optional, default 5. Must be >= 1
    waitTime: 500,          // optional, default 0 milliseconds. Must be >= 0.
    requestOptions: {       // optional, default none. These are the options that can be specified for the 'request' module
        encoding: null,     // Save raw responses
        headers: {          // Set a custom user agent
            'User-Agent': 'Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405'
        }
    }
});


// Start requesting the links. This can be cancelled with reqm.stop()
reqm.start();


// Cancels the currently running RequestManager. If reqm.start() is 
// called again, it starts back from the beginning.
reqm.stop();


// Every response gets sent back on a 'data' event
reqm.on('data', (err, returned) => {
    // 'err' is any error from the 'request' module
    
    // 'returned' is an object that consists of:
    //      'resp': the response from the 'request' module
    //      'data': the body of the response
    //      'link': the link that was requested
    //      'index': the index of the link in the provided link array
    
    // From here, the results of the request can be managed.
    // NOTE: All requests are transparently handled. Even if a request returns an error
    // message (eg., 404), it is always sent back here (no re-requests are attempted)
});


// When all the links have been requested and returned, the 'end' event
// is emitted
reqm.on('end', () => {
    console.log('Finished requesting all the links');
});


// Any general error message generated by the module emits an 'error' event
reqm.on('error', (err) => {
    console.log(err);
});


// Special functions
// These functions can be called while 'request-manager' is running. The
// values they update will be applied once the current batch of requests 
// is complete.

// The new waitTime in milliseconds. Must be greater than or equal to 0
reqm.updateWaitTime(newWaitTime);

// The new batch amount. Must be greater than or equal to 1
reqm.updateBatchAmount(newBatchAmount);