puppyproxy

On npm @ https://www.npmjs.com/package/puppyproxy

And GitHub @ https://github.com/onlypuppy7/puppyproxy

Robust and reliable free proxy scraper and request wrapper, ported to npm. Another useless file taken out of my projects!

Install

npm install puppyproxy

Usage

import PuppyProxy from 'puppyproxy';
import log from 'puppylog'; //another one of my packages
import { wait } from 'puppymisc'; //another one of my packages!

const ipUrl = 'http://ifconfig.me/ip';
const wsUrl = 'wss://ws.postman-echo.com/raw';

const config = {
    // storeDir: './store', // optional, defaults to platform specific data dir
    ipvanish: {
        // use: true,
        // creds: { username: 'abc', password: '123' },
    },
    scraper: {
        use: true,
        timeBetweenScrapes: 45, // minutes
        maxProxiesToCheck: 5000, // the higher this is, the longer it takes, but the more proxies you get
        limitProxies: 500000,
        timeoutPerProxy: 15, // seconds
        timeoutMax: 5,
        verbose: true, // for logging purposes
        logStatus: true, // shows a convenient summary every so often
        proxyTypes: ['socks4', 'socks5'], //['socks4', 'socks5', 'http', 'https']
        method: 'ws', // 'ws' or 'http'
        wsUrl: wsUrl,

        timeoutFetch: 10000, // ms
        timeoutWs: 7000,
        maxRetries: 10,
    },
    LOG: { connect: true, error: true } // for logging purposes
};

const pp = new PuppyProxy(config);

log.highlight("TESTING: runCollector");

await pp.init(true); // true forces run the collector

console.log(pp.request.createProxyRequestPool().length, "proxies in the pool after init");

log.highlight("TESTING: RequestClient");

log.bold("----- pp.request.fetch")

//fetch with inbuilt resilient retrying in case of error. uses a new proxy from the pool each time
const resRequestFetch = await pp.request.fetch(ipUrl);
log.success("res from request fetch", await resRequestFetch.text());

log.bold("----- pp.request.fetchBasic")

//more basic fetch that just forwards to you the req object with a proxy agent set
try {
    const resFetchBasic = await (pp.request.fetchBasic(ipUrl));
    if (resFetchBasic?.ok) {
        log.success("res from basic fetch", await resFetchBasic.text());
    } else {
        //this may fail because free proxies are unreliable, and it only tries once
        log.error("basic fetch failed", resFetchBasic.status);
    }
} catch (error) {
    log.error("basic fetch error, which is sort of expected. create your own catching logic.", error.message);
}

log.bold("----- pp.request.ws")

//ws with inbuilt resilient retrying in case of error. uses a new proxy from the pool each time
const testWs = async (wsHandler) => {
    return new Promise(async (resolve, reject) => {
        await wsHandler(
            wsUrl,
            {
                // preferredAgent: 'none' // 'ipvanish', 'scraper', or 'none'
            },
            // open
            (ws) => {
                const testMessage = 'echo, test message';
                log.success(`Connected to echo server, sending test message ${testMessage}`);
                ws.send(JSON.stringify(testMessage));
            },
            // message
            (ws, event) => {
                try {
                    const data = JSON.parse(event.data);
                    log.success('Received JSON:', data);
                } catch {
                    log.success('Received text:', event.data);
                }

                // close after we get responses
                ws.close();
                wait(500).then(resolve);
            },
            // close
            (ws, event) => {
                log.italic(`WebSocket closed with code: ${event.code}, reason: ${event.reason}`);
            },
            // error
            (ws, error) => {
                // reject(error);
            }
        );
        resolve();
    });
}

try {
    await testWs(pp.request.ws.bind(pp.request));
} catch (error) {
    log.error("WebSocket connection failed:", error.message);
};

log.bold("----- pp.request.wsBasic");

const testWsBasic = async (wsBasicHandler) => {
    try {
        await new Promise(async (resolve, reject) => {
            const ws = wsBasicHandler(wsUrl, {});

            ws.onopen = (event) => {
                const testMessage = 'echo, test message via wsBasic';
                log.success(`Connected to echo server via wsBasic, sending test message ${testMessage}`);
                ws.send(JSON.stringify(testMessage));
            };

            ws.onmessage = (event) => {
                try {
                    const data = JSON.parse(event.data);
                    log.success('wsBasic Received JSON:', data);
                } catch {
                    log.success('wsBasic Received text:', event.data);
                }

                ws.close();
                wait(500).then(resolve);
            };

            ws.onclose = (event) => {
                log.italic(`wsBasic WebSocket closed with code: ${event.code}, reason: ${event.reason}`);
            };

            ws.onerror = (error) => {
                log.error("wsBasic WebSocket error, which is sort of expected. create your own catching logic.", error.message);
                reject(error);
            };
        });
    } catch (error) {
        // log.error("proxyRequest wsBasic WebSocket connection failed:", error.message);
    };
};

//similar basic method like pp.request.fetchBasic exists for pp.request.wsBasic if you want less resilience and more control.

await testWsBasic(pp.request.wsBasic.bind(pp.request));

log.highlight("TESTING: ProxyRequest");

//proxyRequest is a wrapper that allows you to use a more consistent proxy for multiple requests, with auto-rotation when it fails
const proxyRequest = pp.createProxyRequest(ipUrl, {
    autoNewAgent: true, // auto rotate the proxy when a request fails. set this to false if you really need just one proxy you can manage yourself
});

log.bold("----- proxyRequest.fetch");

//all methods are similar to pp.request but use the same proxy until it fails
const resProxyRequestFetch = await proxyRequest.fetch(); //url can be passed here, but it already was set in constructor
log.success("res from proxyRequest fetch", await resProxyRequestFetch.text());

log.bold("----- proxyRequest.fetchBasic");

const testProxyRequestFetchBasic = async () => {
    try {
        const resProxyRequestFetchBasic = await (proxyRequest.fetchBasic());
        if (resProxyRequestFetchBasic?.ok) {
            log.success("res from proxyRequest basic fetch", await resProxyRequestFetchBasic.text());
        } else {
            log.error("proxyRequest basic fetch failed", resProxyRequestFetchBasic.status);
        }
    } catch (error) {
        log.error("proxyRequest basic fetch error, which is sort of expected. create your own catching logic.", error.message);
    }
};

await testProxyRequestFetchBasic();
//try again to see it use the same proxy
await testProxyRequestFetchBasic();

log.bold("----- proxyRequest.newAgent");

//you can manually rotate the proxy too
proxyRequest.newAgent();

await testProxyRequestFetchBasic();

log.bold("----- proxyRequest.ws");

try {
    await testWs(proxyRequest.ws.bind(proxyRequest));
} catch (error) {
    log.error("proxyRequest WebSocket connection failed:", error.message);
}

log.bold("----- proxyRequest.wsBasic");

await testWsBasic(proxyRequest.wsBasic.bind(proxyRequest));

log.highlight("TESTING: createGlobalPatch");

pp.createGlobalPatch();

log.bold("----- global fetch")

const resGlobalFetch = await fetch(ipUrl);
log.success("res from global fetch", await resGlobalFetch.text());

//note: i dont think you can override node-fetch. sad!

log.bold("----- global WebSocket")

try {
    await new Promise((resolve, reject) => {
        const ws = new WebSocket(wsUrl);
        ws.onopen = (event) => {
            const testMessage = 'echo, test message via global WebSocket';
            log.success(`Connected to echo server via global WebSocket, sending test message ${testMessage}`);
            ws.send(JSON.stringify(testMessage));
        };

        ws.onmessage = (event) => {
            try {
                const data = JSON.parse(event.data);
                log.success('global WebSocket Received JSON:', data);
            } catch {
                log.success('global WebSocket Received text:', event.data);
            }
            ws.close();
            wait(500).then(resolve);
        };

        ws.onclose = (event) => {
            log.italic(`global WebSocket closed with code: ${event.code}, reason: ${event.reason}`);
        };

        ws.onerror = (error) => {
            log.error("global WebSocket error, which is sort of expected. create your own catching logic.", error.message);
            reject(error);
        };
    });
} catch (error) {
    log.error("global WebSocket connection failed:", error.message);
}