Skip to content

Instantly share code, notes, and snippets.

@TomZhuPlanetart

TomZhuPlanetart/concurrent_url_fetcher.js

Last active November 12, 2025 16:01
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save TomZhuPlanetart/0d7f661b7bcc7a2ef9e19fe6109689d2 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save TomZhuPlanetart/0d7f661b7bcc7a2ef9e19fe6109689d2 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
concurrent_url_fetcher.js
#!/usr/bin/env node
/**
* Concurrent URL Fetcher
*
* Usage:
* node concurrent_url_fetcher.js --url=<URL> --concurrency=<N> --total=<N> --output=<file>
*
* Options:
* --url Target URL to call
* --concurrency Number of concurrent requests (default: 5)
* --total Total number of requests to make (default: 10)
* --output Output file path (default: ./output.json)
* --method HTTP method (default: GET)
* --timeout Request timeout in milliseconds (default: 30000)
* --headers JSON string of headers (optional)
* --body Request body (optional, for POST/PUT)
*
*
*
*/
const https = require('https');
const http = require('http');
const fs = require('fs');
const url = require('url');
// Parse command line arguments
function parseArgs() {
const args = {
url: null,
concurrency: 5,
total: 10,
output: './output.json',
method: 'GET',
timeout: 30000,
headers: {},
body: null
};
process.argv.slice(2).forEach(arg => {
// Only split on the first '=' to preserve '=' in values (e.g., URLs with query params)
const equalIndex = arg.indexOf('=');
if (equalIndex === -1) return;
const key = arg.substring(0, equalIndex);
const value = arg.substring(equalIndex + 1);
const cleanKey = key.replace(/^--/, '');
switch(cleanKey) {
case 'url':
args.url = value;
break;
case 'concurrency':
args.concurrency = parseInt(value);
break;
case 'total':
args.total = parseInt(value);
break;
case 'output':
args.output = value;
break;
case 'method':
args.method = value.toUpperCase();
break;
case 'timeout':
args.timeout = parseInt(value);
break;
case 'headers':
try {
args.headers = JSON.parse(value);
} catch (e) {
console.error('Invalid headers JSON:', e.message);
}
break;
case 'body':
args.body = value;
break;
}
});
return args;
}
// Validate arguments
function validateArgs(args) {
if (!args.url) {
console.error('Error: --url is required');
console.log('\nUsage: node concurrent_url_fetcher.js --url=<URL> [options]');
process.exit(1);
}
if (args.concurrency < 1) {
console.error('Error: --concurrency must be at least 1');
process.exit(1);
}
if (args.total < 1) {
console.error('Error: --total must be at least 1');
process.exit(1);
}
}
// Make a single HTTP request
function makeRequest(targetUrl, options, requestBody) {
return new Promise((resolve, reject) => {
const parsedUrl = url.parse(targetUrl);
const isHttps = parsedUrl.protocol === 'https:';
const client = isHttps ? https : http;
const requestOptions = {
hostname: parsedUrl.hostname,
port: parsedUrl.port || (isHttps ? 443 : 80),
path: parsedUrl.path,
method: options.method,
headers: options.headers,
timeout: options.timeout
};
const startTime = Date.now();
const req = client.request(requestOptions, (res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
const endTime = Date.now();
const duration = endTime - startTime;
resolve({
success: true,
statusCode: res.statusCode,
statusMessage: res.statusMessage,
headers: res.headers,
body: data,
duration: duration,
timestamp: new Date().toISOString()
});
});
});
req.on('error', (error) => {
const endTime = Date.now();
const duration = endTime - startTime;
resolve({
success: false,
error: error.message,
duration: duration,
timestamp: new Date().toISOString()
});
});
req.on('timeout', () => {
req.destroy();
const endTime = Date.now();
const duration = endTime - startTime;
resolve({
success: false,
error: 'Request timeout',
duration: duration,
timestamp: new Date().toISOString()
});
});
if (requestBody) {
req.write(requestBody);
}
req.end();
});
}
// Run requests with concurrency control
async function runConcurrentRequests(targetUrl, config) {
const results = [];
let completed = 0;
let inProgress = 0;
let started = 0;
const total = config.total;
const concurrency = config.concurrency;
const startOverallTime = Date.now();
return new Promise((resolve) => {
function startNext() {
while (inProgress < concurrency && started < total) {
inProgress++;
started++;
const requestId = started;
console.error(`[${requestId}/${total}] Starting request...`);
makeRequest(targetUrl, config, config.body)
.then((result) => {
inProgress--;
completed++;
result.requestId = requestId;
results.push(result);
const statusInfo = result.success
? `${result.statusCode} ${result.statusMessage}`
: `ERROR: ${result.error}`;
console.error(`[${requestId}/${total}] Completed in ${result.duration}ms - ${statusInfo}`);
if (completed === total) {
const endOverallTime = Date.now();
const totalDuration = endOverallTime - startOverallTime;
resolve({
results: results,
summary: {
total: total,
successful: results.filter(r => r.success).length,
failed: results.filter(r => !r.success).length,
totalDuration: totalDuration,
averageDuration: results.reduce((sum, r) => sum + r.duration, 0) / results.length
}
});
} else {
startNext();
}
});
}
}
startNext();
});
}
// Main function
async function main() {
const args = parseArgs();
validateArgs(args);
console.log('='.repeat(60));
console.log('Concurrent URL Fetcher');
console.log('='.repeat(60));
console.log(`URL: ${args.url}`);
console.log(`Method: ${args.method}`);
console.log(`Concurrency: ${args.concurrency}`);
console.log(`Total: ${args.total}`);
console.log(`Output: ${args.output}`);
console.log(`Timeout: ${args.timeout}ms`);
console.log('='.repeat(60));
console.log('');
try {
const data = await runConcurrentRequests(args.url, args);
console.log('');
console.log('='.repeat(60));
console.log('Summary:');
console.log('='.repeat(60));
console.log(`Total requests: ${data.summary.total}`);
console.log(`Successful: ${data.summary.successful}`);
console.log(`Failed: ${data.summary.failed}`);
console.log(`Total duration: ${data.summary.totalDuration}ms`);
console.log(`Average duration: ${data.summary.averageDuration.toFixed(2)}ms`);
console.log(`Requests per second: ${(data.summary.total / (data.summary.totalDuration / 1000)).toFixed(2)}`);
console.log('='.repeat(60));
// Write results to file (only results array)
fs.writeFileSync(args.output, JSON.stringify(data.results, null, 2));
console.log(`\nResults written to: ${args.output}`);
} catch (error) {
console.error('Error:', error);
process.exit(1);
}
}
// Run the program
main();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment