What is User Agent Rotation?
User agent rotation is a technique where you dynamically change the User-Agent HTTP header across multiple requests to simulate traffic from different browsers, devices, and operating systems. This prevents websites from detecting patterns, bypasses rate limiting, and helps avoid IP bans in web scraping and automation workflows.
User Agent Header
Structure and Purpose
interface UserAgentStructure {
format: 'Mozilla/[version] ([system]) [platform] ([details]) [extensions]';
example: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36';
components: {
mozilla: 'Historical compatibility token';
system: 'Operating system and version';
platform: 'Browser engine (WebKit, Gecko, etc.)';
details: 'Browser name and version';
extensions: 'Additional capabilities';
};
purpose: {
compatibility: 'Server sends appropriate content';
analytics: 'Track browser/device statistics';
fingerprinting: 'Identify and track users';
};
}
Common User Agents
Desktop Browsers
const desktopUserAgents = {
chrome_windows: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
chrome_mac: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
chrome_linux: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
firefox_windows: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
firefox_mac: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
firefox_linux: 'Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0',
safari_mac: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
edge_windows: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0'
};
Mobile User Agents
const mobileUserAgents = {
iphone: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1',
ipad: 'Mozilla/5.0 (iPad; CPU OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1',
android_chrome: 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
android_samsung: 'Mozilla/5.0 (Linux; Android 13; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36'
};
Simple User Agent Rotation
Basic Implementation
class UserAgentRotator {
private userAgents: string[];
private currentIndex: number = 0;
constructor(userAgents?: string[]) {
this.userAgents = userAgents || this.getDefaultUserAgents();
}
private getDefaultUserAgents(): string[] {
return [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120.0.0.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Firefox/121.0',
'Mozilla/5.0 (X11; Linux x86_64) Chrome/120.0.0.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Safari/605.1.15'
];
}
// Get next user agent (round-robin)
getNext(): string {
const ua = this.userAgents[this.currentIndex];
this.currentIndex = (this.currentIndex + 1) % this.userAgents.length;
return ua;
}
// Get random user agent
getRandom(): string {
const randomIndex = Math.floor(Math.random() * this.userAgents.length);
return this.userAgents[randomIndex];
}
// Add custom user agent
add(userAgent: string): void {
this.userAgents.push(userAgent);
}
// Remove user agent
remove(userAgent: string): void {
this.userAgents = this.userAgents.filter(ua => ua !== userAgent);
}
// Get all user agents
getAll(): string[] {
return [...this.userAgents];
}
}
// Usage
const rotator = new UserAgentRotator();
// Round-robin rotation
for (let i = 0; i < 5; i++) {
console.log(`Request ${i + 1}:`, rotator.getNext());
}
// Random rotation
for (let i = 0; i < 5; i++) {
console.log(`Random ${i + 1}:`, rotator.getRandom());
}
Advanced Rotation with Fetch
HTTP Requests with Rotation
class RotatingFetcher {
private rotator: UserAgentRotator;
constructor(userAgents?: string[]) {
this.rotator = new UserAgentRotator(userAgents);
}
async fetch(url: string, options: RequestInit = {}): Promise<Response> {
const userAgent = this.rotator.getRandom();
const response = await fetch(url, {
...options,
headers: {
...options.headers,
'User-Agent': userAgent
}
});
return response;
}
async fetchWithProxy(url: string, apiKey: string, options: RequestInit = {}): Promise<Response> {
const userAgent = this.rotator.getRandom();
const response = await fetch(
`https://corsproxy.io/?url=${encodeURIComponent(url)}`,
{
...options,
headers: {
...options.headers,
'User-Agent': userAgent,
'x-cors-api-key': apiKey
}
}
);
return response;
}
async scrapeMultiplePages(urls: string[]): Promise<any[]> {
const results = [];
for (const url of urls) {
const userAgent = this.rotator.getNext();
console.log(`Fetching ${url} with UA: ${userAgent.substring(0, 50)}...`);
const response = await this.fetch(url);
const data = await response.text();
results.push({
url,
userAgent,
statusCode: response.status,
data
});
// Random delay between requests
await this.delay(1000 + Math.random() * 2000);
}
return results;
}
private delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
}
// Usage
const fetcher = new RotatingFetcher();
// Scrape with rotating user agents
const urls = [
'https://example.com/page1',
'https://example.com/page2',
'https://example.com/page3'
];
const results = await fetcher.scrapeMultiplePages(urls);
console.log('Scraped results:', results);
Weighted User Agent Rotation
Realistic Distribution
interface WeightedUserAgent {
userAgent: string;
weight: number; // Probability weight
browserName: string;
platform: string;
}
class WeightedUserAgentRotator {
private userAgents: WeightedUserAgent[];
private totalWeight: number;
constructor() {
this.userAgents = this.getRealisticDistribution();
this.totalWeight = this.userAgents.reduce((sum, ua) => sum + ua.weight, 0);
}
private getRealisticDistribution(): WeightedUserAgent[] {
// Based on real-world browser market share (2024)
return [
{
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0',
weight: 65, // 65% Chrome
browserName: 'Chrome',
platform: 'Windows'
},
{
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/120.0.0.0',
weight: 15, // 15% Chrome macOS
browserName: 'Chrome',
platform: 'macOS'
},
{
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Firefox/121.0',
weight: 8, // 8% Firefox
browserName: 'Firefox',
platform: 'Windows'
},
{
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Safari/605.1.15',
weight: 7, // 7% Safari
browserName: 'Safari',
platform: 'macOS'
},
{
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Edge/120.0.0.0',
weight: 5, // 5% Edge
browserName: 'Edge',
platform: 'Windows'
}
];
}
getWeighted(): WeightedUserAgent {
let random = Math.random() * this.totalWeight;
for (const ua of this.userAgents) {
random -= ua.weight;
if (random <= 0) {
return ua;
}
}
return this.userAgents[0]; // Fallback
}
getByPlatform(platform: string): string {
const filtered = this.userAgents.filter(ua => ua.platform === platform);
if (filtered.length === 0) {
throw new Error(`No user agents found for platform: ${platform}`);
}
const randomIndex = Math.floor(Math.random() * filtered.length);
return filtered[randomIndex].userAgent;
}
getByBrowser(browserName: string): string {
const filtered = this.userAgents.filter(ua => ua.browserName === browserName);
if (filtered.length === 0) {
throw new Error(`No user agents found for browser: ${browserName}`);
}
const randomIndex = Math.floor(Math.random() * filtered.length);
return filtered[randomIndex].userAgent;
}
}
// Usage
const weightedRotator = new WeightedUserAgentRotator();
// Get realistic distribution (65% chance of Chrome, etc.)
for (let i = 0; i < 10; i++) {
const ua = weightedRotator.getWeighted();
console.log(`${ua.browserName} on ${ua.platform}`);
}
// Get specific platform
const windowsUA = weightedRotator.getByPlatform('Windows');
console.log('Windows UA:', windowsUA);
Dynamic User Agent Generation
Versioned Rotation
class DynamicUserAgentGenerator {
generateChrome(version?: number): string {
const chromeVersion = version || (120 + Math.floor(Math.random() * 5)); // 120-124
const platforms = [
`Windows NT 10.0; Win64; x64`,
`Macintosh; Intel Mac OS X 10_15_7`,
`X11; Linux x86_64`
];
const platform = platforms[Math.floor(Math.random() * platforms.length)];
return `Mozilla/5.0 (${platform}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeVersion}.0.0.0 Safari/537.36`;
}
generateFirefox(version?: number): string {
const firefoxVersion = version || (120 + Math.floor(Math.random() * 5));
const platforms = [
`Windows NT 10.0; Win64; x64`,
`Macintosh; Intel Mac OS X 10.15`,
`X11; Linux x86_64`
];
const platform = platforms[Math.floor(Math.random() * platforms.length)];
return `Mozilla/5.0 (${platform}; rv:${firefoxVersion}.0) Gecko/20100101 Firefox/${firefoxVersion}.0`;
}
generateSafari(): string {
const safariVersions = ['16.6', '17.0', '17.1', '17.2'];
const version = safariVersions[Math.floor(Math.random() * safariVersions.length)];
return `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/${version} Safari/605.1.15`;
}
generateMobile(): string {
const mobileUAs = [
() => {
const iosVersion = (16 + Math.random() * 2).toFixed(1);
return `Mozilla/5.0 (iPhone; CPU iPhone OS ${iosVersion.replace('.', '_')} like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/${iosVersion} Mobile/15E148 Safari/604.1`;
},
() => {
const chromeVersion = 120 + Math.floor(Math.random() * 5);
return `Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeVersion}.0.0.0 Mobile Safari/537.36`;
}
];
const generator = mobileUAs[Math.floor(Math.random() * mobileUAs.length)];
return generator();
}
generateRandom(): string {
const generators = [
() => this.generateChrome(),
() => this.generateFirefox(),
() => this.generateSafari(),
() => this.generateMobile()
];
const generator = generators[Math.floor(Math.random() * generators.length)];
return generator();
}
}
// Usage
const generator = new DynamicUserAgentGenerator();
console.log('Chrome:', generator.generateChrome());
console.log('Firefox:', generator.generateFirefox());
console.log('Safari:', generator.generateSafari());
console.log('Mobile:', generator.generateMobile());
console.log('Random:', generator.generateRandom());
Session-Based Rotation
Consistent User Agent per Session
interface Session {
id: string;
userAgent: string;
createdAt: Date;
lastUsed: Date;
requestCount: number;
}
class SessionBasedRotator {
private sessions: Map<string, Session> = new Map();
private rotator: UserAgentRotator;
private sessionDuration: number = 3600000; // 1 hour
constructor() {
this.rotator = new UserAgentRotator();
}
getSession(sessionId: string): Session {
let session = this.sessions.get(sessionId);
if (!session || this.isSessionExpired(session)) {
// Create new session with new user agent
session = {
id: sessionId,
userAgent: this.rotator.getRandom(),
createdAt: new Date(),
lastUsed: new Date(),
requestCount: 0
};
this.sessions.set(sessionId, session);
} else {
// Update existing session
session.lastUsed = new Date();
session.requestCount++;
}
return session;
}
private isSessionExpired(session: Session): boolean {
const now = Date.now();
const sessionAge = now - session.createdAt.getTime();
return sessionAge > this.sessionDuration;
}
async fetchWithSession(sessionId: string, url: string): Promise<Response> {
const session = this.getSession(sessionId);
console.log(`Session ${sessionId}: Request #${session.requestCount} with UA: ${session.userAgent.substring(0, 50)}...`);
return fetch(url, {
headers: {
'User-Agent': session.userAgent
}
});
}
clearExpiredSessions(): void {
for (const [id, session] of this.sessions.entries()) {
if (this.isSessionExpired(session)) {
this.sessions.delete(id);
}
}
}
}
// Usage
const sessionRotator = new SessionBasedRotator();
// Multiple requests in same session use same UA
await sessionRotator.fetchWithSession('user123', 'https://example.com/page1');
await sessionRotator.fetchWithSession('user123', 'https://example.com/page2');
await sessionRotator.fetchWithSession('user123', 'https://example.com/page3');
// Different session uses different UA
await sessionRotator.fetchWithSession('user456', 'https://example.com/page1');
Best Practices
Effective Rotation
interface RotationBestPractices {
consistency: {
rule: 'Use same UA for related requests';
reason: 'Changing UA mid-session is suspicious';
example: 'Login + browse + checkout = same UA';
};
realism: {
rule: 'Use realistic, current user agents';
reason: 'Outdated UAs trigger detection';
example: 'Chrome 120+, not Chrome 50';
};
distribution: {
rule: 'Match real-world browser market share';
reason: 'Unusual distributions get flagged';
example: '65% Chrome, not 100% Firefox';
};
headers: {
rule: 'Match UA with other headers';
reason: 'Inconsistent headers are obvious';
example: 'Safari UA + Safari Accept headers';
};
timing: {
rule: 'Add delays between rotations';
reason: 'Too fast = bot behavior';
example: '1-3 second delays between requests';
};
}
Detection Avoidance
Complementary Techniques
interface UARotationWithEvasion {
userAgent: 'Rotate user agent string';
headers: 'Match Accept, Accept-Language, Accept-Encoding';
fingerprint: 'Canvas, WebGL must match UA';
behavior: 'Mouse movements, timing match device';
ip: 'Combine with proxy rotation';
}
// Complete evasion
async function fetchWithFullEvasion(url: string, apiKey: string) {
const generator = new DynamicUserAgentGenerator();
const userAgent = generator.generateChrome();
const response = await fetch(
`https://corsproxy.io/?url=${encodeURIComponent(url)}`,
{
headers: {
'User-Agent': userAgent,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'DNT': '1',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'x-cors-api-key': apiKey
}
}
);
return response;
}