动态代理+并发控制实现无痕数据洪流

const { HttpsProxyAgent } = require('https-proxy-agent');
const Bottleneck = require('bottleneck');
const axios = require('axios');
​
// 代理池类
class ProxyPool {
  constructor() {
    this.proxies = []; // 存储代理对象 { ip, port, protocol, lastUsed }
    this.index = 0;
    this.testURL = 'http://httpbin.org/ip'; // 代理测试地址
  }
​
  // 添加新代理
  addProxy(proxy) {
    if (!this.proxies.some(p => p.ip === proxy.ip)) {
      this.proxies.push({ ...proxy, lastUsed: 0, failedCount: 0 });
    }
  }
​
  // 获取下一个可用代理（轮询）
  async getNextProxy() {
    if (this.proxies.length === 0) throw new Error('No proxies available');
    
    // 按使用时间和失败次数排序
    this.proxies.sort((a, b) => 
      a.failedCount - b.failedCount || a.lastUsed - b.lastUsed
    );
    
    const proxy = this.proxies[0];
    proxy.lastUsed = Date.now();
    return `${proxy.protocol}://${proxy.ip}:${proxy.port}`;
  }
​
  // 标记代理失败
  markFailed(proxyUrl) {
    const proxy = this.proxies.find(p => 
      `${p.protocol}://${p.ip}:${p.port}` === proxyUrl
    );
    if (proxy) {
      proxy.failedCount++;
      if (proxy.failedCount > 3) { // 连续失败3次移除
        this.proxies = this.proxies.filter(p => p !== proxy);
        console.log(`Removed bad proxy: ${proxyUrl}`);
      }
    }
  }
}
​
// 爬虫管理器
class ConcurrentCrawler {
  constructor() {
    this.proxyPool = new ProxyPool();
    this.limiter = new Bottleneck({
      minTime: 100,       // 最小请求间隔(ms)
      maxConcurrent: 50,  // 最大并发数
      reservoir: 100,     // 初始令牌数
      reservoirRefreshInterval: 60 * 1000, // 每分钟补充
      reservoirRefreshAmount: 100
    });
  }
​
  // 初始化代理池
  async initProxies() {
    // 从代理提供商获取IP（示例）
    const newProxies = await this.fetchProxyList();
    newProxies.forEach(p => this.proxyPool.addProxy(p));
    console.log(`Proxy pool initialized with ${this.proxyPool.proxies.length} proxies`);
  }
​
  // 获取代理列表（示例）
  async fetchProxyList() {
    // 实际使用中替换为真实代理API
    return [
      { ip: '123.123.123.123', port: 8080, protocol: 'http' },
      { ip: '45.45.45.45', port: 3128, protocol: 'http' },
      // 更多代理...
    ];
  }
​
  // 执行爬取任务
  async crawl(url) {
    return this.limiter.schedule(async () => {
      let attempt = 0;
      const maxAttempts = 5;
      
      while (attempt < maxAttempts) {
        attempt++;
        try {
          const proxyUrl = await this.proxyPool.getNextProxy();
          const agent = new HttpsProxyAgent(proxyUrl);
          
          const response = await axios.get(url, {
            httpsAgent: agent,
            timeout: 10000,
            headers: {
              'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ...'
            }
          });
          
          console.log(`Success: ${url} via ${proxyUrl}`);
          return response.data;
        } catch (error) {
          console.warn(`Attempt ${attempt} failed: ${error.message}`);
          if (error.config?.httpsAgent?.proxy) {
            this.proxyPool.markFailed(error.config.httpsAgent.proxy.href);
          }
          
          if (attempt >= maxAttempts) {
            throw new Error(`All attempts failed for ${url}`);
          }
          
          // 指数退避重试
          await new Promise(res => setTimeout(res, 500 * Math.pow(2, attempt)));
        }
      }
    });
  }
​
  // 批量爬取
  async batchCrawl(urls) {
    await this.initProxies();
    return Promise.all(urls.map(url => this.crawl(url)));
  }
}
​
// 使用示例
(async () => {
  const crawler = new ConcurrentCrawler();
  
  // 准备URL列表
  const urls = Array(100).fill().map((_, i) => 
    `https://example.com/data?page=${i+1}`
  );
  
  try {
    const results = await crawler.batchCrawl(urls);
    console.log(`Crawled ${results.length} pages successfully`);
    // 处理结果数据...
  } catch (error) {
    console.error('Crawling failed:', error);
  }
})();

// 根据成功率动态调整并发
setInterval(() => {
  const successRate = calculateSuccessRate();
  this.limiter.updateSettings({
    maxConcurrent: Math.floor(50 * successRate)
  });
}, 5000);

// 代理分级管理
class ProxyPool {
  constructor() {
    this.premiumProxies = [];  // 高质量代理
    this.normalProxies = [];   // 普通代理
  }
}

npm install https-proxy-agent axios bottleneck