如何判断网站流量飙升是搜索引擎爬虫导致的？

log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent "$http_referer" '
                  '"$http_user_agent" "$http_x_forwarded_for"';

#!/bin/bash
# 定义日志文件路径（根据自己的服务器配置修改）
LOG_FILE="/var/log/nginx/access.log"
# 定义搜索引擎爬虫UA关键词（可根据需要补充）
CRAWLERS=("Baiduspider" "Googlebot" "360Spider" "Sogou Spider" "YisouSpider" "bingbot")

# 统计总访问量
TOTAL_REQUESTS=$(wc -l < $LOG_FILE)
echo "=== 网站总访问请求数：$TOTAL_REQUESTS ==="

# 遍历爬虫关键词，统计各爬虫访问量
for crawler in "${CRAWLERS[@]}"; do
    CRAWLER_REQUESTS=$(grep -i "$crawler" $LOG_FILE | wc -l)
    # 计算占比
    if [ $TOTAL_REQUESTS -gt 0 ]; then
        RATIO=$(echo "scale=2; $CRAWLER_REQUESTS/$TOTAL_REQUESTS*100" | bc)
    else
        RATIO=0
    fi
    echo "$crawler 访问量：$CRAWLER_REQUESTS，占比：$RATIO%"
done

# 统计所有爬虫总访问量
ALL_CRAWLER_REQUESTS=$(grep -i -E "$(IFS='|'; echo "${CRAWLERS[*]}")" $LOG_FILE | wc -l)
ALL_RATIO=$(echo "scale=2; $ALL_CRAWLER_REQUESTS/$TOTAL_REQUESTS*100" | bc)
echo "=== 所有搜索引擎爬虫总访问量：$ALL_CRAWLER_REQUESTS，占比：$ALL_RATIO% ==="

import re
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime

# 配置项（根据实际情况修改）
LOG_FILE = "/var/log/nginx/access.log"
# 爬虫UA关键词
CRAWLER_UA_KEYWORDS = ["baiduspider", "googlebot", "360spider", "sogou", "yisou", "bingbot"]
# 日志解析正则（匹配Nginx main格式）
LOG_PATTERN = r'^(\S+) - \S+ \[(\d+/\w+/\d+:\d+:\d+:\d+) \+\d+\] "(\S+) (\S+) \S+" \d+ \d+ "\S+" "([^"]+)"'

def parse_nginx_log(log_file):
    """解析Nginx日志，提取关键信息"""
    data = []
    with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
        for line in f:
            match = re.match(LOG_PATTERN, line)
            if match:
                ip = match.group(1)
                time_str = match.group(2)
                method = match.group(3)
                url = match.group(4)
                ua = match.group(5).lower()
                
                # 转换时间格式
                try:
                    access_time = datetime.strptime(time_str, "%d/%b/%Y:%H:%M:%S")
                except:
                    access_time = None
                
                # 判断是否为爬虫
                is_crawler = any(keyword in ua for keyword in CRAWLER_UA_KEYWORDS)
                
                data.append({
                    "ip": ip,
                    "access_time": access_time,
                    "method": method,
                    "url": url,
                    "ua": ua,
                    "is_crawler": is_crawler
                })
    return pd.DataFrame(data)

def analyze_traffic_trend(df):
    """按小时统计爬虫/非爬虫访问量趋势"""
    if df.empty:
        print("日志数据为空，请检查日志文件路径或格式")
        return
    
    # 按小时分组
    df['hour'] = df['access_time'].dt.floor('H')
    trend_df = df.groupby(['hour', 'is_crawler']).size().unstack(fill_value=0)
    trend_df.columns = ['非爬虫流量', '爬虫流量']
    
    # 绘制趋势图
    plt.rcParams['font.sans-serif'] = ['SimHei']  # 解决中文显示问题
    plt.figure(figsize=(12, 6))
    trend_df.plot(kind='line', ax=plt.gca())
    plt.title('网站爬虫/非爬虫流量小时趋势')
    plt.xlabel('时间')
    plt.ylabel('访问量')
    plt.grid(True, alpha=0.3)
    plt.savefig('crawler_traffic_trend.png', dpi=300, bbox_inches='tight')
    print("流量趋势图已保存为 crawler_traffic_trend.png")

def main():
    # 解析日志
    print("正在解析Nginx日志...")
    df = parse_nginx_log(LOG_FILE)
    
    # 基础统计
    total = len(df)
    crawler_total = df['is_crawler'].sum()
    crawler_ratio = (crawler_total / total) * 100 if total > 0 else 0
    
    print(f"\n=== 日志分析结果 ===")
    print(f"总访问请求数：{total}")
    print(f"爬虫访问请求数：{crawler_total}")
    print(f"爬虫流量占比：{crawler_ratio:.2f}%")
    
    # 统计Top 10爬虫IP
    crawler_ips = df[df['is_crawler']]['ip'].value_counts().head(10)
    print(f"\n=== 访问量Top 10爬虫IP ===")
    print(crawler_ips)
    
    # 生成趋势图
    analyze_traffic_trend(df)

if __name__ == "__main__":
    main()

#!/bin/bash

# 代理配置信息
proxyHost="www.16yun.cn"
proxyPort="5445"
proxyUser="16QMSOML"
proxyPass="280651"

# 待验证的IP（替换为你要检查的IP）
IP="123.125.71.106"

# 定义带代理的nslookup执行函数
nslookup_with_proxy() {
    # nslookup通过socks5代理执行（需确保系统安装了nslookup和socat）
    # 原理：通过socat将nslookup的DNS请求转发到代理服务器
    socat - SOCKS5:$proxyHost:$1:53,proxyport=$proxyPort,proxyauth=$proxyUser:$proxyPass << EOF
$(nslookup $1 2>/dev/null)
EOF
}

# 反向解析IP（通过代理）
echo "正在通过代理反向解析IP: $IP..."
REVERSE_DNS=$(nslookup_with_proxy $IP | grep 'name =' | awk '{print $4}')
# 处理可能的空值
if [ -z "$REVERSE_DNS" ]; then
    echo "警告：反向解析结果为空，可能是代理配置错误或IP解析失败"
    exit 1
fi
echo "IP $IP 反向解析结果：$REVERSE_DNS"

# 验证是否为百度爬虫（百度爬虫反向解析域名以.baidu.com结尾）
if [[ $REVERSE_DNS == *.baidu.com ]]; then
    # 正向解析域名（通过代理）
    echo "正在通过代理正向解析域名: $REVERSE_DNS..."
    FORWARD_IP=$(nslookup_with_proxy $REVERSE_DNS | grep 'Address:' | grep -v '#' | awk '{print $2}')
    if [[ $FORWARD_IP == $IP ]]; then
        echo "验证通过：该IP是百度官方爬虫"
    else
        echo "验证失败：该IP伪装成百度爬虫"
    fi
else
    echo "该IP不是百度爬虫IP"
fi