如何只过滤文件中的特定行？

Question

我有一个文件，它有以下几行。我只是想看到IP地址，它是第一行，第二线国家只有一次，而分数行应该是最高值，在这种情况下是7.1。

{
  "ip": "86.75.227.72",
  "history": [
    {
     "created": "2012-03-22T07:26:00.000Z",
     "reason": "Regional Internet Registry",
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.64.0.0/12",
     "categoryDescriptions": {},
     "reasonDescription": "One of the five RIRs announced a (new) location mapping of the IP.",
     "score": 1,
     "cats": {}
   },
   {
     "created": "2012-04-13T13:34:00.000Z",
     "reason": "DNS heuristics",
     "cats": {
       "Dynamic IPs": 100
     },
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.64.0.0/12",
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
     },
     "reasonDescription": "Based on statistical DNS analysis.",
     "score": 1
   },
   {
     "created": "2014-01-22T19:08:00.000Z",
     "reason": "DNS heuristics",
     "cats": {
       "Dynamic IPs": 86
     },
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.72.0.0/14",
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
     },
     "reasonDescription": "Based on statistical DNS analysis.",
     "score": 1
   },
   {
     "created": "2014-03-09T13:11:00.000Z",
     "reason": "DNS heuristics",
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.75.224.0/21",
     "cats": {
       "Dynamic IPs": 71
     },
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
     },
     "reasonDescription": "Based on statistical DNS analysis.",
     "score": 1
   },
   {
     "created": "2017-07-26T06:24:00.000Z",
     "reason": "Regional Internet Registry",
     "asns": {
       "15557": {
         "Company": "LDCOMNET, FR",
         "cidr": 12
       }
     },
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.75.224.0/21",
     "cats": {
       "Dynamic IPs": 71
     },
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
     },
     "reasonDescription": "One of the five RIRs announced a (new) location mapping of the IP.",
     "score": 1
   },
   {
     "created": "2017-10-10T06:23:00.000Z",
     "reason": "Regional Internet Registry",
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.75.224.0/21",
     "cats": {
       "Dynamic IPs": 71
     },
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
     },
     "reasonDescription": "One of the five RIRs announced a (new) location mapping of the IP.",
     "score": 1
   },
   {
     "created": "2017-10-18T06:23:00.000Z",
     "reason": "Regional Internet Registry",
     "asns": {
       "15557": {
         "Company": "LDCOMNET, FR",
         "cidr": 12
       }
     },
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.75.224.0/21",
     "cats": {
       "Dynamic IPs": 71
     },
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
     },
     "reasonDescription": "One of the five RIRs announced a (new) location mapping of the IP.",
     "score": 1
   },
   {
     "created": "2017-11-20T18:16:00.000Z",
     "reason": "Third party feed",
     "asns": {
       "15557": {
         "Company": "LDCOMNET, FR",
         "cidr": 12
       }
     },
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.75.227.72/32",
     "cats": {
       "Dynamic IPs": 71,
       "Bots": 71
     },
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines.",
       "Bots": "IPs known for botnet-member activity. Devices using these IPs are obviously infected and take part in DDoS-attacks, port-scanning, spam-sending etc."
     },
     "reasonDescription": "This data was imported from a third party feed.",
     "score": 7.1
   },
   {
     "created": "2017-11-25T21:46:00.000Z",
     "reason": "Third party feed",
     "asns": {
       "15557": {
         "Company": "LDCOMNET, FR",
         "cidr": 12
       }
     },
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.75.227.72/32",
     "cats": {
       "Dynamic IPs": 71
     },
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
     },
     "reasonDescription": "This data was imported from a third party feed.",
     "score": 1
   }
 ],
 "subnets": [
   {
     "created": "2017-10-18T06:23:00.000Z",
     "reason": "Regional Internet Registry",
     "asns": {
       "15557": {
         "Company": "LDCOMNET, FR",
         "cidr": 12
       }
     },
     "geo": {
       "country": "France",
       "countrycode": "FR"
     },
     "ip": "86.64.0.0",
     "categoryDescriptions": {},
     "reasonDescription": "One of the five RIRs announced a (new) location mapping of the IP.",
     "score": 1,
     "cats": {},
     "subnet": "86.64.0.0/12"
   },
   {
     "created": "2014-03-09T13:11:00.000Z",
     "reason": "DNS heuristics",
     "cats": {
       "Dynamic IPs": 71
     },
     "ip": "86.75.224.0",
     "categoryDescriptions": {
       "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
     },
     "reasonDescription": "Based on statistical DNS analysis.",
     "score": 1,
     "subnet": "86.75.224.0/21"
   }
 ],
 "cats": {
   "Dynamic IPs": 71
 },
 "geo": {
   "country": "France",
   "countrycode": "FR"
 },
 "score": 1,
 "reason": "Third party feed",
 "reasonDescription": "This data was imported from a third party feed.",
 "categoryDescriptions": {
   "Dynamic IPs": "This category contains IP addresses of dialup hosts and DSL lines."
 },
 "tags": []
}

“机器人”：“因僵尸网络成员活动而闻名的in。使用这些in的设备明显被感染并参与DDoS攻击。”

"score":7.1}
"geo":{"country":"France"
"score":1}]
"geo":{"country":"France"
"score":1
"score":1
"geo":{"country":"France"
"score":1

Answer 1

$ jq -r '.history | max_by(.score) | .ip' file.json
86.75.227.72/32

这将使用jq查找具有最大.score值的.history数组中的条目。一旦找到该值，就会从查找的条目中提取.ip值。

输出格式略有变化，这里有CSV输出的IP地址、国家、公司名称(如果有的话)，以及来自.history数组的分数：

$ jq -r '.history[] | [.ip, .geo.country, .asns."15557".Company, .score] | @csv' file.json
"86.64.0.0/12","France",,1
"86.64.0.0/12","France",,1
"86.72.0.0/14","France",,1
"86.75.224.0/21","France",,1
"86.75.224.0/21","France","LDCOMNET, FR",1
"86.75.224.0/21","France",,1
"86.75.224.0/21","France","LDCOMNET, FR",1
"86.75.227.72/32","France","LDCOMNET, FR",7.1
"86.75.227.72/32","France","LDCOMNET, FR",1

同样，但只取最高分数：

$ jq -r '.history | max_by(.score) | [.ip, .geo.country, .asns."15557".Company, .score] | @csv' file.json
"86.75.227.72/32","France","LDCOMNET, FR",7.1

Answer 2

awk方法：

$ awk '$1=="\"ip\":"{
        ip=$2
       }
       $1=="\"country\":"{
        c[ip]=$2
       }
       $1=="\"score\":" && s[ip]<$2{ 
         s[ip]=$2
       }
       END{
           for(ip in c){
            print ip,c[ip],s[ip]
           }
       }' file 
"86.72.0.0/14", "France", 1
"86.64.0.0/12", "France", 1,
"86.75.224.0/21", "France", 1
"86.75.227.72/32", "France", 7.1
"86.75.227.72", "France", 
"86.75.224.0", "France", 1,

或者，如果你只想要得分最高的IP，什么也不想要：

$ awk '$1=="\"ip\":"{ip=$2}$1=="\"score\":" && score<$2{score=$2;sip=ip}END{print sip} ' file 
"86.75.227.72/32"