在Python中对NSE选项链数据进行NSE抓取

Question

在这段代码中，我试图通过Python代码获取NSE选项链数据。

工具- Spyder4 Python - 3.7

代码没有抛出任何错误，我不知道我做错了什么。PRINT 1给出了我正确的JSON数据输出，但是PRINT 2& PRINT 3没有显示任何输出。有没有人可以帮我调试这段代码。

import requests
import json
import pandas as pd
import xlwings as xw
from df2gspread import df2gspread as d2g

import gspread 
from oauth2client.service_account import  ServiceAccountCredentials

pd.set_option('display.width', 1500)
pd.set_option('display.max_columns', 75)
pd.set_option('display.max_row', 2500)

url = "https://www.nseindia.com/api/option-chain-indices?symbol=NIFTY"

headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36',
"accept-language": "en-US,en;q=0.9,hi;q=0.8","accept-encoding": "gzip, deflate, br"}

    cookie_dict = {'bm_sv' : 'AA02590AB18B4FC4A036CC62F5230694~8py6nqGfKvu3P4aKZoNpf4HZOUYQJ4i6JMyPMX14ksLZYE+0HlglIA3S2AAa9JGJPvXrBHcJ7uS2ZMcMq3f+FZ/ttHuqFzuAmMf1ZnI9hFgpqB7USISOoa3NfzMufwVAd0U7MgeSxF7+GjuyOuApyOQcoHmyr53hB4JLSqd0U1s'}
    
    session = requests.session()
    
    for cookie in cookie_dict:
        session.cookies.set(cookie,cookie_dict[cookie])


expiry = '16-Jul-2020'


def fetch_oi():
   
   r = session.get(url, headers=headers).json()
   #print(r)      PRINT 1 - THIS PRINT IS WORKING 

   if expiry:
      ce_values = [data['CE'] for data in r ['records']['data'] if "CE" in data and str(data['expiryDate'].lower() == str(expiry).lower())]
      pe_values = [data['PE'] for data in r ['records']['data'] if "PE" in data and str(data['expiryDate'].lower() == str(expiry).lower())]
   else:
     ce_values = [data['CE'] for data in r ['filtered']['data'] if "CE" in data]
     pe_values = [data['PE'] for data in r ['filtered']['data'] if "PE" in data]
     print(ce_values) # PRINT 2 NO OUTPUT NO ERROR
     
     ce_data = pd.DataFrame(ce_values)
     pe_data = pd.DataFrame(pe_values)
     ce_data = ce_data.sort_values(['strikePrice'])
     pe_data = pe_data.sort_values(['strikePrice'])
     print(ce_values)      # PRINT 3 NO OUTPUT NO ERROR    
    

def main():
    fetch_oi()

if __name__ == '__main__':
    main()

Answer 1

您的字符串转换失败，请求句柄缺少参数，我已经修改了您的代码，应该可以在下面工作

import requests
import json
import pandas as pd

new_url = 'https://www.nseindia.com/api/option-chain-indices?symbol=BANKNIFTY'

headers = {'User-Agent': 'Mozilla/5.0'}
page = requests.get(new_url,headers=headers)
dajs = json.loads(page.text)


def fetch_oi(expiry_dt):
    ce_values = [data['CE'] for data in dajs['records']['data'] if "CE" in data and data['expiryDate'] == expiry_dt]
    pe_values = [data['PE'] for data in dajs['records']['data'] if "PE" in data and data['expiryDate'] == expiry_dt]

    ce_dt = pd.DataFrame(ce_values).sort_values(['strikePrice'])
    pe_dt = pd.DataFrame(pe_values).sort_values(['strikePrice'])
    
    print(ce_dt[['strikePrice','lastPrice']])

def main():
    
    expiry_dt = '27-Aug-2020'
    fetch_oi(expiry_dt)

if __name__ == '__main__':
    main()

Answer 2

现在，他们添加了2个主要cookie，用于确定您是否是真正的用户

Cookie名称为nsit，nseappid

找不到这两个cookie是如何设置到浏览器中的。

在第一次访问NSE站点时，这两个cookie以某种方式被设置，当然有一些过期。对于每个资源请求，例如https://www.nseindia.com/api/option-chain-indices?COUNTER，需要将这两个cookie设置到请求标头中，以便获取数据。

Answer 3

您可以重复调用url，直到您获得数据：

url = 'https://www.nseindia.com/api/option-chain-indices?symbol='+symbol
found = False
while not found:
    try:
        data = requests.get(url, headers=urlheader).content
        data2 = data.decode('utf-8')
        df = json.loads(data2)
        expiry_dt = df['records']['expiryDates'][0]

        found = True
    except:
        pass