如何避免使用aiohttp的“太多请求”错误

Question

下面是我的解析器代码片段。它异步执行120个请求。但是，每个响应都返回429个“太多请求”错误。我如何使它“变慢”，这样api就不会拒绝我了？

def get_tasks(self, session):
    tasks = []
    for url in self.list_of_urls:
        tasks.append(asyncio.create_task(session.get(url, ssl=False)))
    return tasks


async def get_symbols(self):
    print('Parsing started')
    async with aiohttp.ClientSession() as session:
        tasks = self.get_tasks(session)
        responses = await asyncio.gather(*tasks)
        for response in responses:
            response = await response.json()
            print(response)

错误：

{'message': 'Too many requests'}
{'message': 'Too many requests'}
{'message': 'Too many requests'}
{'message': 'Too many requests'}
{'message': 'Too many requests'}
...

Answer 1

尝试使用asyncio.Semaphore

# Initialize a semaphore object with a limit of 3 (max 3 downloads concurrently)
limit = asyncio.Semaphore(3)


async def make_one_request(url):
    async with limit:
        return await session.get(url, ssl=False)


def get_tasks(self, session):
    tasks = []
    for url in self.list_of_urls:
        tasks.append(asyncio.create_task(make_one_request(url)))
    return tasks


async def get_symbols(self):
    print("Parsing started")
    async with aiohttp.ClientSession() as session:
        tasks = self.get_tasks(session)
        responses = await asyncio.gather(*tasks)
        for response in responses:
            response = await response.json()
            print(response)