异步爬虫实战——爬取西游记小说

Python异步爬虫基础知识：异步爬虫

使用异步爬取西游记

import json

import requests
import asyncio
import aiohttp  # pip install aiohttp
import aiofiles  # pip install aiofiles


async def getCatalog(url):
    """
    获取小说的章节目录
    :param url: 获取小说的章节目录的URL
    :return: 
    """
    resp = requests.get(url)
    data = resp.json()['data']['novel']
    print(data)
    tasks = []  # 异步任务列表
    # 获取每一个章节的id获取小说内容
    for item in data['items']:
        cid = item['cid']
        title = item['title']
        tasks.append(asyncio.create_task(getChapterContent(title, cid)))
        # break
    await asyncio.wait(tasks)


async def getChapterContent(title, cid):
    """
    根据章节id获取小说内容
    :param title: 章节名称
    :param cid: 章节id
    :return:
    """
    data = {
        'book_id': book_id,
        'cid': book_id + '|' +cid,
        'need_bookinfo': 1
    }
    # 获取章节内容的url
    url = "https://dushu.baidu.com/api/pc/getChapterContent?data=" + json.dumps(data)
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as resp:
            data = await resp.json()
            print(data)
            # 将章节内容写入文件（异步形式写入）
            async with aiofiles.open(f'西游记/{title}', mode='w', encoding='utf-8') as f:
                await f.write(data['data']['novel']['content'])


if __name__ == '__main__':
    book_id = '4306063500'  # 书本id
    data = {'book_id': book_id}
    # 获取章节目录的URL
    url = 'https://dushu.baidu.com/api/pc/getCatalog?data=' + json.dumps(data)
    asyncio.run(getCatalog(url))