python 3.7使用aiohttp写爬虫来抓取数据

# !/usr/bin/env python
# encoding: utf-8

"""
file: gbk_test.py
time: 2019/7/9 14:13
Author: twy
"""
import aiohttp
import asyncio

async def test_aio(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            # 这里有个坑：使用GBK编码的时候，一般要加上ignore，不然可能会报错
            print(await response.text("GBK", 'ignore'))


# python 3.7用法: gather 
async def main():
    await asyncio.gather(*[test_aio(url="https://www.dytt8.net/html/gndy/dyzz/list_23_2.html") for i in range(2)])


# python 3.7的写法，使用新api：run
def start():
    asyncio.run(main())


# python 3.6的写法使用get_event_loop API
def main_py36():
    loop = asyncio.get_event_loop()
    tasks = [asyncio.ensure_future(test_aio('https://www.dytt8.net/html/gndy/dyzz/list_23_2.html')) for i in range(2)]
    loop.run_until_complete(asyncio.wait(tasks))


if __name__ == "__main__":
    main_py36()
    start()

更多有趣的爬虫demo可以看这里