asyncio,aiohttp

import aiohttp
import asyncio

def get_ip():
    ips = ["121.231.215.111:4276",
           "220.189.98.172:4236",
           "123.186.228.166:4223",
           "121.226.45.47:4212",
           "36.34.27.126:4270",
           "111.73.243.185:4232",
           "183.165.11.222:4235",
           "101.206.239.90:4284",
           "60.188.42.243:4257",
           "115.152.142.60:4213",
           "115.211.189.113:4231",
           "117.91.115.10:4282",
           "49.77.211.155:4263",
           "115.239.21.66:4264",
           "114.103.28.160:4265", ]
    return "http://" + random.choice(ips)


def get_headers():
    USER_AGENTS = [
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
        "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
        "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
        "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
        "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
        "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
        "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
        "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    ]
    return {"User-Agent": random.choice(USER_AGENTS)}


async def spider(sem):
    async with sem:
        async with aiohttp.ClientSession() as session:
            for i in list(db.starcomm.find().sort([("_id", 1)])):
                url = "https://m.weibo.cn/api/container/getIndex?type=uid&value={uid}&containerid=100505{uid}"
                async with session.get(url.format(uid=i.get("uid")), headers=get_headers(),
                                       proxy=get_ip(), ssl=False) as resp:
                    try:
                        data = await resp.json()
                    except Exception as e:
                        print(e)
                for id in i.get("community"):
                    async with session.get(url.format(uid=id), headers=get_headers(), proxy=get_ip(),
                                           ssl=False) as resp:
                        try:
                            data = await resp.json()
                            print(data)
                        except Exception as e:
                            print(e)
                    await asyncio.sleep(2)




def main():
    loop = asyncio.get_event_loop()
    sem = asyncio.Semaphore(100)  # 控制线程最大连接数,防止session过多报错
    tasks = [asyncio.ensure_future(spider(sem))]
    loop.run_until_complete(asyncio.gather(*tasks))
    loop.close()

# main()

import asyncio
import time


now = lambda: time.time()

async def do_some_work(x):
    print("waiting:",x)
    await asyncio.sleep(x)
    return "Done after {}s".format(x)

async def main():
    coroutine1 = do_some_work(1)
    coroutine2 = do_some_work(2)
    coroutine3 = do_some_work(4)
    tasks = [
        asyncio.ensure_future(coroutine1),
        asyncio.ensure_future(coroutine2),
        asyncio.ensure_future(coroutine3)
    ]
    return await asyncio.gather(*tasks)
    #return await asyncio.wait(tasks)也可以使用。注意gather方法需要*这个标记

start = now()

loop = asyncio.get_event_loop()
results = loop.run_until_complete(main())
for result in results:
    print("Task ret:",result)

print("Time:", now()-start)