获取微博关注者的信息并抓取微博里的发布的照片原图

import re,os,urllib,time,requests
from bs4 import BeautifulSoup
from urllib import request

url = "https://weibo.cn/xxxxxxxxx/follow?page="
headers = {"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'}
cookies = {"cookie": "Your cookie"}
dir = "C:\\Users\\xxxxx\\Desktop\\Python\\Weibo"

微博里发布的是单张图片

def single(html,imgdir):
    reg_ori = r'src="(.+?)wap180/.+?\.jpg"' #获取图片url的开头
    reg_end = r'src=".+?/wap180/(.+?\.jpg)"' #获取图片url的结尾
    reg_orire = re.compile(reg_ori)
    reg_endre = re.compile(reg_end)
    url_ori = reg_orire.findall(html)[0]
    url_end = reg_endre.findall(html)[0]
    url = url_ori + "large" + '/' + url_end #生成新的url，把wap180换成large就是照片原图的url
    print(url)
    x = url_end.split('.')[0]
    curdir = imgdir + '\\'
    urllib.request.urlretrieve(url, '{}{}.jpg'.format(curdir, x))  

微博里发布的是多张图片

def group(html,imgdir):
    reg = r'<(a href=".+?">.+?)</a>'
    regre = re.compile(reg)
    lists = regre.findall(html) #找到所有的url
    for i in lists:
        if u'组图' in i: #判断是不是包含照片url
            ureg = r'a href="(https.+?)">'
            uregre = re.compile(ureg)
            gro_url = uregre.findall(i)[0]
            html = requests.get(gro_url,cookies=cookies,headers=headers).text
            img = r'img src="(http.+?\.jpg)"' #抓取照片的url
            imgre = re.compile(img)
            imgurl = imgre.findall(html)
            for u in imgurl:
                s = r'(.+?)thumb180/.+?\.jpg'
                e = r'.+?/thumb180/(.+?\.jpg)'
                ss = re.compile(s).findall(u)[0]
                ee = re.compile(e).findall(u)[0]
                uu = ss + "large" + '/' + ee #将thumb180换成large就是照片原图url
                print(uu)
                curdir = imgdir + '\\'
                x = ee.split('.')[0] #以照片url结尾的字符作为照片的名字保存
                urllib.request.urlretrieve(uu, '{}{}.jpg'.format(curdir, x))
                time.sleep(2)
        time.sleep(2)  

获取所有微博的总页数

def Page(url):
    response = requests.get(url, cookies=cookies, headers=headers)
    reg = r'value="(\d+)"'
    page = int(re.compile(reg).findall(response.text)[0])
    return page  

遍历所有页数的微博

def GetFollow(url,dir):
    ori_url = url + str(1)
    num = Page(ori_url)
    for i in range(1,num+1):
        print("第" + str(i) + "页")
        curr_url = url + str(i)
        try:
            response = requests.get(curr_url, cookies=cookies, headers=headers)
            while response.status_code != 200:
                response = requests.get(curr_url, cookies=cookies, headers=headers)
            soup = BeautifulSoup(response.text,'lxml')
            weibo = soup.find_all('div',class_ = "c")
            for w in weibo:
                i = str(w)
                if u'原图' in i:
                    if u'组图' in i:
                        print("多图")
                        group(i,dir)
                        time.sleep(2)
                    else:
                        print("单图")
                        single(i,dir)
                        time.sleep(2)
        except:
            time.sleep(2)
            continue
        time.sleep(2)  

获取所有关注的人，并抓取粉丝信息，主页网址信息，创建以微博ID为名字的目录，输出基本信息到txt文件中

def GetAll(url):
    for p in range(1,56):
        curr_url = url + str(p)
        html = requests.get(curr_url,cookies=cookies,headers=headers).text
        #print(html)
        soup = BeautifulSoup(html,'lxml')
        follow = soup.find_all("td")
        list = []
        for i in range(1,20,2):
            list.append(follow[i])
        for i in list:
            con = str(i)
            #print(con)
            reg_url = r'td valign="top"><a href="(https://weibo.cn.+?)">.+?</a>' #匹配出微博主页url
            reg_name = r'td valign="top"><a href="https://weibo.cn.+?">(.+?)</a>' #匹配出微博ID
            reg_fans = r'<br/>(.+?)<br/>' #匹配出微博粉丝
            urlre = re.compile(reg_url)
            namere = re.compile(reg_name)
            fansre = re.compile(reg_fans)
            fourl = urlre.findall(con)[0]
            foname = namere.findall(con)[0]
            fofans = fansre.findall(con)[0]
            print(fourl,foname,fofans)
            people_dir = dir + '\\' + foname
            if not os.path.isdir(people_dir):
                os.mkdir(people_dir)
            os.chdir(people_dir)
            file = people_dir + '\\' + foname + ".txt"
            ff = open(file,'at',encoding=('utf-8'))
            out = foname + ' ' + fofans + ' ' + fourl + '\n'
            ff.write(out)
            ff.close()
            full_url = fourl + "?page="
            GetFollow(full_url, people_dir)
            os.chdir(dir)
            time.sleep(5)

GetAll(url)