爬取 http://www.netbian.com/

from bs4 import BeautifulSoup
import requests
import os

headers = {
    'Host': 'www.netbian.com',
    'Referer': 'https://www.netbian.com/',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
}

domain = 'http://www.netbian.com'
category = '/dongman'
baseDir = '/home/opengwe/Desktop/netbian/'

beginPage = 1
endPage = 136

session = requests.session()
session.headers = headers

def getElements(url, selector):
    res = session.get(url)
    res.encoding = res.apparent_encoding
    soup = BeautifulSoup(res.text, 'html.parser')
    return soup.select(selector)

def saveImage(url, path):
    ret = requests.get(url)
    with open(path, "wb") as f:
        f.write(ret.content)

def mkDir(path):
    if not os.path.exists(path):
        os.makedirs(path)

url = "/index.htm"
for i in range(beginPage, endPage):
    if (i > 1):
        url = "/index_%d.htm" % i
    folder = baseDir + 'page%d' % i
    mkDir(folder)
    for item in getElements(domain + category + url, '.list ul li'):
        imghref = item.find('a')['href']
        if (imghref[0] == '/'):
            bighref = getElements(domain+imghref, '#main .endpage .pic a')[0]['href']
            imgs = getElements(domain+bighref, '#main table td img')
            if (imgs):
                imgsrc = imgs[0]['src']
                imgname = imgsrc.split('/')[-1]
                saveImage(imgsrc, '%s/%s' % (folder, imgname))
    print('complete page%d' % i)