爬取伯乐在线

标签（空格分隔）： python scrapy

项目创建

• pycharm 本身是不会自带 scrapy 框架的

# 虚拟环境安装
mkvirtualenv --python=python地址 article_spider
# 跳转虚拟环境下 下载scrapy
pip install -i https://pypi.douban.com/simple/ scrapy
# 这里排个坑 我本地安装失败了 是由于twisted 有问题 访问下载 http://www.lfd.uci.edu/~gohlke/pythonlibs/
pip install 下载好的文件直接拖进来(C:\Users\JQ\Downloads\Twisted-17.9.0-cp35-cp35m-win_amd64.whl)
pip install -i https://pypi.douban.com/simple/ scrapy
# 创建项目 一般习惯项目路径 我这里放在e盘
workon article_spider
scrapy startproject ArticleSpider
# 创建文件 起始页 
scrapy genspider jobbole blog.jobbole.com
# 这里项目默认的环境设置

页面内容提取方式

• 项目调试文件配置

main.py # 主项目目录下

from scrapy.cmdline import excute # 调试执行

import sys # 系统文件
import os # 内容

sys.path.append(os.path.dirname(os.path.abspath(__file__)))
excute(["scrapy","crawl","jobbole"])

• 简介</br>
  1.xpath使用路径表达式在xml和html中进行导航.</br>
  2.xpath包含标准函数库.</br>
  3.xpath是一个w3c的标准.

• xpath选择器

注意上面后三个表格特殊符号|用\代替.

• css选择器

注意提取内容用伪类选择器"::text".

伯乐在线单文页面提取

• shell调试

scrapy shell http://blog.jobbole.com/112713/

• xpath页面信息提取

title = response.xpath('//*[@id="post-112713"]/div[1]/h1/text()').extract()
create_time = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/text()').extract()[0].strip().replace("·","").strip()
praise_nums = response.xpath('//*[contains(@class,"vote-post-up")]/h10/text()').extract()[0]
fav_nums = response.xpath('//*[contains(@class,"bookmark-btn")]/text()').extract()[0]

regex_nums = ".*?(\d+).*"
match_fav = re.match(regex_nums,fav_nums)
if match_fav:
    fav_nums = match_fav.group(1)
else:
    fav_nums = 0

comment_nums = response.xpath('//a[@href="#article-comment"]/span/text()').extract()[0]
match_comment = re.match(regex_nums,comment_nums)
if match_comment:
    comment_nums = match_comment.group(1)
else:
    comment_nums = 0

content = response.xpath('//div[@class="entry"]').extract()[0]
tag_list = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/a/text()').extract()
tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
tags = ",".join(tag_list)

• css页面信息提取

title = response.css('.entry-header h1::text').extract()[0]
create_time = response.css('.entry-meta-hide-on-mobile::text').extract()[0].strip().replace('·','').strip()
praise_nums = response.css('.vote-post-up h10::text').extract()[0]
fav_nums = response.css('.bookmark-btn::text').extract()[0]
mactch_fav = re.match(regex_nums,fav_nums)
if mactch_fav:
    fav_nums = mactch_fav.group(1)
else:
    fav_nums = 0
comment_nums = response.css("a[href='#article-comment'] span::text").extract()[0]
match_comment = re.match(regex_nums, comment_nums)
if match_comment:
    comment_nums = match_comment.group(1)
else:
    comment_nums = 0

content = response.css('div.entry').extract()[0]
tag_list = response.css('.entry-meta-hide-on-mobile a::text').extract()
tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
tags = ",".join(tag_list)

提取伯乐在线所有文章

• 逻辑实现：1. 获取文章的url并交给scrapy进行解析。2.获取下一页url并交给scrapy进行下载，下载后交给parse

• request 访问并获取页面内容并通过yield进行下载

from scrapy.http import Request
Resquest(url=,collback=)

• parse 对于href中链接不是全链接

from urllib import parse python3
import urlparse python2
parse.urljoin(response.url,post_url)

数据入库

• item 类似于字典 但优于字典

class ArticleItem(scrapy.Item):
    title = scrapy.Filed()
    

• 通过节点获取节点下的多个内容

 post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            yield Request(parse.urljoin(response.url,post_url),meta={"format_image_url":image_url},callback=self.parse_detail)
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            yield Request(parse.urljoin(response.url,next_url),callback=self.parse)

 # 封面图传递
fornat_image_url = response.meta.get("fornat_image_url","")

• 图片保存

pip install -i https://pypi.douban.com/simple pillow

# 文章爬去py文件
article_items["title"] = title
        article_items["url"] = response.url
        article_items["create_time"] = create_time
        article_items["praise_nums"] = praise_nums
        article_items["fav_nums"] = fav_nums
        article_items["comment_nums"] = comment_nums
        article_items["content"] = content
        article_items["fornat_image_url"] = [fornat_image_url]
        article_items["tags"] = tags
        yield article_items # 这个很重要 保证内容进入管道
        
# setting 文件        
ITEM_PIPELINES = {
   'ArticleSpider.pipelines.ArticlespiderPipeline': 300,
    'scrapy.pipelines.images.ImagesPipeline':1, # 文件存储类库 1代表级别 越低则越靠前运行
}
IMAGES_URLS_FIELD = "fornat_image_url" # url链接地址
project_dir = os.path.abspath(os.path.dirname(__file__))
IMAGES_STORE = os.path.join(project_dir,'images') #文件存储位置

• 存储图片路径存储

class ArticleImagePipeline(ImagesPipeline):
    # 重写提取文件存储路径
    def item_completed(self, results, item, info):
        for ok,value in results:
            image_file_path = value['path']
            item["fornat_image_path"] = image_file_path
        return item

• url进行md5(摘要算法)压缩

def get_md5(url):
    if isinstance(url,str):
        url = url.encode("utf-8")
    m = hashlib.md5()
    m.update(url)
    return m.hexdigest()

• 内容转换为json文件保存

  1.codecs 避免打开文件时出现编码错误
  
  2.json.dumps : dict转成str
  
  3.json.loads: str转成dict
  
  4.ensure_ascii=False: 避免处理英文以外语言时出错
  
  5.return item：交给下一个pipeline处理

# 自定义json文件导出
class JsonWithArticlePipeline(object):
    def __init__(self):
        self.file = codecs.open('article.json','w',encoding='utf-8')

    def process_item(self, item, spider):
        lines = json.dumps(dict(item),ensure_ascii= False)+'\n'
        self.file.write(lines)
        return item

    def spider_closed(self,spider):
        self.file.close()

# 调用scrapy的json文件导出
class JsonExporterPipeline(object):
    def __init__(self):
        self.file = open('articleexporter.json','wd')
        self.exporter = JsonItemExporter(self.file,encoding='utf-8',ensure_ascii = False)
        self.exporter.start_exporting()
    def closed_spider(self,spider):
        self.exporter.finish_exporting()
        self.file.close()
    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item

• 内容入库

1.安装mysql驱动

pip install mysqlclient


# 会出现数据堵塞
class MysqlPipeline(object):
   def __init__(self):
       self.conn = MySQLdb.Connect("127.0.0.1","root","123456","article_spider",charset="utf8",use_unicode = True)
       self.cursor = self.conn.cursor()

   def process_item(self, item, spider):
       insert_sql = """
           insert into article(title,url,url_object_id,create_date,
           praise_nums,fav_nums,comment_nums,content,tags,fornat_image_url,
           fornat_image_path) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
       """
       self.cursor.execute(insert_sql,(item["title"],item["url"],item["url_object_id"],item["create_time"],
                                       item["praise_nums"],item["fav_nums"],item["comment_nums"],
                                       item["content"],item["tags"],item["fornat_image_url"],
                                       item["fornat_image_path"]))
       self.conn.commit()

2.数据堵塞解决方案

# 异步方式进行数据插入
class MysqlTwistedPipeline:
    def __init__(self,dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls,settings):
        dbparms = dict(
            host = settings["MYSQL_HOST"],
            db = settings["MYSQL_DBNAME"],
            user = settings["MYSQL_USER"],
            passwd = settings["MYSQL_PASSWORD"],
            charset = 'utf8',
            cursorclass = MySQLdb.cursors.DictCursor,
            use_unicode = True,
        )
        dbpool =  adbapi.ConnectionPool("MySQLdb",**dbparms)
        return cls(dbpool)

    def process_item(self, item, spider):
        query = self.dbpool.runInteraction(self.insert_sql,item)
        query.addErrback(self.handle_error) # 处理异常

    def handle_error(self,failure):
        print(failure)

    def insert_sql(self,cursor,item):
        insert_sql = """
                    insert into article(title,url,url_object_id,create_date,
                    praise_nums,fav_nums,comment_nums,content,tags,fornat_image_url,
                    fornat_image_path) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
                """
        cursor.execute(insert_sql, (item["title"], item["url"], item["url_object_id"], item["create_time"],
                                         item["praise_nums"], item["fav_nums"], item["comment_nums"],
                                         item["content"], item["tags"], item["fornat_image_url"],
                                         item["fornat_image_path"]))

项目优化

• itemloader

from scrapy.loader import ItemLoader

item_loader = ItemLoader(item = 字典,response )

item_loader.addcss("","") # 分别为字典 以及 提取类型
item_loader.addcss("","")
item_loader.addvalue("", ) # 这里则为获取的值

• 原视频UP主慕课网（聚焦Python分布式爬虫必学框架Scrapy 打造搜索引擎）
• 本篇博客撰写人: XiaoJinZi 个人主页 转载请注明出处
• 学生能力有限 附上邮箱: 986209501@qq.com 不足以及误处请大佬指责