管道模块
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql
class MyspiderPipeline(object):
def __int__(self):
print('my pipelines is ready....__init__ invoking....')
def open_spider(self,spider):
'''
建立数据库连接
:paramspider:
:return:
'''
self.my_conn = pymysql.connect(
host ="localhost",
port =3306,
database ="jobs",
user ='root',
password ='123',
charset ='utf8'
)
#获取游标对象
self.my_cursor =self.my_conn.cursor()
def process_item(self,item,spider):
print('----------------------process_item is invoking----------------------')
# print item['name']
# print item['salary']
# print item['company']
# print item['day']
# print item['experience']
# print item['area']
# print item['number']
# print item['nature']
# print item['education']
# print item['description']
insert_sql ="insert into wuxi(name, salary, company, day,experience,area,number,nature,education,description) values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
self.my_cursor.execute(insert_sql,[item['name'],item['salary'],item['company'],item['day'],item['experience'],item['area'],item['number'],item['nature'],item['education'],item['description']])
def close_spider(self, spider):
#提交MySQL语句
self.my_conn.commit()
#关闭游标对象与数据库连接
self.my_cursor.close()
self.my_conn.close()
items
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class MyspiderItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
name = scrapy.Field()
salary = scrapy.Field()
company = scrapy.Field()
day = scrapy.Field()
experience = scrapy.Field()
area = scrapy.Field()
number = scrapy.Field()
nature = scrapy.Field()
education = scrapy.Field()
description = scrapy.Field()
