速度很快，但写起来繁，最好加上代理
# -*- coding: utf-8 -*-
#使用requests的方法
import csv
import urllib
import sys,requests
import os
import time
import re
import threading
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
##进程锁
#lock=threading.Lock()
###############获取csv文件中的行数(暂时不用管它....因为没用上)############
def raw_num(filepath):
count = 0
cs=open("2017.csv","r",errors="ignore")
reader=csv.reader(cs)
for index,line in enumerate(reader):
count += 1
cs.close()
return count
############模拟登陆############
def post_get(html,name,password):
#浏览器抓包时accept-charset要求使用的是gbk，但python使用的却是utf-8
#name=name.encode("gbk")不可以
grade=1
name=urllib.parse.quote(name.encode("gbk"))
#不编码提交不了，这个地方弄了很久
params="id=%s&name=%s"%(password,name)
html="http://cet.99sushe.com/getscore"+password
header={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
"Host":"cet.99sushe.com",
"Referer":"http://cet.99sushe.com/",
"Cache-Control":"max-age=0",
"Content-Length":"36",
#"cookies":"score=; id=510020162100101",
"Upgrade-Insecure-Requests":"1",
"Origin":"http://cet.99sushe.com",
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding":"gzip, deflate",
"Content-Type":"application/x-www-form-urlencoded",
"Accept-Language":"zh-CN,zh;q=0.8",
"Connection":"keep-alive"}
r=requests.post(url=html,headers=header,data=params)
r=r.text
#print(r)
time.sleep(1)
p=re.compile("[0-9]+")
for i,j in enumerate(p.finditer(r)):
if(i==5):
grade=j.group()
return grade
####################从csv文件读数据############################
def read_out():
global name
global password
cs=open("2017.csv","r",errors="ignore")
reader=csv.reader(cs)
for num,line in enumerate(reader):
name.append(line[6])
password.append(line[5])
#print("准考证号: "+password[len(password)-1]+" 姓名: "+name[len(name)-1])
cs.close()
#######往csv文件写数据(内嵌爬取网页操作)#########
def write_in(i):
global writer
global error
global lock
html="http://cet.99sushe.com"
while(len(password)!=0):
grade=0
print("第%d个线程"%i)
try:
lock.acquire()
n=name.pop()
p=password.pop()
lock.release()
grade=post_get(html,n,p)
print("第%d个线程:"%i)
print("准考证号: "+p+" 姓名: "+n)
print(grade)
lock.acquire()
writer.writerow([n,p,grade])
lock.release()
except Exception as e:
print("/****error in grade scraping.")
print("第%d个线程:"%i)
print("准考证号: "+p+" 姓名: "+n+"****/")
print(e)
lock.acquire()
error.writerow([n,p])
lock.release()
continue
###########主函数####################
##存储堆栈
threads=[]
password=[]
name=[]
##读取数据到堆栈
read_out()
##############
###线程个数
thread_num=2
###csv文件指针等.....####
html="http://cet.99sushe.com"
error_file=open("error.csv","w",newline="",errors="ignore")
csv_file=open("result.csv","w",newline="",errors="ignore")
writer=csv.writer(csv_file)
error=csv.writer(error_file)
writer.writerow("空行!")
error.writerow("空行!")
###########多进程部分###############
for i in range(thread_num):
print("第%d个线程入栈:"%i)
t=threading.Thread(target=write_in,args=(i,))
#因为是args=(A,B)所以之前写出arge=(driver)结果炸了
threads.append(t)
for i in range(thread_num):
print("start第%d个线程"%i)
threads[i].start()
for i in range(thread_num):
threads[i].join()
#####################################
print("一共花了:")
print(time.clock())
csv_file.close()
error_file.close()
