不知道什么时候写的了，发上来备份一下吧。
# -*- coding: utf-8 -*-
'''A simple spider fetching data from the url http://www.icourse163.org/search.htm.
@author: uchkks
@file: Spider.py
@time: 2020-2-27 23:47
'''
import requests
import urllib.parse
import json
import time
from typing import Union, Optional, Tuple
__all__ = [
"urlencode",
"urldecode",
"CourseItem",
"spider_page_by_index",
"spider_all"
]
urlencode = urllib.parse.quote
urldecode = urllib.parse.unquote
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"
class CourseItem():
"""The item fetched in the website. Properties are the following:
``title``: The title of this course.
``photourl``: The url of this course photo.
``intro``: The introduction of this course.
``number``: The number of the students attending this course.
``courseurl``: The url of this course.
"""
def __init__(self, title: str, photourl: str, intro: str, number: int, courseurl: str):
self.title = title
self.photourl = photourl
self.intro = intro
self.number = number
self.courseurl = courseurl
def __str__(self):
return json.dumps(self.__dict__)
def __repr__(self):
return json.dumps(self.__dict__)
def newCourseItem(info: dict) -> Tuple[str, CourseItem]:
"""Create an instacne of CourseItem using the given ``info``(a ``dict``).
Returns:
A ``(str, CourseItem)`` will be returned. The first item is the ``courseid``
(A format string, ``'{schoolid}-{courseid}'``), the second item is ``CourseItem.``
"""
courseID = f'{info["schoolPanel"]["shortName"]}-{info["id"]}'
title = info["name"]
photourl = info["termPanel"]["bigPhotoUrl"]
intro = info["termPanel"]["jsonContent"]
number = info["termPanel"]["enrollCount"]
courseurl = 'https://www.icourse163.org/course/' + courseID
return courseID, CourseItem(title, photourl, intro, number, courseurl)
def spider_page_by_index(keyword: Union[str, bytes, bytearray],
pageindex: int = 1,
session: Optional[requests.Session] = None) -> dict:
"""Fetches data from the url www.icourse163.org/search.htm, using
the given ``keyword`` and ``pageindex`` (optional) and returns a dict of result.
Args:
``keyword``: must be a ``str``, ``bytes`` or ``bytearray`` instance, the keyword
you want to lookup in the website.
``pageindex``: optional, an int, the current index of the page, default 1.
``session``: optional, an instance of requests.Session, if it is not provieded,
a new session will be created.
Returns:
A ``dict`` containing search result will be returned.
Raises:
``UnicodeError``: if keyword is a str and startwith the UTF-8 BOM '\ufeff', or
the decode function cannot decode the bytes or bytearray correctly.
``TypeError``: if keyword is not an instance of str, bytes or bytearray, or
pageindex is not an integer.
``RequestException``: an exception from module requests.exception, raises when
an exception occurrs while handling a request.
"""
if isinstance(keyword, str):
if keyword.startswith('\ufeff'):
raise UnicodeError("Unexpected UTF-8 BOM (decode using utf-8-sig)")
else:
if not isinstance(keyword, (bytes, bytearray)):
raise TypeError(f'the keyword must be str, bytes or bytearray, '
f'not {keyword.__class__.__name__}')
if not isinstance(pageindex, int):
raise TypeError(f'the pageindex must be int, '
f'not {keyword.__class__.__name__}')
keyword = keyword.decode()
url = f"http://www.icourse163.org/search.htm?search={urlencode(keyword)}#/"
if session == None:
session = requests.Session()
session.get(url=url, headers={"User-Agent": UserAgent})
csrfKey = session.cookies["NTESSTUDYSI"]
postbody = f'query={{"keyword":"{keyword}","pageIndex":{pageindex}' \
',"highlight":true,"orderBy":0,"stats":30,"pageSize":20}'
headers = session.headers
headers["edu-script-token"] = csrfKey
headers["Host"] = "www.icourse163.org"
headers["Origin"] = "http://www.icourse163.org"
headers["Referer"] = url
headers["Content-Type"] = "application/x-www-form-urlencoded"
res = session.post("https://www.icourse163.org/web/j/mocSearchBean.searchMocCourse.rpc"
f"?csrfKey={csrfKey}", data=postbody.encode(), headers=headers)
text = res.text
resultmap = json.loads(text)
return resultmap
def spider_all(keyword: Union[str, bytes, bytearray]) -> dict:
"""Fetches data from the url www.icourse163.org/search.htm, using
the given ``keyword`` and returns a dict of result.
Args:
``keyword``: must be a ``str``, ``bytes`` or ``bytearray`` instance, the keyword
you want to lookup in the website.
Returns:
A ``dict`` containing search result will be returned.
Raises:
``UnicodeError``: if keyword is a str and startwith the UTF-8 BOM '\ufeff', or
the decode function cannot decode the bytes or bytearray.
``TypeError``: if keyword is not an instance of str, bytes or bytearray.
``ValueError``: if fetching fails.
``RequestException``: an exception from module requests.exception, raises when
an exception occurrs while handling a request.
"""
session = requests.Session()
url = f"http://www.icourse163.org/search.htm?search={urlencode(keyword)}#/"
session.get(url=url, headers={"User-Agent": UserAgent})
resultmap = spider_page_by_index(keyword, session=session)
if resultmap["result"] == None:
raise ValueError(f"The result map is not valid. The error message is {resultmap['message']}")
resultmap = resultmap["result"]
totlePageCount = resultmap["pagination"]["totlePageCount"] + 1
result = {}
for info in resultmap["result"]:
courseID, course = newCourseItem(info["mocCourseCardDto"])
result[courseID] = course
for pageindex in range(2, totlePageCount):
time.sleep(1)
resultmap = spider_page_by_index(keyword, pageindex=pageindex, session=session)
if resultmap["result"] == None:
raise ValueError(f"The result map is not valid. The error message is {resultmap['message']}")
resultmap = resultmap["result"]
for info in resultmap["result"]:
courseID, course = newCourseItem(info["mocCourseCardDto"])
result[courseID] = course
# print(result)
return result
if __name__ == "__main__":
print(spider_all("Python"))
