前言：
一直想找一个功能不错的fofa爬虫脚本，奈何github上都似乎都不能满足我的需求，我又穷无法充值高级会员，每天只能使用100次的API，只好自己写个批量查询脚本，顺便锻炼下python脚本编写能力。
header信息：
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 OPR/52.0.2871.40',
'Cookie': '_fofapro_ars_session=a497d09beffa61681c2909701e1198d0' # 请输入你的session
}
获得总页数：
def get_page(key):
key_base64 = base64.b64encode(key.encode('utf-8')).decode()
key_base64 = urllib.parse.quote(key_base64)
url = f'https://fofa.so/result?page=1&qbase64={key_base64}'
r = requests.get(url=url, headers=headers, verify=False)
html = r.text
response = HtmlResponse(html, body=html, encoding='utf-8')
selector = Selector(response=response)
for i in [7, 6, 5, 4, 3, 2, 1]:
path_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[11]/div[2]/a[{i}])'
page = selector.xpath(path_xpath).extract()
page = " ".join(page)
if page:
break
if not page:
page = 1
return page
获取查询url链接：
def get_url(key, count):
key_base64 = base64.b64encode(key.encode('utf-8')).decode()
key_base64 = urllib.parse.quote(key_base64)
scanurl = f'https://fofa.so/result?page={count}&qbase64={key_base64}'
return scanurl
通过xpath定位获取数据：
def get_data(xpath):
result = selector.xpath(xpath).extract()
data = " ".join(result)
return data
主功能模块，通过正则获取目标信息：
def scan(target):
global count, page_count, selector
host = ''
count += 1
leave_count = page_count - count - 1
print(f'这是第{count}页的内容，还有{leave_count}页的内容：')
result = ''
r = requests.get(url=target, headers=headers, verify=False)
if headers['Cookie'] in str(r.cookies):
result = True
html = r.text
if '出错了' in html:
print(html)
print('某个地方出现了问题，请查看html代码')
sys.exit()
response = HtmlResponse(html, body=html, encoding='utf-8')
selector = Selector(response=response)
if result:
for i in range(1, 11):
for j in range(1, 3):
host_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[1]/div[1]/a[{j}])'
host_result = get_data(host_xpath)
if host_result:
host = host_result
port_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[2]/div[1]/a)'
title_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[1]/div[2])'
header_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[2]/div[2]/div/div[1])'
certificate_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[2]/div[4])'
server_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[1]/div[8]/a)'
isp_xpath = f'normalize-space(/html/body/div[1]/div[6]/div[1]/div[2]/div[{i}]/div[1]/div[6]/a)'
port = get_data(port_xpath)
title = get_data(title_xpath)
header = get_data(header_xpath)
certificate = get_data(certificate_xpath)
server = get_data(server_xpath)
isp = get_data(isp_xpath)
if port:
port = int(port)
ssl_domain = re.findall(r'(?<=CommonName: ).*(?=Subject Public)', certificate)
ssl_domain = " ".join(ssl_domain).strip()
language = re.findall(r'(?<=X-Powered-By: ).*(?=)', header)
language = " ".join(language).strip()
if 'PHPSESSID' in header and language == '':
language = 'php'
elif 'JSESSIONID' in header and language == '':
language = 'jsp'
try:
ssl_domain = ssl_domain.split(' CommonName: ')[1]
except:
ssl_domain = ''
if not ssl_domain and 'domain=' in header:
ssl_domain = re.findall(r'(?<=domain=).*(?=;)', header)
ssl_domain = " ".join(ssl_domain).strip()
ssl_domain = ssl_domain.split(';')[0]
try:
status = int(header.split(' ')[1].strip())
if status not in [200, 301, 302, 303, 304, 307, 400, 401, 403, 404, 405, 407,
500, 501, 502, 503, 504, 508]:
status = ''
except:
status = ''
if port == '' and status == '' and isp == '' and title == '':
host = ''
print(f'{host} {port} {status} {ssl_domain} {language} {server} {title}')
# print(certificate)
# print(header)
else:
print('cookie无效，请重新获取cookie')
主函数模块
def main():
global count, page_count
count = 0
key = 'title="小程序后台管理系统"'
page_count = get_page(key)
page_count = int(page_count) + 1
for page in range(1, page_count):
url = get_url(key, page)
print(url)
scan(url)
time.sleep(5)
使用条件：拥有fofa会员账号
使用说明：
单线程同步爬取，而且每翻一页需等待5秒，主要是防止被Ban
1.需先自行登录fofa账号，把获取到的cookie值填入脚本中：_fofapro_ars_session
2.需填写查询关键词，脚本里有注释
这只是一个demo，可自行修改，比如生成.csv或者.xls文件，或者写入数据库都行，还可以改成批量查询都没有问题。
我还是习惯用.xls文件查看，效果示图：
代码本身还存在不足的地方，我只是个菜鸟，大佬轻喷~
