我之前懂点python的语法，近期发现想写一个简单的爬虫爬取一下网站的图片,在有chatgpt的加持下岂不是分分钟搞定。
1. 在浏览器发起一个请求，生成curl
下面相当于抓到了最原始的、最完整的请求参数
curl 'https://stock.xinpianchang.com/photo/list/p-1?sort=hotDownload&secondCategoryId=99&thirdCategoryId=105' \
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \
-H 'Accept-Language: zh-CN,zh;q=0.9' \
-H 'Cache-Control: max-age=0' \
-H 'Connection: keep-alive' \
-H 'Cookie: _csrf=B3BjIVKe81t7GRFoCSvNDjyl; Device_ID=2cjqm1j7szje; sid=s%3AZd22ifqmMtsu_gNagfEJkR0b-S21Tlzy.k8Lrsi6yKOmLaXMSalPZtRhQxZd8KGHdS4YkoYykIVg; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%7D; Hm_lvt_e62ca320e624d95f87f6991b10628c17=1727350360; HMACCOUNT=0B2C8CA6804CAD41; Authorization=AA906011B43DAE6A2B43DA4C16B43DA984FB43DAC25790C192A0; MEIQIA_TRACK_ID=2mbfnLDrRs4Vy2qG2PBFU9Iccjl; MEIQIA_VISIT_ID=2mbfnGBSBv3eniStWKaLPnvxhZi; acw_tc=0aef82e417273530771207622e00c47a8f906ea8d239b36b407c2537f5102d; Hm_lpvt_e62ca320e624d95f87f6991b10628c17=1727353129' \
-H 'If-None-Match: "35217-1yX7tAQx7fOSSHyUeT8xnlL4Aoc"' \
-H 'Sec-Fetch-Dest: document' \
-H 'Sec-Fetch-Mode: navigate' \
-H 'Sec-Fetch-Site: none' \
-H 'Sec-Fetch-User: ?1' \
-H 'Upgrade-Insecure-Requests: 1' \
-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36' \
-H 'sec-ch-ua: "Google Chrome";v="129", "Not=A?Brand";v="8", "Chromium";v="129"' \
-H 'sec-ch-ua-mobile: ?0' \
-H 'sec-ch-ua-platform: "macOS"''
然后我在命令行执行可以如期获取到数据。
2.让chatgpt生成python代码
import requests
from bs4 import BeautifulSoup
import os
# 创建保存图片的目录
output_dir = 'images'
os.makedirs(output_dir, exist_ok=True)
# 设置请求头，模拟浏览器
# curl_command = f"""curl 'https://stock.xinpianchang.com/photo/list/p-1?sort=hotDownload&secondCategoryId=99&thirdCategoryId=105' \
# -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \
# -H 'Accept-Language: zh-CN,zh;q=0.9' \
# -H 'Cache-Control: max-age=0' \
# -H 'Connection: keep-alive' \
# -H 'Cookie: _csrf=B3BjIVKe81t7GRFoCSvNDjyl; Device_ID=2cjqm1j7szje; sid=s%3AZd22ifqmMtsu_gNagfEJkR0b-S21Tlzy.k8Lrsi6yKOmLaXMSalPZtRhQxZd8KGHdS4YkoYykIVg; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%7D; Hm_lvt_e62ca320e624d95f87f6991b10628c17=1727350360; HMACCOUNT=0B2C8CA6804CAD41; Authorization=AA906011B43DAE6A2B43DA4C16B43DA984FB43DAC25790C192A0; MEIQIA_TRACK_ID=2mbfnLDrRs4Vy2qG2PBFU9Iccjl; MEIQIA_VISIT_ID=2mbfnGBSBv3eniStWKaLPnvxhZi; acw_tc=0aef82e417273530771207622e00c47a8f906ea8d239b36b407c2537f5102d; Hm_lpvt_e62ca320e624d95f87f6991b10628c17=1727353129' \
# -H 'If-None-Match: "35217-1yX7tAQx7fOSSHyUeT8xnlL4Aoc"' \
# -H 'Sec-Fetch-Dest: document' \
# -H 'Sec-Fetch-Mode: navigate' \
# -H 'Sec-Fetch-Site: none' \
# -H 'Sec-Fetch-User: ?1' \
# -H 'Upgrade-Insecure-Requests: 1' \
# -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36' \
# -H 'sec-ch-ua: "Google Chrome";v="129", "Not=A?Brand";v="8", "Chromium";v="129"' \
# -H 'sec-ch-ua-mobile: ?0' \
# -H 'sec-ch-ua-platform: "macOS"'"""
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Cookie': '_csrf=B3BjIVKe81t7GRFoCSvNDjyl; Device_ID=2cjqm1j7szje; sid=s%3AZd22ifqmMtsu_gNagfEJkR0b-S21Tlzy.k8Lrsi6yKOmLaXMSalPZtRhQxZd8KGHdS4YkoYykIVg; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%7D; Hm_lvt_e62ca320e624d95f87f6991b10628c17=1727350360; HMACCOUNT=0B2C8CA6804CAD41; Authorization=AA906011B43DAE6A2B43DA4C16B43DA984FB43DAC25790C192A0; MEIQIA_TRACK_ID=2mbfnLDrRs4Vy2qG2PBFU9Iccjl; MEIQIA_VISIT_ID=2mbfnGBSBv3eniStWKaLPnvxhZi; acw_tc=0aef82e417273530771207622e00c47a8f906ea8d239b36b407c2537f5102d; Hm_lpvt_e62ca320e624d95f87f6991b10628c17=1727353129',
'If-None-Match': '"35217-1yX7tAQx7fOSSHyUeT8xnlL4Aoc"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
'sec-ch-ua': '"Google Chrome";v="129", "Not=A?Brand";v="8", "Chromium";v="129"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
}
# 发起请求
url = "https://stock.xinpianchang.com/photo/list/p-1?sort=hotDownload&secondCategoryId=99&thirdCategoryId=105"
response = requests.get(url, headers=headers)
response.raise_for_status() # 确保请求成功
我直接点击执行，心想百分百成功，竟然提示的是405
image.png
卧槽，这怎么可能，python怎么不能执行，看了一下没有参数，也没有错误呢，debug了一下发现request框架没有加任何其他的参数呢，那服务器是怎么识别到是爬虫的？http参数都在里面了不少呀，难道是Python的代码有问题，我换一个语言吧。
3. go语言版本的可以成功
package main
import (
"fmt"
"io/ioutil"
"net/http"
)
func main() {
// Generated by curl-to-Go: https://mholt.github.io/curl-to-go
// curl 'https://stock.xinpianchang.com/photo/list/p-1?sort=hotDownload&secondCategoryId=99&thirdCategoryId=105' \
// -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \
// -H 'Accept-Language: zh-CN,zh;q=0.9' \
// -H 'Cache-Control: max-age=0' \
// -H 'Connection: keep-alive' \
// -H 'Cookie: _csrf=B3BjIVKe81t7GRFoCSvNDjyl; Device_ID=2cjqm1j7szje; sid=s%3AZd22ifqmMtsu_gNagfEJkR0b-S21Tlzy.k8Lrsi6yKOmLaXMSalPZtRhQxZd8KGHdS4YkoYykIVg; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%7D; Hm_lvt_e62ca320e624d95f87f6991b10628c17=1727350360; HMACCOUNT=0B2C8CA6804CAD41; Authorization=AA906011B43DAE6A2B43DA4C16B43DA984FB43DAC25790C192A0; MEIQIA_TRACK_ID=2mbfnLDrRs4Vy2qG2PBFU9Iccjl; MEIQIA_VISIT_ID=2mbfnGBSBv3eniStWKaLPnvxhZi; acw_tc=0aef82e417273530771207622e00c47a8f906ea8d239b36b407c2537f5102d; Hm_lpvt_e62ca320e624d95f87f6991b10628c17=1727353129' \
// -H 'If-None-Match: "35217-1yX7tAQx7fOSSHyUeT8xnlL4Aoc"' \
// -H 'Sec-Fetch-Dest: document' \
// -H 'Sec-Fetch-Mode: navigate' \
// -H 'Sec-Fetch-Site: none' \
// -H 'Sec-Fetch-User: ?1' \
// -H 'Upgrade-Insecure-Requests: 1' \
// -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36' \
// -H 'sec-ch-ua: "Google Chrome";v="129", "Not=A?Brand";v="8", "Chromium";v="129"' \
// -H 'sec-ch-ua-mobile: ?0' \
// -H 'sec-ch-ua-platform: "macOS"'
req, err := http.NewRequest("GET", "https://stock.xinpianchang.com/photo/list/p-1?sort=hotDownload&secondCategoryId=99&thirdCategoryId=105", nil)
if err != nil {
// handle err
}
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9")
req.Header.Set("Cache-Control", "max-age=0")
req.Header.Set("Connection", "keep-alive")
req.Header.Set("Cookie", "_csrf=B3BjIVKe81t7GRFoCSvNDjyl; Device_ID=2cjqm1j7szje; sid=s%3AZd22ifqmMtsu_gNagfEJkR0b-S21Tlzy.k8Lrsi6yKOmLaXMSalPZtRhQxZd8KGHdS4YkoYykIVg; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%221922e1ac8652b49-0ce856e4e13ed3-16525637-1484784-1922e1ac8662153%22%7D; Hm_lvt_e62ca320e624d95f87f6991b10628c17=1727350360; HMACCOUNT=0B2C8CA6804CAD41; Authorization=AA906011B43DAE6A2B43DA4C16B43DA984FB43DAC25790C192A0; MEIQIA_TRACK_ID=2mbfnLDrRs4Vy2qG2PBFU9Iccjl; MEIQIA_VISIT_ID=2mbfnGBSBv3eniStWKaLPnvxhZi; acw_tc=0aef82e417273530771207622e00c47a8f906ea8d239b36b407c2537f5102d; Hm_lpvt_e62ca320e624d95f87f6991b10628c17=1727353129")
req.Header.Set("If-None-Match", "\"35217-1yX7tAQx7fOSSHyUeT8xnlL4Aoc\"")
req.Header.Set("Sec-Fetch-Dest", "document")
req.Header.Set("Sec-Fetch-Mode", "navigate")
req.Header.Set("Sec-Fetch-Site", "none")
req.Header.Set("Sec-Fetch-User", "?1")
req.Header.Set("Upgrade-Insecure-Requests", "1")
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36")
req.Header.Set("Sec-Ch-Ua", "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"")
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
req.Header.Set("Sec-Ch-Ua-Platform", "\"macOS\"")
resp, err := http.DefaultClient.Do(req)
if err != nil {
// handle err
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
// handle err
fmt.Println("Error reading response body:", err)
return
}
fmt.Println(string(body)) // 打印响应内容
defer resp.Body.Close()
}
我惊讶的发现go语言可以的，我更加摸不到头脑了，似乎世界遇到了bug，随后我将python换了环境，版本都一样的。这他妈是咋回事，见鬼了，再对比还是发现python代码没错误呢。
4. 真相大白
问了其他人也不太懂，没有得到答案，突然想到之前知乎推荐过一个问题：服务器能否拒绝非浏览器发起的HTTP请求？, 但是很多老哥冷嘲热讽或者调侃到不能，因为服务器需要什么参数程序都可以模拟，这一点我也非常熟悉，但是记得一个老哥说有个什么指纹的，虽然有点印象但是自己不是专门做爬虫的，所以就看了一眼就没有多留意。一句话：服务器是通过TLS的指纹识别的，参考：https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/，看评论区说这个专门针对python的，go语言本身没事，具体我也不太懂，基本上应该是通过这个思路实现的。具体的破解办法，参考：https://zhuanlan.zhihu.com/p/601474166?utm_psn=1731683376740827136，或者用Selenium实现
还是要多学习呀，不能墨守成规，遇到问题刨根问底还是有收获的，长见识了。
