网络不给力是安全验证了,我也在学,不过问题我已经解决,你要根据你正常浏览器里的 header 添加,如果实在不会,那就把出现的 所有 header 内容加进去,下面是我测试写的代码,我这边是可以采集的,50多页完全没问题,不会出现网络不给力,自己参考下
import urllib.request
import re
keywords = 'python爬虫'
keywords = urllib.request.quote(keywords)
pages = 0 #页码数
pat_res = [] #定义集合,接收采集标题
fh = open('spider\\title.txt','a') #打开文件,以追加方式
for i,j in enumerate(range(0,50)):
url = '
https://www.baidu.com/s?wd=' + keywords + '&pn='+str(pages)+'&oq=' + keywords + '&ie=utf-8&usm=2&rsv_idx=1&rsv_pq=e4f7528300143ded&rsv_t=f9e5fE0a3Gy7yaLQX6JFJhnR6I1jYnAfL2t5SCcGSoJSAr3LogHutYZxP%2FY'
opener = urllib.request.build_opener()
Bdpagetype = ('Bdpagetype', '2')
Bdqid = ('Bdqid', '0xe4fd4d4f003005ff')
Cache_Control = ('Cache-Control', 'private')
Ckpacknum = ('Ckpacknum', '2')
Ckrndstr = ('Ckrndstr', 'f003005ff')
Connection = ('Connection', 'Keep-Alive')
Content_Encoding = ('Content-Encoding', 'gzip')
Content_Type = ('Content-Type', 'text/html;charset=utf-8')
Strict_Transport_Security = ('Strict-Transport-Security', 'max-age=172800')
Traceid = ('Traceid', '1576546102028799668216500429611456267775')
Transfer_Encoding = ('Transfer-Encoding', 'chunked')
Vary = ('Vary', 'Accept-Encoding')
X_Ua_Compatible = ('X-Ua-Compatible', 'IE=Edge,chrome=1')
Accept = ('Accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9')
Referer = ('Referer', '
https://www.baidu.com')
Sec_Fetch_Mode = ('Sec-Fetch-Mode', 'navigate')
Sec_Fetch_Site = ('Sec-Fetch-Site', 'same-origin')
Sec_Fetch_User = ('Sec-Fetch-User', '?1')
Upgrade_Insecure_Requests = ('Upgrade-Insecure-Requests', '1')
User_Agent = ('User-Agent',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36 Edg/79.0.309.51')
opener.addheaders = [Bdpagetype]
opener.addheaders = [Bdqid]
opener.addheaders = [Cache_Control]
opener.addheaders = [Ckpacknum]
opener.addheaders = [Ckrndstr]
opener.addheaders = [Connection]
opener.addheaders = [Content_Encoding]
opener.addheaders = [Content_Type]
opener.addheaders = [Strict_Transport_Security]
opener.addheaders = [Traceid]
opener.addheaders = [Transfer_Encoding]
opener.addheaders = [Vary]
opener.addheaders = [X_Ua_Compatible]
opener.addheaders = [Accept]
opener.addheaders = [Referer]
opener.addheaders = [Sec_Fetch_Mode]
opener.addheaders = [Sec_Fetch_Site]
opener.addheaders = [Sec_Fetch_User]
opener.addheaders = [Upgrade_Insecure_Requests]
opener.addheaders = [User_Agent]
print('正在爬第:'+str(i)+'页')
data = opener.open(url)
print('当前地址:'+str(data.geturl()))
code = data.read().decode('utf-8')
pat = 'data-tools=\'{"title":"(.*?)"' #提取标题正则
print('成功获取数据,等待入库')
res_temp = re.compile(pat).findall(code)
pat_res += res_temp
print('添加入库成功')
pages = pages+10
print('采集完毕,等待写出文件...')
for w,j in enumerate(pat_res):
fh.write(str(pat_res[w])+'\n')
print('写入文件完毕!')
fh.close()