python查询各搜索引擎收录情况

# -*- coding: utf-8 -*-
import requests_html
import json
import time# 格式化数值函数
def num(num):nums = 0num = num.replace('找到相关结果约','').replace('个','')num = num.replace('找到约','').replace('条结果','')num = num.replace(',','').replace(' ','')if '亿' in num:numr = num.split('亿')nums += int(numr[0])*10000*10000num = numr[1]if '万' in num:numr = num.split('万')nums += int(numr[0])*10000num = numr[1]if not num:num=0nums += int(num)return nums# 请求接口
QueryUrlBaidu = 'https://www.baidu.com/s?ie=utf-8&wd=%s'
QueryUrlHaosou = 'https://www.so.com/s?q=%s'
QueryUrlHaosouM = 'https://m.so.com/s?q=%s'
QueryUrlSogou = 'https://www.sogou.com/web?query=%s'
QueryUrlShenma = 'https://m.sm.cn/s?q=%s'# 创建请求
# cookie属于百度需求参数，万一失效可以去复制一下即可
s = requests_html.HTMLSession()
s.headers.update({'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36','Cookie': ''\'PSTM=1578469724; BAIDUID=0401106C23030C7C6758A2EEDC00E967:FG=1; delPer=0; BD_CK_SAM=1; PSINO=3; H_PS_PSSID=1459_21103_30210_30496_26350_22159; BIDUPSID=50F03331BB9348FA0982DB22EE7F245E; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BD_UPN=12314353; H_PS_645EC=077dTzRh%2FuK7a64A8%2BHXcVlxSDZpdSuxDYLrTquT1QClk0BRAmSqjmPn5ek'\'__guid=34870781.1471611762366714400.1578476949585.0554; QiHooGUID=2FFB2A68E2B5180C7D8EDC555794A7EE.1578477208864; _S=d0c11butf4e2pg07naoo0j4b91; opqopq=b94e4818125a8e53b6c4c258100f3e23.1578477208; count=1; dpr=1; webp=1; __huid=11NUowBiKSdaKDraeZ71uFZk%2FmOKYjfZl0CjTkGIqGMzw%3D; gtHuid=1'\'ABTEST=4|1578477296|v17; SNUID=F484FCC2D8DC43453A49B7D8D93A1423; IPLOC=CN4406; SUID=2C5C241B2513910A000000005E15A6F0; ld=VZllllllll2WVzbZlllllVcqIsllllllphStalllll9llllljllll5@@@@@@@@@@; SUV=00AD855B1B245C2C5E15A6F13B70B719; browerV=3; osV=1'\,
})# 抽取收录数函数，考虑量比较小，此函数不考虑效率问题
def getnum(url,sname=''):if not url:print('[warnning]','请设置url')try:r = s.get(url)# r.encoding = 'utf-8'if sname=='Baidu':text = r.html.find('.c-border b',first=True)elif sname=='Haosou':text = r.html.find('span.nums',first=True)elif sname=='HaosouM':text = r.html.find('p.site-s-result strong',first=True)elif sname=='Sogou':text = r.html.find('p.sr-num',first=True)elif sname=='Shenma':text = r.html.find('div.site-body p i',first=True)else:print('[warnning]','没有匹配到结果',url)return 0if text :return num(text.text)else:print('[warnning]','没有匹配到结果,通常为反爬限制',url)return 0except BaseException as e:# print(r.text)print('[error]','抽取失败',url)print('[error]','抽取失败详情',e)return 0# 主体函数
def checksite(domainarr=[],sleep=0,filename=''):# 判断是否有参数if not domainarr:print('[warnning]','请设置domain')returnprint('[running]','共获取域名数：',len(domainarr))# 构建请求连接data = []for domain in domainarr:query = 'site:%s'%domain# 百度num = getnum(QueryUrlBaidu%query,'Baidu')print('[running]',domain,'Baidu',num)data.append({'domain':domain,'search':'baidu','value':num,})# 搜狗num = getnum(QueryUrlSogou%query,'Sogou')print('[running]',domain,'Sogou',num)data.append({'domain':domain,'search':'sogou','value':num,})# 神马num = getnum(QueryUrlShenma%query,'Shenma')print('[running]',domain,'Shenma',num)data.append({'domain':domain,'search':'shenma','value':num,})# 好搜,双端接口参数不一样if domain.split('.')[0] == 'm' : num = getnum(QueryUrlHaosouM%query,'HaosouM')else:num = getnum(QueryUrlHaosou%query,'Haosou')print('[running]',domain,'Haosou',num)data.append({'domain':domain,'search':'haosou','value':num,})# 休眠间隔if sleep:print('[running]','休眠：',sleep)time.sleep(sleep)# print(json.dumps(data,indent=4))print('[running]',json.dumps(data))if filename:print('[running]','执行保存文件',filename)# 保存新数据with open(filename, 'w', encoding='utf-8') as filejson:filejson.write(json.dumps(data))if __name__ == '__main__':domainr = ['www.seowhy.com',# 'www.aizhan.com',# 'm.seowhy.com',# 'm.aizhan.com',]sleep = 0date = time.strftime("%Y%m%d",time.localtime(int(time.time())))filename = 'site.%s.json'%datechecksite(domainr,sleep,filename)# 脚本说明：#     python3.6版本#     需要安装requests-html模块 pip install requests_html#     搜狗经常反爬，不宜大批量域名查询#     百度m端没有查询数值 Baidu#     搜狗双端结果相同 Sogou#     好搜双端不同 Haosou#     神马只有m端 Shenma#     parama  domainarr   array   查询域名数组#     parama  sleep       intval  查询循环休眠时间#     parama  filename    string  保存文件名称# by 薛一# 2018-07-08

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！