上次留了个坑，采取两种方式对需要爬取的网站进行数据抓取。首先使用selenium+chromedriver进行抓取，这种方式速度慢，但是可以跳过各种反爬设置，并对各种反爬能够直接以网页的形式进行显示。因此我们可以采取这种方式进行测试，找到其中的反爬规则，然后采用通用爬虫进行爬取。今天下午闲着没事对boss直聘进行了测试

第一种:selenium+chromedriver

鉴于代码不长，就直接贴代码了：

import pymysql
import re
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from lxml import etree
import time
from datetime import datetime,timedeltapath=r'I:\code\selenimu_chromedriver\chromedriver_win32\chromedriver.exe'
class LagouSpider(object):path = r'I:\code\selenimu_chromedriver\chromedriver_win32\chromedriver.exe'def __init__(self):self.dicts={}self.driver=webdriver.Chrome(executable_path=LagouSpider.path)self.url='https://www.zhaopin_spider.com/jobs/list_%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98?labelWords=&fromSearch=true&suginput='def run(self):self.driver.get(self.url)time.sleep(50)# 在调用的chromedriver中提前登录，避免爬取过程中被要求登录while True:#  显示等待‘下一页’避免爬取太快获取不到该元素WebDriverWait(driver=self.driver,timeout=15).until(EC.presence_of_element_located((By.XPATH,"//div[@class='pager_container']/span[last()]")))next_page=self.driver.find_element_by_xpath("//div[@class='pager_container']/span[last()]")#以下这种方法会丢失最后一页数据，可以使用for循环，先查看总页数总数，避免数据丢失if 'pager_next pager_next_disabled' not in next_page.get_attribute('class'):passelse:source = self.driver.page_sourceself.parse_list_source(source)next_page.click()def parse_list_source(self,source):html=etree.HTML(source)list_url=html.xpath("//div[@class='list_item_top']/div[@class='position']/div[@class='p_top']/a/@href")for link in list_url:self.request_detail_source(link)print(link)time.sleep(3)def request_detail_source(self,url):self.driver.execute_script("window.open('%s')"%url)# 打开新的窗口self.driver.switch_to.window(self.driver.window_handles[1])# 切换到新的窗口source=self.driver.page_sourcesource=etree.HTML(source)salary = source.xpath("//dd[@class='job_request']/p[1]/span[@class='salary']/text()")# 该if-else 是为了避开验证信息，如果没有筛选到'salary'元素直接关掉该页面，缺点是丢失一页数据if len(salary) ==0:self.driver.close()self.driver.switch_to.window(self.driver.window_handles[0])else:salary=source.xpath("//dd[@class='job_request']/p[1]/span[@class='salary']/text()")[0].strip()adress=source.xpath("//dd[@class='job-address clearfix']/div[@class='work_addr']/a/text()")adress.pop(-1)adress="".join(adress)company = source.xpath("//h2[@class='fl']/em[@class='fl-cn']/text()")[0].strip()experience=source.xpath("//dd[@class='job_request']/p[1]/span[3]/text()")[0].strip('/').strip()education=source.xpath("//dd[@class='job_request']/p[1]/span[4]/text()")[0].strip('/').strip()tempting = source.xpath("//dd[@class='job-advantage']/p/text()")[0]# 以下是将requirement划分为responsibility和demand，由于这部分内容无特定格式，可能会丢失数据requirement = source.xpath("//dl[@id='job_detail']/dd[@class='job_bt']/div//text()")#  这种替换方法比较蠢，因此不用#temporary_variable = 0# for i in requirement:#     i = str(i).strip()#     requirement[temporary_variable] = i#     temporary_variable += 1requirement = "".join(requirement)requirement=re.sub('\n','',requirement).strip()requirement=re.sub(r"\xa0","",requirement)a = re.split(r'(工作要求|任职要求|任职资格|岗位要求|岗位职责|工作职责|岗位职能|我们期望|工作内容)', requirement, 3)demand=0responsibility=0if len(a)==5:if a[1] in ("岗位职责","工作职责","岗位职能",'工作内容','关于职位'):# 判断条件用in 而非 orresponsibility=a[1] + a[2]elif a[1]==0:demand=a[1] + a[2]if a[3] in ("岗位职责","工作职责","岗位职能",'工作内容','关于职位'):#  带上括号控制优先级responsibility = a[3] + a[4]else:demand = a[3] + a[4]else:pass# 对取到的三种时间格式进行处理，最终得到发布日期（没有精确到分，为了格式统一）times = source.xpath("//p[@class='publish_time']/text()")[0]start_time = re.match('(\d\d:\d\d)|(\d天前)|(\d*-\d*-\d*)', times)start_time=start_time.group()if len(start_time) == 3:now = datetime.now()start_time = str(now - timedelta(days=int(list(start_time)[0])))elif len(start_time) == 5:start_time = str(datetime.now())else:pass#print("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s"%(salary,adress,company,experience,tempting,responsibility,demand,start_time))# 存入数据库conn=pymysql.connect(host="localhost",port=3306,user='root',password='as123456',database='zhaopin_spider',charset='utf8')cursor=conn.cursor() #创建游标对象sql = "INSERT INTO data_analysis(salary,adress,company, experience, tempting, responsibility,demand,start_time,url)\VALUES ('%s','%s','%s','%s','%s','%s','%s',cast('%s' as datetime),'%s')"%(salary,adress,company,experience,tempting,responsibility,demand,start_time,str(url))cursor.execute(sql)conn.commit()self.driver.close()self.driver.switch_to.window(self.driver.window_handles[0])if __name__=='__main__':lagou=LagouSpider()lagou.run()
#

第二种:基于httpclient

从上面的测试来看，boss直聘基本没设置什么反爬措施，甚至连cookie都不用传，这样就比较简单了，代码不长，直接上代码了：

import requests
from lxml import etree
class ZhiPin(object):def __init__(self):self.header={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134","Accept-Language": "zh-Hans-CN,zh-Hans;q=0.5","Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Upgrade-Insecure-Requests": "1",#"Accept-Encoding": "gzip, deflate, br","Host": "www.zhipin.com",#"Connection": "Keep-Alive"}self.add_url='https://www.zhipin.com'self.start_url = 'https://www.zhipin.com/c100010000/?query=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98&page=1&ka=page-2'def run(self):response = requests.get(self.start_url, headers=self.header)data = etree.HTML(response.text)  # etree.HTML解析字符串类型数据self.get_urls(data)self.get_lists_page(data)def get_lists_page(self,data):next_page_url = data.xpath("//div[@class='job-list']/div[@class='page']/a[@class='next']/@href")if len(next_page_url) != 0:next_page_url=next_page_url[0]print(next_page_url)if '/c100010000' in next_page_url:next_page_url = self.add_url + next_page_urlself.start_url=next_page_urlself.run()else:print(data.text)else:print('爬完了')def get_urls(self,data):urls=data.xpath("//div[@class='info-primary']/h3[@class='name']/a/@href")for url in urls:url=self.add_url+urlself.url=urlself.get_detial_page(url)def get_detail_page(self,url):print(url)if __name__=="__main__":zhipin=ZhiPin()zhipin.run()

注意上面代码没有爬取详情页具体内容，只是进入详情页测试了一下，需要爬具体内容的可以自己写哈

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！