某房源网信息爬虫

import random
import re
import requests
import time
from lxml import etree
from pymongo import MongoClient
from multiprocessing import Pool
from requests.exceptions import RequestExceptiondef get_response(url):'''给出url,获取网页响应'''user_agent = 'Mozilla/5.0 (Windows NT 6.'cookie = '5Od66EiO2sfsd‘try:response = requests.get(url, headers=headers).textreturn responseexcept RequestException:return Nonedef get_one_page(response):'''利用正则匹配，发送请求，获取整页源代码'''pattern = re.compile(r'class="houselist-mod houselist-mod-new"(.*?)', re.S)item = pattern.findall(response)return itemdef get_content(item):'''利用XPath，正则，解析网页内容'''selector = etree.HTML(item)  # 将源码转化为能被XPath匹配的格式title = selector.xpath('//div[@class="house-details"]/div[@class="house-title"]/a/@title')  # 返回为一列表pattern = re.compile(r'.*?(.*?).*?class="spe-lines".*?(.*?)'r'.*?class="spe-lines".*?(.*?).*?class="spe-lines".*?(.*?).*?'r'class="brokername">.*?.*?class="price-det".*?(.*?)'r'.*?class="unit-price">(.*?)', re.S)item1 = pattern.findall(item)address = selector.xpath('//div[@class="details-item"]/span[@class="comm-address"]/@title')for i, name, add in zip(item1, title, address):yield name, i[0], i[1], i[2], i[3], i[4]+'万', i[5], adddef write_to_DB(title, hx, size, lc, build, price, u_price, address):'''保存数据到mongoDB数据库'''client = MongoClient('localhost', 27017)  # 链接数据库db = client['House']db.House.save({"介绍": title, "户型": hx, "大小": size, "楼层": lc,"年代": build, "价钱": price, "单价": u_price, "地址": address})def main(page):url = 'https://beijing.anjuke.com/sale/p{num}/#filtersort'for p in [page]:url = url.format(num=p)response = get_response(url)time.sleep(random.randint(5, 20))one_page = get_one_page(response)title = get_content(str(one_page))for i in title:write_to_DB(i[0], i[1], i[2], i[3], i[4], i[5], i[6], i[7])print('正在爬取: %s' % i[0])if __name__ == "__main__":'''开启5个进程加快信息爬取'''start_time = time.time()pool = Pool(processes=5)page = [x+1 for x in range(5)]pool.map(main, page)pool.close()            # 关闭进程池，表示不能在往进程池中添加进程pool.join()             # 等待进程池中的所有进程执行完毕，必须在close()之后调用end_time = time.time()print("所用时间: %d 秒" % (end_time - start_time))print("爬取结束!")