使用selenium 模拟爬取谷歌图片代码实例）
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created on 2019/02/15
@filename: get_pic_from_google.py
@author: sdhzdtwhm
Description:1.此脚本为使用selenium模拟访问爬取谷歌图片中的搜索结果2.运行环境为python3 需要安装selenium、bs4、requests库3.访问谷歌环境需要FQ4.需要下载谷歌浏览器的驱动
"""
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
from bs4 import BeautifulSoup as bs
import uuid
import requests
import osclass Crawler:def __init__(self):self.url = base_url_part1 + search_query + base_url_part2"""启动Chrome浏览器驱动"""def start_brower(self):chrome_options = Options()chrome_options.add_argument("--disable-infobars")"""谷歌浏览器驱动地址"""executable_path = "C:\\Windows\\System32\\chromedriver.exe""""启动Chrome浏览器"""driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=executable_path)"""最大化窗口，因为每一次爬取只能看到视窗内的图片"""driver.maximize_window()"""浏览器打开爬取页面"""driver.get(self.url)return driverdef downloadImg(self, driver):headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ''(KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36','Connection': 'keep - alive','content-type': 'application/json'}"""滑动滚动条至：加载更多处"""for i in range(5):pos = i * 50000js = "document.documentElement.scrollTop=%d" % posdriver.execute_script(js)time.sleep(3)print("准备点击：加载更多")driver.find_element_by_xpath("./*//input[@value='显示更多结果']").click()time.sleep(2)for i in range(5):pos = i * 50000js = "document.documentElement.scrollTop=%d" % posdriver.execute_script(js)time.sleep(3)time.sleep(2)html_page = driver.page_source"""利用Beautifulsoup4创建soup对象并进行页面解析"""soup = bs(html_page, "html.parser")"""通过soup对象中的findAll函数图像信息提取"""imglist = soup.findAll('img', {'class': 'rg_ic rg_i'})url_list = []"""获取图片地址，并添加进图片列表中"""for i in imglist:try:url_list.append(i['data-src'])except Exception as ex1:url_list.append(i['src'])print(ex1)print("总共包含图片个数为：" + str(len(url_list)))for i in url_list:try:ir = requests.get(i, headers=headers)s_uuid = str(uuid.uuid1())l_uuid = s_uuid.split('-')name = ''.join(l_uuid)print("开始下载图片：%s.jpg" % name)open(localPath + '%s.jpg' % name, 'wb').write(ir.content)except Exception as ex2:print('下载图片出错！！' + str(ex2))def run(self):driver = self.start_brower()self.downloadImg(driver)driver.close()print("Download has finished.")if __name__ == '__main__':print('\t\t\t**************************************\n\t\t\t**\t\tWelcome to Use Spider\t\t**\n\t\t\t*''*************************************')""" base_url_part1以及base_url_part2都是固定不变的，无需更改"""base_url_part1 = 'https://www.google.com/search?q='base_url_part2 = '&source=lnms&tbm=isch'"""爬取关键字"""search_list = ['管制刀具']for search_query in search_list:localPath = 'D:/weijin/'try:os.mkdir(localPath)except Exception as e:print(e)craw = Crawler()craw.run()转载于:https://www.cnblogs.com/sdhzdtwhm/p/10437005.html
本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！