猫眼电影爬取

headers设置

user_agent='Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
cookie='__mta=45932315.1608295245480.1609932302195.1609932384970.73; uuid_n_v=v1; _lxsdk_cuid=17675dde089c8-07a468a14a4d2b-59442e11-144000-17675dde089c8; __mta=45932315.1608295245480.1608345693705.1608347288332.39; uuid=23932090420011EB938EB13948D660ED4C5735DFBB854F1482C8F32A7D2A939F; _lxsdk=23932090420011EB938EB13948D660ED4C5735DFBB854F1482C8F32A7D2A939F; _csrf=0ad5150c37ad8b0d35bfa75267641e55060d7a0395b0a070219bb914e540fd37; Hm_lvt_703e94591e87be68cc8da0da7cbd0be2=1608385740,1608450456,1608536146,1609932146; Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2=1609932385; _lxsdk_s=176d76f040c-1b4-538-b6b%7C%7C8'
headers={"user-agent":user_agent,"cookie":cookie}

数据爬取

server=[]
for i in range(10):url="https://maoyan.com/board/4?offset="+str(i*10)cat=requests.get(url,headers=headers,proxies = {"http":"58.253.157.136"})cat_text=cat.textserver.append(cat.text)

建立空表

maoyan=pd.DataFrame()

信息提取

for i in server:data=etree.HTML(i)pos=data.xpath('//dl[@class="board-wrapper"]')for j in pos:ranking=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/i/text()')picture=j.xpath('.//a/img/@data-src')movie_web=j.xpath('.//div/div[1]/div[1]/p[1]/a/@href')movie_name=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[1]/p[1]/a/text()')performer=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[1]/p[2]/text()')Release_date=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[1]/p[3]/text()')grades1=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[2]/p/i[1]/text()')grades2=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[2]/p/i[2]/text()')score=[]for h in range(0,len(grades1)):score.append(grades1[h]+grades2[h])result=pd.DataFrame({"电影排名":ranking,"图片地址":picture,"电影详情页地址":movie_web,"电影名称":movie_name,"电影主演":performer,"首映时间":Release_date,"评分":score})maoyan=maoyan.append(result,ignore_index=True)

补全

maoyan['详情页'] ='https://maoyan.com'+maoyan['电影详情页地址']
del(maoyan['电影详情页地址'])

导出

maoyan.to_excel("猫眼数据top100爬取.xlsx",index=False)

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！