财经资讯爬虫

1. 数据来源
   港股金融界 ：http://hk.jrj.com.cn/
   资本邦 ：http://www.chinaipo.com/
   智通财经网：http://www.zhitongcaijing.com/
   腾讯财经：https://pacaio.match.qq.com/

2. 资讯表结构
   

3. 使用 requests 模块爬取资讯，清洗数据，存放到Django models中

"""
发现页新闻爬取
定时任务
"""
import json
import re
import threading
from io import BytesIOfrom bs4 import BeautifulSoup
import requests
from django.core.files import File
from lxml import etreeimport os, djangoos.environ.setdefault("DJANGO_SETTINGS_MODULE", "goldshell_end.settings")
django.setup()from operation.models import Articleclass JinRongJie(object):"""港股金融界"""headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36","Accept": 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',"Accept-Encoding": "gzip, deflate","Accept-Language": "zh-CN,zh;q=0.9","Connection": "keep-alive"}def get_url_list(self):"""获取新闻列表:return:"""for i in ["1", "2", "3", "4"]:if i == "1":url = 'http://hk.jrj.com.cn/list/hgxw.shtml'else:url = f'http://hk.jrj.com.cn/list/hgxw-{i}.shtml'try:response = requests.get(url=url, headers=self.headers)except Exception:return False, '服务器网络故障'if response.status_code != 200:return False, '来源服务器错误'context = response.content.decode('gbk')"""提取有效数据新闻列表"""detail_html = etree.HTML(context)url_list = detail_html.xpath('//div[@class="list-s1"]/ul/li/a/@href')self.detail_data(url_list)  # 获取新闻详情return Truedef detail_data(self, url_list):"""获取新闻详情:param data::return:"""context_list = []for url in url_list:try:response = requests.get(url=url, headers=self.headers)except Exception:return False, '服务器网络故障'if response.status_code != 200:return False, '来源服务器错误'context = response.content.decode('gb18030')# context = response.textdetail_html = etree.HTML(context)"""提取有效数据headline  # 文章标题time  # 发布时间cover  # 封面图context  #  正文url  # 连接source  # 文章来源"""item = {}try:item["time"] = detail_html.xpath('//p[@class="inftop"]/span/text()')[0]item["time"] = ''.join(item["time"].split("\r\n"))item["headline"] = detail_html.xpath('//div[@class="titmain"]/h1/text()')item["headline"] = ''.join(''.join(item["headline"]).split("\r\n"))con_list = detail_html.xpath('//div[@class="texttit_m1"]/p/text()')item["context"] = ''for con in con_list:item["context"] = item["context"] + '
' + con + '
'item["context"] = ''.join(''.join(item["context"]).split('\u3000\u3000'))item["url"] = urlitem["source"] = "金融界"try:if len(Article.objects.filter(url=item['url'])) == 0:Article.objects.create(headline=item["headline"],time=item["time"],context=item["context"],url=item["url"],source=item["source"],status="WAITING")except Exception:'数据写入失败'context_list.append(item)except Exception:'数据提取失败'print(f'数据爬取完成,数量{len(context_list)}')return True, '数据保存成功'class ChinaIpo(object):"""港股资本邦"""headers = {"Accept": "application/json, text/javascript, */*; q=0.01","Accept-Encoding": "gzip, deflate","Accept-Language": "zh-CN,zh;q=0.9","Connection": "keep-alive","Content-Length": "40","Content-Type": "application/x-www-form-urlencoded","Cookie": "PHPSESSID=sejm46mvdu9gmuidfpnem5v894; UM_distinctid=16bdedf96a6da-0c4487051c35f3-37677e02-1fa400-16bdedf96a7458; XSBlang=zh-cn; CNZZDATA1255725096=2112891795-1562809799-%7C1563172642; Hm_lvt_61a2d81fc23a3a8087c8791bf55f7e6e=1562826422,1562929083,1563175398,1563176881; Hm_lpvt_61a2d81fc23a3a8087c8791bf55f7e6e=1563176886","Host": "www.chinaipo.com","Origin": "http://www.chinaipo.com","Referer": "http://www.chinaipo.com/hk/","User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36","X-Requested-With": "XMLHttpRequest"}def get_url_list(self):"""获取新闻列表:return:"""url = "http://www.chinaipo.com/index.php?app=information&mod=Index&act=getMoreNewsFromCate"data = {"catid": "100022","type": "1","t": "0.4098812290943108"}try:response = requests.post(url=url, headers=self.headers, data=data)except Exception:return False, '服务器网络故障'if response.status_code != 200:return False, '来源服务器错误'context = response.text.encode('utf8').decode("unicode-escape")context = context[20:-2:1]data_list = []  # 消息数据"""提取有效数据新闻列表"""detail_html = etree.HTML(context)url_list = detail_html.xpath('//div[@class="htn-de clearfix"]/a/@href')img_list = detail_html.xpath('//div[@class="htn-de clearfix"]/a/img/@src')for index, val in enumerate(url_list):url_text = ''.join(val.split("\\"))img_text = ''.join(img_list[index].split("\\"))data_list.append({'url': url_text, 'img': img_text})self.detail_data(data_list)return Truedef detail_data(self, data_list):"""获取新闻详情:param data::return:"""context_list = []for data in data_list:url = data['url']headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36",}try:response = requests.get(url=url, headers=headers)except Exception:return False, '服务器网络故障'context = response.content.decode('utf8')detail_html = etree.HTML(context)"""提取有效数据headline  # 文章标题time  # 发布时间cover  # 封面图context  #  正文url  # 连接source  # 文章来源"""item = {}try:item['cover'] = data['img']item["time"] = detail_html.xpath('//p[@class="news-from"]/text()')[1][3:13:]# item["time"] = ''.join(item["time"].split("\r\n"))item["headline"] = detail_html.xpath('//h1[@class="tt-news"]/text()')[0]con_list = detail_html.xpath('//div[@class="newscont"]/p/text()')item["context"] = ''for con in con_list:item["context"] = item["context"] + '
' + con + '
'item["context"] = ''.join(''.join(item["context"]).split('\u3000\u3000'))item["context"] = ''.join(item["context"].split('\r\n'))item["context"] = ''.join(item["context"].split('\t'))item["context"] = ''.join(item["context"].split('
：123RF
'))item["url"] = urlitem["source"] = "资本邦"try:if len(Article.objects.filter(url=item['url'])) == 0:article = Article.objects.create(headline=item["headline"],time=item["time"],context=item["context"],url=item["url"],source=item["source"],status="WAITING")try:file = requests.get(url=item['cover'], headers=self.headers)article.cover.save("_{}.jpg".format('zibenbang'),File(BytesIO(file.content)))except Exception:'图片写入失败'except Exception:'数据写入失败'except Exception:'数据提取失败'context_list.append(item)print(f'数据爬取完成,数量{len(context_list)}')return True, '数据保存成功'class ZhiTongCaiJing(object):"""美股智通财经"""headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}base_url = "https://www.zhitongcaijing.com"def get_url_list(self):"""获取新闻列表:return:"""data_list = []  # 消息数据for i in ["1", "2", "3", "4", "5"]:url = 'https://www.zhitongcaijing.com/content/meigu.html?page=' + itry:response = requests.get(url=url, headers=self.headers)except Exception:return False, '服务器网络故障'if response.status_code != 200:return False, '来源服务器错误'context = response.content.decode('utf8')"""提取有效数据"""soup = BeautifulSoup(context, 'html.parser')dt_list = soup.select('dt > a')for dt in dt_list:url = dt.get('href')dt = str(dt)img = re.findall('src="https://img.zhitongcaijing.com/image/.*?\?', dt)if not len(img):img = Noneelse:img = img[0][5:-1:1]data_list.append({"url": url, "img": img})self.detail_data(data_list)return True, 'success'def detail_data(self, data_list):"""获取新闻详情:param data::return:"""context_list = []for data in data_list:url = self.base_url + data['url']try:response = requests.get(url=url, headers=self.headers)except Exception:return False, '服务器网络故障'context = response.content.decode('utf8')detail_html = etree.HTML(context)"""提取有效数据headline  # 文章标题time  # 发布时间cover  # 封面图context  #  正文url  # 连接source  # 文章来源"""item = {}try:item["time"] = detail_html.xpath('//div[@class="h-30 line-h-30 color-c size-14 padding-b10"]/text()')[0]item['time'] = re.sub(' ', '', item['time'])item['time'] = ' '.join(('-'.join(('-'.join(item['time'].split('年'))).split('月'))).split('日'))item["headline"] = detail_html.xpath('//h1/text()')[0]item["cover"] = data['img']con_list = detail_html.xpath('//article/p/text()')item["context"] = ''for con in con_list:item["context"] = item["context"] + '
' + con + '
'item["url"] = self.base_url + data['url']item["source"] = "智通财经网"try:if len(Article.objects.filter(url=item['url'])) == 0:  # 如果链接存在则不保存article = Article.objects.create(headline=str(item["headline"]),time=item['time'],context=item["context"],url=item["url"],source=item["source"],status="WAITING")try:file = requests.get(url=item['cover'], headers=self.headers)article.cover.save("_{}.jpg".format('zhitongcaijing'),File(BytesIO(file.content)))except Exception:'图片写入失败'except Exception:'数据写入失败'context_list.append(item)except Exception:'数据提取失败'print(f'数据爬取完成,数量{len(context_list)}')return True, '数据保存成功'class TengXun(object):"""美股腾讯网"""headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36","Accept": 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',"Accept-Encoding": "gzip, deflate","Accept-Language": "zh-CN,zh;q=0.9","Connection": "keep-alive"}def get_url_list(self):"""获取新闻列表:return:"""data_list = []for i in [1, 2, 3, 4]:url = f"https://pacaio.match.qq.com/irs/rcd?cid=52&token=8f6b50e1667f130c10f981309e1d8200&ext=3921,3916&page={i}&isForce=1&expIds=20190713A0ARSH|20190713A04T3S|20190713A035IC|20190712A0IF3G|20190712A0I6BK|20190712A05PZC|20190713A04O0U|20190713A018KR|20190713A00OMV|20190713A007BT|20190712A0UA2N|20190712A0U944|20190712A0TZC0|20190712A0TOTJ|20190712A0T2S2|20190712A0Q8IC|20190712A0C70J|20190712A0BRMX|20190712A0BD56|20190712A0BAAM&callback=__jp6"try:con = requests.get(url=url).content.decode('utf8')[6:-1:]except Exception:return False, '服务器网络故障'con = json.loads(con)"""提取有效数据新闻列表"""for data in con["data"]:img_url = data["img"]title = data["title"]url_text = data["url"]data_list.append({'img_url': img_url, 'title': title, 'url': url_text})data_list_01 = []#  列表去重for data in data_list:if data not in data_list_01:data_list_01.append(data)self.detail_data(data_list_01)  # 获取新闻详情return Truedef detail_data(self, data_list):"""获取新闻详情:param data_list::return:"""context_list = []for data in data_list:url = data['url']try:response = requests.get(url=url, headers=self.headers)except Exception:return False, '服务器网络故障'context = response.content.decode('utf8')"""提取有效数据headline  # 文章标题time  # 发布时间cover  # 封面图context  #  正文url  # 连接source  # 文章来源"""item = {}try:item["cover"] = data['img_url']item["headline"] = data['title']item["url"] = urlcon_list = json.loads(re.findall('contents:.*?}]', context)[0][10::])item['time'] = re.findall('articleid:.+', context)[0][12:20:]item['time'] = item['time'][0:4:] + '-' + item['time'][4:6:] + '-' + item['time'][6:8:] + ' 00:00'item["context"] = ''try:for con in con_list:item["context"] = item["context"] + '
' + con['value'] + '
'except Exception:passitem["source"] = "美股腾讯网"context_list.append(item)try:if len(Article.objects.filter(url=item['url'])) == 0:  # 如果链接存在则不保存article = Article.objects.create(headline=str(item["headline"]),time='2019-07-16 11:22',context=item["context"],url=item["url"],source=item["source"],status="WAITING")try:file = requests.get(url=item['cover'], headers=self.headers)article.cover.save("_{}.jpg".format('tengxun'),File(BytesIO(file.content)))except Exception:'图片写入失败'except Exception:'数据写入失败'except Exception:'数据提取失败'print(f'数据爬取完成,数量{len(context_list)}')return True, '数据保存成功'class TuiJian(object):"""推荐智通财经"""headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}base_url = "https://www.zhitongcaijing.com"def get_url_list(self):"""获取新闻列表:return:"""data_list = []  # 消息数据for i in ["1", "2", "3", "4", "5"]:url = 'https://www.zhitongcaijing.com/content/recommend.html?page=' + itry:response = requests.get(url=url, headers=self.headers)except Exception:return False, '服务器网络故障'if response.status_code != 200:return False, '来源服务器错误'context = response.content.decode('utf8')"""提取有效数据"""soup = BeautifulSoup(context, 'html.parser')dt_list = soup.select('dt > a')for dt in dt_list:url = dt.get('href')# img = dt.get('src')dt = str(dt)img = re.findall('src="https://img.zhitongcaijing.com/image/.*?\?', dt)if not len(img):img = Noneelse:img = img[0][5:-1:1]data_list.append({"url": url, "img": img})self.detail_data(data_list)return True, 'success'def detail_data(self, data_list):"""获取新闻详情:param data::return:"""context_list = []for data in data_list:url = self.base_url + data['url']try:response = requests.get(url=url, headers=self.headers)except Exception:return False, '服务器网络故障'context = response.content.decode('utf8')detail_html = etree.HTML(context)"""提取有效数据headline  # 文章标题time  # 发布时间cover  # 封面图context  #  正文url  # 连接source  # 文章来源"""item = {}try:item["time"] = detail_html.xpath('//div[@class="h-30 line-h-30 color-c size-14 padding-b10"]/text()')[0]item['time'] = re.sub(' ', '', item['time'])item['time'] = ' '.join(('-'.join(('-'.join(item['time'].split('年'))).split('月'))).split('日'))item["headline"] = detail_html.xpath('//h1/text()')[0]item["cover"] = data['img']con_list = detail_html.xpath('//article/p/text()')item["context"] = ''for con in con_list:item["context"] = item["context"] + '
' + con + '
'item["url"] = self.base_url + data['url']item["source"] = "智通财经网"try:if len(Article.objects.filter(url=item['url'])) == 0:  # 如果链接存在则不保存article = Article.objects.create(headline=str(item["headline"]),time=item['time'],context=item["context"],url=item["url"],source=item["source"],status="WAITING")try:file = requests.get(url=item['cover'], headers=self.headers)article.cover.save("_{}.jpg".format('tuijian'),File(BytesIO(file.content)))except Exception:'图片写入失败'except Exception:'数据写入失败'context_list.append(item)except Exception:'数据提取失败'print(f'数据爬取完成,数量{len(context_list)}')return True, '数据保存成功'def run():"""异步爬取数据:return:"""threading.Thread(target=JinRongJie().get_url_list).start()  # 港股金融界threading.Thread(target=ChinaIpo().get_url_list).start()  # 港股资本邦threading.Thread(target=ZhiTongCaiJing().get_url_list).start()  # 美股智通财经网threading.Thread(target=TengXun().get_url_list).start()  # 美股腾讯网threading.Thread(target=TuiJian().get_url_list).start()  # 推荐智通财经网# multiprocessingif __name__ == '__main__':run()

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！