python爬虫入门之爬取贴吧标题

#!/usr/bin/env python
# -*- coding: utf-8 -*-import urllib2
import re#加载页面内容
def load_page(url):'''发送url请求返回url请求的静态html页面:param url::return:'''user_agent = "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50"headers = {"User-Agent" : user_agent}request = urllib2.Request(url,headers = headers)response = urllib2.urlopen(request)html = response.read()#print html#print "--------------------------"getTitle(html)#生成url地址，加载页面内容
def tieba_spider(url,startPage,endPage):for i in range(startPage,endPage + 1):page = (i - 1) * 50my_url = url + str(page)load_page(my_url)print "--------第%d页----------" % i#获得贴吧的标题
def getTitle(html):info = re.findall(r'class="j_th_tit ">(.*?)',html,re.S)for titleList in info:print titleListprint "---------------"if __name__ == '__main__':url = "http://tieba.baidu.com/f?kw=java&ie=utf-8&pn="startPage = 1endPage = 8tieba_spider(url, startPage, endPage)print "---------------------结束------------------"

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！