天天减肥网 > python 爬取静态静态静态网页

python 爬取静态静态静态网页

时间：2023-03-13 05:49:35

相关推荐

python 爬取静态静态静态网页

该文章是我在练习基础是写的一个小项目，爬取的是一个网站的图片（妹子才是学习的动力）。其中用到了线程和xpath等知识的应用，属于小白入门篇，各位大佬要是在浏览过程中发现可以优化的地方尽情留言，先给各位道谢啦。

首先，我们来看一下导入的库：

使用requests.get(url).text方法进行一个字符串格式的HTML网页的获取导入lxml库的etree模块，用lxml中的etree.HTML(response)方法进行初始化，这样就成功构造了一个XPath解析对象，其中etree.HTML模块可以自动修正HTML文本。接着我们导入线程库threading，创建多个线程相当于多条腿走路，肯定快一点嘛。队列Queue是在线程安全的基础上进行消息通信的，但是本文中好像没有需要通信的必要，所以大家权当混个眼熟吧time用来计时，os用来创建文件夹，copy.deepcopy()进行深度拷贝。

import requestsfrom lxml import etreefrom threading import Threadfrom multiprocessing import Queueimport time, os, copy

下面来复盘一下核心代码：

其中包括的方法有：

网页请求

def send_request(self,url):i = 0while i < 3: #请求不成功的情况下最多请求三次try:print(u'[INFO]请求URL: '+ url)#获取字符串格式的网页html = requests.get(url = url,headers = self.headers).text except Exception as e:i += 1print(u'再次请求[INFO]%s%s'%(e,url))else:return html

网页解析

爬取网站的首页地址，有疑问的可以点进去对照着看。

def parse_page(self):'''解析页面使用xpath基本语法：层级： /直接子级 //跳级属性： @属性访问方法： contains()、text()等'''response = self.send_request(self.url)#构建xpath解析对象html = etree.HTML(response)#跳级直接找属性class = 'item masonry_brick masonry-brick'的div#接着跳级找属性target='_blank'的a标签，取其属性href中的链接#最后用set()进行去重link_list = set(html.xpath("//div[@class = 'item masonry_brick masonry-brick']//a[@target='_blank']/@href"))#print('正在爬取该网页，请稍等......')#每一大页有24个不同封面，进行遍历爬取for link in link_list:print('正在爬取该div {} 的附属图片，请稍等......'.format(link))times = 0#pic_parse()的主要任务的获取图片位置以及下一张照片的链接pic_info = self.pic_parse(link)pic_path = str(copy.deepcopy(link))#save_pic()是通过图片位置进行保存到本机self.save_pic(pic_info[0])#但是这里突然想到，有点多次重复访问了，嗨害嗨while times < 100: #可以通过一直获取的链接一直访问#print("第{}次请求该div下的图片，请稍等.....".format(times+1))#因为每次获取的下一步链接形式不一致，需要进行修改#带有https：开头的是完整链接，否则就属于部分跳转链接if pic_info[1][0].split('/')[0] == 'https:':pic_path = pic_info[1][0]else:#使用a.replace(旧字符串，新字符串)进行替换出完整链接pic_path = pic_path.replace(pic_path.split('/')[-1],pic_info[1][0])#进行新一次的图片信息获取pic_info = self.pic_parse(pic_path)self.save_pic(pic_info[0])times += 1print("解析完该网页，等待结束......")

子网页分析：主要任务的获取图片位置以及下一张照片的链接

使用同一方法进行子网页分析来获取目标

def pic_parse(self,link):picture = self.send_request(link)pic_html = etree.HTML(picture)pic_addr = pic_html.xpath("//div[@class='big-pic']//a/img/@src")#pic_name = pic_html.xpath("//div[@class='wrapper clearfix imgtitle']/h1/text()") #中文乱码next_pic_url = pic_html.xpath("//div[@class='big-pic']//a/@href")return (pic_addr,next_pic_url)

保存图片

def save_pic(self,pic_url):picture = requests.get(url = pic_url[0], headers = self.headers)file_name = pic_url[0].split('/')[-1]path = "E://girls"#创建存放的文件夹if not os.path.exists(path):os.makedirs(path)with open(path+ '//' + file_name, 'wb') as file:file.write(picture.content)print("完成一张照片储存......")

定义main方法，进行整体调度

def main():# 创建一个队列用来保存进程获取到的数据q = Queue()# 构造所有ｕｒｌhome_pages = ['/mmtp/list_9_{}.html'.format(page) for page in range(1,12)]# 保存线程Thread_list = []# 创建并启动线程for home_page in home_pages:p = girlsPicture_spider(home_page,q)p.start()Thread_list.append(p)# 让主线程等待子线程执行完成for i in Thread_list:i.join()while not q.empty():print(q.get())if __name__ == "__main__":start = time.time()#home_pages = ['/mmtp/list_9_{}.html'.format(page) for page in range(1, 12)]#使用多线程运行代码main()#这里使用的是单线程的方法运行代码# for home_page in home_pages:#spider = girlsPicture_spider(home_page)#spider.run()print('[info]耗时：%s' % (time.time() - start))

完整代码，直接run就行

import requestsfrom lxml import etreefrom threading import Threadfrom multiprocessing import Queueimport time, os, copyclass girlsPicture_spider(Thread): # 参数中可加参数Thread线程导入from threading import Threaddef __init__(self, url,q): # 参数中可加线程参数qsuper(girlsPicture_spider, self).__init__()self.url = urlself.q = qself.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46',}def send_request(self, url):i = 0while i < 3:try:print(u'[INFO]请求URL: ' + url)html = requests.get(url=url, headers=self.headers).textexcept Exception as e:i += 1print(u'再次请求[INFO]%s%s' % (e, url))else:return htmldef pic_parse(self, link):picture = self.send_request(link)pic_html = etree.HTML(picture)pic_addr = pic_html.xpath("//div[@class='big-pic']//a/img/@src")# pic_name = pic_html.xpath("//div[@class='wrapper clearfix imgtitle']/h1/text()") #乱码next_pic_url = pic_html.xpath("//div[@class='big-pic']//a/@href")return (pic_addr, next_pic_url)def save_pic(self, pic_url):print(pic_url[0])picture = requests.get(url=pic_url[0], headers=self.headers)file_name = pic_url[0].split('/')[-1]path = "E://girls"if not os.path.exists(path):os.makedirs(path)with open(path + '//' + file_name, 'wb') as file:file.write(picture.content)print("完成一张照片储存......")def parse_page(self):'''解析页面使用xpath'''response = self.send_request(self.url)html = etree.HTML(response)link_list = set(html.xpath("//div[@class = 'item masonry_brick masonry-brick']//a[@target='_blank']/@href"))print('正在爬取该网页，请稍等......')for link in link_list:print('正在爬取该div {} 的附属图片，请稍等......'.format(link))# links = [str(link)]# print(links)times = 0pic_info = self.pic_parse(link)pic_path = str(copy.deepcopy(link))print(pic_path)self.save_pic(pic_info[0])while times < 100:print("第{}次请求该div下的图片，请稍等.....".format(times + 1))# print(link)print(pic_info)if pic_info[1][0].split('/')[0] == 'https:':pic_path = pic_info[1][0]else:pic_path = pic_path.replace(pic_path.split('/')[-1], pic_info[1][0])# print(pic_path)time.sleep(0.1)pic_info = self.pic_parse(pic_path)self.save_pic(pic_info[0])# self.q.put(str(times) + "\t" + str(pic_info))times += 1print("解析完该网页，等待结束......")def run(self):self.parse_page()def main():# 创建一个队列用来保存进程获取到的数据q = Queue()# 构造所有ｕｒｌhome_pages = ['/mmtp/list_9_{}.html'.format(page) for page in range(1,12)]# 保存线程Thread_list = []# 创建并启动线程for home_page in home_pages:p = girlsPicture_spider(home_page,q)p.start()Thread_list.append(p)# 让主线程等待子线程执行完成for i in Thread_list:i.join()while not q.empty():print(q.get())if __name__ == "__main__":start = time.time()#home_pages = ['/mmtp/list_9_{}.html'.format(page) for page in range(1, 12)]main()# for home_page in home_pages:#spider = girlsPicture_spider(home_page)#spider.run()print('[info]耗时：%s' % (time.time() - start))