Python爬蟲|你真的會寫爬蟲嗎？

class URLManager(object):    def __init__(self):        self.new_urls = set()        self.old_urls = set()
    def has_new_url(self):        # 判斷是否有未爬取的url        return self.new_url_size()!=0
    def get_new_url(self):        # 獲取一個未爬取的連結        new_url = self.new_urls.pop()        # 提取之後，將其新增到已爬取的連結中        self.old_urls.add(new_url)        return new_url
    def add_new_url(self, url):        # 將新連結新增到未爬取的集合中(單個連結)        if url is None:            return        if url not in self.new_urls and url not in self.old_urls:            self.new_urls.add(url)
    def add_new_urls(self,urls):        # 將新連結新增到未爬取的集合中(集合)        if urls is None or len(urls)==0:            return        for url in urls:            self.add_new_url(url)
    def new_url_size(self):        # 獲取未爬取的url大小        return len(self.new_urls)
    def old_url_size(self):        # 獲取已爬取的url大小        return len(self.old_urls)

import requestsclass HTMLDownload(object):    def download(self, url):        if url is None:            return        s = requests.Session()        s.essay-headers['User-Agent'] ='Mozilla / 5.0(Windows NT 10.0;WOW64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 63.0.3239.132Safari / 537.36'        res = s.get(url)        # 判斷是否正常獲取        if res.status_code == 200:            res.encoding='utf-8'            res = res.text            return res        return None

import refrom bs4 import BeautifulSoupclass HTMLParser(object):
    def parser(self, page_url, html_cont):        '''        用於解析網頁內容，抽取URL和資料        :param page_url: 下載頁面的URL        :param html_cont: 下載的網頁內容        :return: 傳回URL和資料        '''        if page_url is None or html_cont is None:            return        soup = BeautifulSoup(html_cont, 'html.parser')        new_urls = self._get_new_urls(page_url, soup)        new_data = self._get_new_data(page_url, soup)        return new_urls, new_data
    def _get_new_urls(self,page_url,soup):        '''        抽取新的URL集合        :param page_url:下載頁面的URL        :param soup: soup資料        :return: 傳回新的URL集合        '''        new_urls = set()        for link in range(1,100):            # 新增新的url            new_url = "http://www.runoob.com/w3cnote/page/"+str(link)            new_urls.add(new_url)            print(new_urls)        return new_urls
    def _get_new_data(self,page_url,soup):         '''         抽取有效資料         :param page_url:下載頁面的url         :param soup:         :return: 傳回有效資料         '''         data={}         data['url'] = page_url         title = soup.find('div', class_='post-intro').find('h2')         print(title)         data['title'] = title.get_text()         summary = soup.find('div', class_='post-intro').find('p')         data['summary'] = summary.get_text()         return data

import codecsclass DataOutput(object):
    def __init__(self):        self.datas = []
    def store_data(self,data):        if data is None:            return        self.datas.append(data)
    def output_html(self):        fout = codecs.open('baike.html', 'a', encoding='utf-8')        fout.write("")        fout.write("")        fout.write("")        fout.write("

”

)        for data in self.datas:            fout.write(""

)            fout.write("%s

“%data[‘url’])

            fout.write("《%s》

” % data[‘title’])

            fout.write("[%s]

” % data[‘summary’])

            fout.write("

from base.DataOutput import DataOutputfrom base.HTMLParser import HTMLParserfrom base.HTMLDownload import HTMLDownloadfrom base.URLManager import URLManager
class SpiderMan(object):    def __init__(self):        self.manager = URLManager()        self.downloader = HTMLDownload()        self.parser = HTMLParser()        self.output = DataOutput()

    def crawl(self, root_url):        # 新增入口URL        self.manager.add_new_url(root_url)        # 判斷url管理器中是否有新的url，同時判斷抓取多少個url        while(self.manager.has_new_url() and self.manager.old_url_size()<100):            try:                # 從URL管理器獲取新的URL                new_url = self.manager.get_new_url()                print(new_url)                # HTML下載器下載網頁                html = self.downloader.download(new_url)                # HTML解析器抽取網頁資料                new_urls, data = self.parser.parser(new_url, html)                print(new_urls)                # 將抽取的url新增到URL管理器中                self.manager.add_new_urls(new_urls)                # 資料儲存器儲存檔案                self.output.store_data(data)                print("已經抓取%s個連結" % self.manager.old_url_size())            except Exception as e:                print("failed")                print(e)            # 資料儲存器將檔案輸出成指定的格式            self.output.output_html()

if __name__ == '__main__':    spider_man = SpiderMan()    spider_man.crawl("http://www.runoob.com/w3cnote/page/1")