python爬取小说

运行结果：

代码：

 import requests
 from bs4 import BeautifulSoup
 from selenium import webdriver
 import os
 
 class NovelSpider:
     def __init__(self):
         self.start_url = 'https://www.biqukan.com/1_1680/'
 
     def get_novel(self):
         response = requests.get(self.start_url)
         soup = BeautifulSoup(response.text, 'html.parser')
         div_chapter = soup.find(class_="listmain")
         chapter_list = div_chapter.find_all('a')
         chapter_list = chapter_list[12:]
         chapter = []
         chapter_num = len(chapter_list)
         count = 0
         print('《凡人修仙传仙界篇》开始下载:')
         for cl in chapter_list:
             chapter_dict = {}
             chapter_name = cl.get_text()
             chapter_dict['name'] = chapter_name
             chapter_url = cl.get('href')
             chapter_dict['value'] = 'https://www.biqukan.com' + chapter_url
             if chapter_dict not in chapter:
                 chapter.append(chapter_dict)
             print(f"已下载:{count}/{chapter_num}")
             self.download_novel(chapter_dict)
             count += 1
 
     def parse_novel(self, url):
         browser = webdriver.PhantomJS(executable_path=r'F:\Spider\novelSpider\phantomjs.exe')
         browser.get(url)
         soup = BeautifulSoup(browser.page_source, 'html.parser')
         find_txt = soup.find(class_='showtxt')
         # print(type(find_txt.get_text()))
         return find_txt.get_text()
 
     def download_novel(self, data):
         filename = data['name']
         url = data['value']
         txt = self.parse_novel(url)
 
         path = r"F:\Spider\novelSpider"
         isExists = os.path.exists(path)
         if not isExists:
             os.mkdir(path)
         else:
             pass
 
         with open(path + f'\凡人修仙传仙界篇.txt', 'a', encoding='utf-8') as f:
             f.write(f'{filename}\n\n')
             f.write(txt)
             f.write('\n======\n\n')
             f.close()
 
 if __name__ == '__main__':
     ns = NovelSpider()
     ns.get_novel()