• 利用python爬取上证指数股吧评论并保存到mongodb数据库

           大家好，我是带我去滑雪！

           东方财富网是中国领先的金融服务网站之一，以提供全面的金融市场数据、资讯和交易工具而闻名。其受欢迎的“股吧”论坛特别适合爬取股票评论，东方财富网的股吧聚集了大量投资者和金融分析师，他们经常在此分享投资观点、分析报告和市场动态。这些内容对于进行市场情绪分析、投资策略研究或金融模型训练非常有价值。此外，东方财富网的用户基础庞大且活跃，每日都有大量的新帖子和评论产生。这种活跃的讨论环境可以提供实时的市场反馈和投资者情绪的动态变化。相比于其他金融网站，东方财富网的股吧系统更加集中和规范，容易进行数据爬取和分析。每个股票的讨论都有其专属的页面和结构化的评论区，便于自动化工具识别和抽取数据。

           在2022年的时候，我就尝试爬取了东方财富网的股吧评论，链接如下:http://t.csdnimg.cn/F45MZ。但是最近做文本的情感分析时，需要最新的上证指数评论时，再次运行代码，出现了爬取列表为空的问题，后面我查看了东方财富网的网页结构，发现结构已经变化。基于此，本文应运而生，主要解决爬取上证指数股吧评论问题，后续可能会对评论进行数据处理和情感分析。下面开始代码实战。

    目录

    （1）页面爬取

    （2）解析评论信息

    （3）保存数据

    （4）主函数

    （1）页面爬取

    1. from selenium import webdriver
    2. from selenium.webdriver.common.by import By
    3. import time
    4. import random
    5. import pandas as pd
    6. import os
    7. from selenium.webdriver.support.ui import WebDriverWait
    8. from selenium.webdriver.support import expected_conditions as EC
    9. from selenium.common.exceptions import TimeoutException
    10.  
    11. from mongodb import MongoAPI
    12. from parser import PostParser
    13. from parser import CommentParser
    14.  
    15.  
    16. class PostCrawler(object):
    17.  
    18.     def __init__(self, stock_symbol: str):
    19.         self.browser = None
    20.         self.symbol = stock_symbol
    21.         self.start = time.time()
    22.  
    23.     def create_webdriver(self):
    24.         options = webdriver.ChromeOptions()
    25.         options.add_argument('lang=zh_CN.UTF-8')
    26.         options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, '
    27.                              'like Gecko) Chrome/111.0.0.0 Safari/537.36"')
    28.         self.browser = webdriver.Chrome(options=options)
    29.  
    30.         current_dir = os.path.dirname(os.path.abspath(__file__))
    31.         js_file_path = os.path.join(current_dir, 'stealth.min.js')
    32.         with open(js_file_path) as f:
    33.             js = f.read()
    34.         self.browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
    35.             "source": js
    36.         })
    37.  
    38.     def get_page_num(self):
    39.         self.browser.get(f'http://guba.eastmoney.com/list,{self.symbol},f_1.html')
    40.         page_element = self.browser.find_element(By.CSS_SELECTOR, 'ul.paging > li:nth-child(7) > a > span')
    41.         return int(page_element.text)
    42.  
    43.     def crawl_post_info(self, page1: int, page2: int):
    44.         self.create_webdriver()
    45.         max_page = self.get_page_num()
    46.         current_page = page1
    47.         stop_page = min(page2, max_page)
    48.  
    49.         parser = PostParser()
    50.         postdb = MongoAPI('post_info', f'post_{self.symbol}')
    51.  
    52.         while current_page <= stop_page:
    53.             time.sleep(abs(random.normalvariate(0, 0.1)))
    54.             url = f'http://guba.eastmoney.com/list,{self.symbol},f_{current_page}.html'
    55.  
    56.             try:
    57.                 self.browser.get(url)
    58.                 dic_list = []
    59.                 list_item = self.browser.find_elements(By.CSS_SELECTOR, '.listitem')
    60.                 for li in list_item:
    61.                     dic = parser.parse_post_info(li)
    62.                     if 'guba.eastmoney.com' in dic['post_url']:
    63.                         dic_list.append(dic)
    64.                 postdb.insert_many(dic_list)
    65.                 print(f'{self.symbol}: 已经成功爬取第 {current_page} 页帖子基本信息，'
    66.                       f'进度 {(current_page - page1 + 1)*100/(stop_page - page1 + 1):.2f}%')
    67.                 current_page += 1
    68.  
    69.             except Exception as e:
    70.                 print(f'{self.symbol}: 第 {current_page} 页出现了错误 {e}')
    71.                 time.sleep(0.01)
    72.                 self.browser.refresh()
    73.                 self.browser.delete_all_cookies()
    74.                 self.browser.quit()
    75.                 self.create_webdriver()
    76.  
    77.         end = time.time()
    78.         time_cost = end - self.start
    79.         start_date = postdb.find_last()['post_date']
    80.         end_date = postdb.find_first()['post_date']
    81.  
    82.         row_count = postdb.count_documents()
    83.         self.browser.quit()
    84.  
    85.         print(f'成功爬取 {self.symbol}股吧共 {stop_page - page1 + 1} 页帖子，总计 {row_count} 条，花费 {time_cost/60:.2f} 分钟')
    86.         print(f'帖子的时间范围从 {start_date} 到 {end_date}')
    87.  
    88.  
    89. class CommentCrawler(object):
    90.  
    91.     def __init__(self, stock_symbol: str):
    92.         self.browser = None
    93.         self.symbol = stock_symbol
    94.         self.start = time.time()
    95.         self.post_df = None
    96.         self.current_num = 0
    97.  
    98.     def create_webdriver(self):
    99.         options = webdriver.ChromeOptions()
    100.         options.add_argument('lang=zh_CN.UTF-8')
    101.         options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, '
    102.                              'like Gecko) Chrome/111.0.0.0 Safari/537.36"')
    103.         self.browser = webdriver.Chrome(options=options)
    104.  
    105.         current_dir = os.path.dirname(os.path.abspath(__file__))
    106.         js_file_path = os.path.join(current_dir, 'stealth.min.js')
    107.         with open(js_file_path) as f:
    108.             js = f.read()
    109.         self.browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
    110.             "source": js
    111.         })
    112.  
    113.     def find_by_date(self, start_date, end_date):
    114.  
    115.         """
    116.         :param start_date: '2003-07-21' 字符串格式 ≥
    117.         :param end_date: '2024-07-21' 字符串格式 ≤
    118.         """
    119.         postdb = MongoAPI('post_info', f'post_{self.symbol}')
    120.         time_query = {
    121.             'post_date': {'$gte': start_date, '$lte': end_date},
    122.             'comment_num': {'$ne': 0}
    123.         }
    124.         post_info = postdb.find(time_query, {'_id': 1, 'post_url': 1})  # , 'post_date': 1
    125.         self.post_df = pd.DataFrame(post_info)
    126.  
    127.     def find_by_id(self, start_id: int, end_id: int):
    128.         """
    129.         :param start_id: 721 整数 ≥
    130.         :param end_id: 2003 整数 ≤
    131.         """
    132.         postdb = MongoAPI('post_info', f'post_{self.symbol}')
    133.         id_query = {
    134.             '_id': {'$gte': start_id, '$lte': end_id},
    135.             'comment_num': {'$ne': 0}
    136.         }
    137.         post_info = postdb.find(id_query, {'_id': 1, 'post_url': 1})
    138.         self.post_df = pd.DataFrame(post_info)
    139.  
    140.     def crawl_comment_info(self):
    141.         url_df = self.post_df['post_url']
    142.         id_df = self.post_df['_id']
    143.         total_num = self.post_df.shape[0]
    144.  
    145.         self.create_webdriver()
    146.         parser = CommentParser()
    147.         commentdb = MongoAPI('comment_info', f'comment_{self.symbol}')
    148.  
    149.         for url in url_df:
    150.             try:
    151.                 time.sleep(abs(random.normalvariate(0.03, 0.01)))
    152.  
    153.                 try:
    154.                     self.browser.get(url)
    155.                     WebDriverWait(self.browser, 0.2, poll_frequency=0.1).until(
    156.                         EC.presence_of_element_located((By.CSS_SELECTOR, 'div.reply_item.cl')))
    157.                 except TimeoutException:
    158.                     self.browser.refresh()
    159.                     print('------------ refresh ------------')
    160.                 finally:
    161.                     reply_items = self.browser.find_elements(By.CSS_SELECTOR, 'div.allReplyList > div.replylist_content > div.reply_item.cl')  # some have hot reply list avoid fetching twice
    162.  
    163.                 dic_list = []
    164.                 for item in reply_items:
    165.                     dic = parser.parse_comment_info(item, id_df.iloc[self.current_num].item())
    166.  
    167.                     dic_list.append(dic)
    168.  
    169.                     if parser.judge_sub_comment(item):
    170.                         sub_reply_items = item.find_elements(By.CSS_SELECTOR, 'li.reply_item_l2')
    171.  
    172.                         for subitem in sub_reply_items:
    173.                             dic = parser.parse_comment_info(subitem, id_df.iloc[self.current_num].item(), True)
    174.  
    175.                             dic_list.append(dic)
    176.  
    177.                 commentdb.insert_many(dic_list)
    178.                 self.current_num += 1
    179.                 print(f'{self.symbol}: 已成功爬取 {self.current_num} 页评论信息，进度 {self.current_num*100/total_num:.3f}%')
    180.  
    181.             except TypeError as e:
    182.                 self.current_num += 1
    183.                 print(f'{self.symbol}: 第 {self.current_num} 页出现了错误 {e} （{url}）')  # maybe the invisible comments
    184.                 print(f'应爬取的id范围是 {id_df.iloc[0]} 到 {id_df.iloc[-1]}, id {id_df.iloc[self.current_num - 1]} 出现了错误')
    185.                 self.browser.delete_all_cookies()
    186.                 self.browser.refresh()
    187.                 self.browser.quit()
    188.                 self.create_webdriver()
    189.  
    190.         end = time.time()
    191.         time_cost = end - self.start
    192.         row_count = commentdb.count_documents()
    193.         self.browser.quit()
    194.         print(f'成功爬取 {self.symbol}股吧 {self.current_num} 页评论，共 {row_count} 条，花费 {time_cost/60:.2f}分钟')

    （2）解析评论信息

    1. from selenium.webdriver.common.by import By
    2. from selenium import webdriver
    3.  
    4.  
    5. class PostParser(object):
    6.  
    7.     def __init__(self):
    8.         self.year = None
    9.         self.month = 13
    10.         self.id = 0
    11.  
    12.     @staticmethod
    13.     def parse_post_title(html):
    14.         title_element = html.find_element(By.CSS_SELECTOR, 'td:nth-child(3) > div')
    15.         return title_element.text
    16.  
    17.     @staticmethod
    18.     def parse_post_view(html):
    19.         view_element = html.find_element(By.CSS_SELECTOR, 'td > div')
    20.         return view_element.text
    21.  
    22.     @staticmethod
    23.     def parse_comment_num(html):
    24.         num_element = html.find_element(By.CSS_SELECTOR, 'td:nth-child(2) > div')
    25.         return int(num_element.text)
    26.  
    27.     @staticmethod
    28.     def parse_post_url(html):
    29.         url_element = html.find_element(By.CSS_SELECTOR, 'td:nth-child(3) > div > a')
    30.         return url_element.get_attribute('href')
    31.  
    32.     def get_post_year(self, html):
    33.         driver = webdriver.Chrome()
    34.         driver.get(self.parse_post_url(html))
    35.         date_str = driver.find_element(By.CSS_SELECTOR, 'div.newsauthor > div.author-info.cl > div.time').text
    36.         self.year = int(date_str[:4])
    37.         driver.quit()
    38.  
    39.     @staticmethod
    40.     def judge_post_date(html):
    41.         try:
    42.             judge_element = html.find_element(By.CSS_SELECTOR, 'td:nth-child(3) > div > span')
    43.             if judge_element.text == '问董秘':
    44.                 return False
    45.         except:
    46.             return True
    47.  
    48.     def parse_post_date(self, html):
    49.         time_element = html.find_element(By.CSS_SELECTOR, 'div.update.pub_time')
    50.         time_str = time_element.text
    51.         month, day = map(int, time_str.split(' ')[0].split('-'))
    52.  
    53.         if self.judge_post_date(html):
    54.             if self.month < month == 12:
    55.                 self.year -= 1
    56.             self.month = month
    57.  
    58.         if self.id == 1:
    59.             self.get_post_year(html)
    60.  
    61.         date = f'{self.year}-{month:02d}-{day:02d}'
    62.         time = time_str.split(' ')[1]
    63.         return date, time
    64.  
    65.     def parse_post_info(self, html):
    66.         self.id += 1
    67.         title = self.parse_post_title(html)
    68.         view = self.parse_post_view(html)
    69.         num = self.parse_comment_num(html)
    70.         url = self.parse_post_url(html)
    71.         date, time = self.parse_post_date(html)
    72.         post_info = {
    73.             '_id': self.id,
    74.             'post_title': title,
    75.             'post_view': view,
    76.             'comment_num': num,
    77.             'post_url': url,
    78.             'post_date': date,
    79.             'post_time': time,
    80.         }
    81.         return post_info
    82.  
    83.  
    84. class CommentParser(object):
    85.  
    86.     @staticmethod
    87.     def judge_sub_comment(html):
    88.         sub = html.find_elements(By.CSS_SELECTOR, 'ul.replyListL2')  # must use '_elements' instead of '_element'
    89.         return bool(sub)
    90.  
    91.     @staticmethod
    92.     def parse_comment_content(html, sub_bool):
    93.         if sub_bool:
    94.             content_element = html.find_element(By.CSS_SELECTOR, 'div.reply_title > span')
    95.         else:
    96.             content_element = html.find_element(By.CSS_SELECTOR, 'div.recont_right.fl > div.reply_title > span')
    97.         return content_element.text
    98.  
    99.     @staticmethod
    100.     def parse_comment_like(html, sub_bool):
    101.         if sub_bool:
    102.             like_element = html.find_element(By.CSS_SELECTOR, 'span.likemodule')
    103.         else:
    104.             like_element = html.find_element(By.CSS_SELECTOR, 'ul.bottomright > li:nth-child(4) > span')
    105.  
    106.         if like_element.text == '点赞':  # website display text instead of '0'
    107.             return 0
    108.         else:
    109.             return int(like_element.text)
    110.  
    111.     @staticmethod
    112.     def parse_comment_date(html, sub_bool):
    113.         if sub_bool:  # situation to deal with sub-comments
    114.             date_element = html.find_element(By.CSS_SELECTOR, 'span.pubtime')
    115.         else:
    116.             date_element = html.find_element(By.CSS_SELECTOR, 'div.publishtime > span.pubtime')
    117.         date_str = date_element.text
    118.         date = date_str.split(' ')[0]
    119.         time = date_str.split(' ')[1][:5]
    120.         return date, time
    121.  
    122.     def parse_comment_info(self, html, post_id, sub_bool: bool = False):  # sub_pool is used to distinguish sub-comments
    123.         content = self.parse_comment_content(html, sub_bool)
    124.         like = self.parse_comment_like(html, sub_bool)
    125.         date, time = self.parse_comment_date(html, sub_bool)
    126.         whether_subcomment = int(sub_bool)  # '1' means it is sub-comment, '0' means it is not
    127.         comment_info = {
    128.             'post_id': post_id,
    129.             'comment_content': content,
    130.             'comment_like': like,
    131.             'comment_date': date,
    132.             'comment_time': time,
    133.             'sub_comment': whether_subcomment,
    134.         }
    135.         return comment_info

    （3）保存数据

    1. from pymongo import MongoClient
    2.  
    3.  
    4. class MongoAPI(object):
    5.  
    6.     def __init__(self, db_name: str, collection_name: str, host='localhost', port=27017):
    7.         self.host = host
    8.         self.port = port
    9.         self.db_name = db_name
    10.         self.collection = collection_name
    11.         self.client = MongoClient(host=self.host, port=self.port)
    12.         self.database = self.client[self.db_name]
    13.         self.collection = self.database[self.collection]
    14.  
    15.     def insert_one(self, kv_dict):
    16.         self.collection.insert_one(kv_dict)
    17.  
    18.     def insert_many(self, li_dict):  # more efficient
    19.         self.collection.insert_many(li_dict)
    20.  
    21.     def find_one(self, query1, query2):
    22.         return self.collection.find_one(query1, query2)
    23.  
    24.     def find(self, query1, query2):
    25.         return self.collection.find(query1, query2)
    26.  
    27.     def find_first(self):
    28.         return self.collection.find_one(sort=[('_id', 1)])
    29.     
    30.     def find_last(self):
    31.         return self.collection.find_one(sort=[('_id', -1)])
    32.  
    33.     def count_documents(self):
    34.         return self.collection.count_documents({})
    35.  
    36.     def update_one(self, kv_dict):
    37.         self.collection.update_one(kv_dict, {'$set': kv_dict}, upsert=True)
    38.  
    39.     def drop(self):
    40.         self.collection.drop()

    （4）主函数

    1. from crawler import PostCrawler
    2. from crawler import CommentCrawler
    3. import threading
    4.  
    5.  
    6. def post_thread(stock_symbol, start_page, end_page):
    7.     post_crawler = PostCrawler(stock_symbol)
    8.     post_crawler.crawl_post_info(start_page, end_page)
    9.  
    10.  
    11. def comment_thread_date(stock_symbol, start_date, end_date):
    12.     comment_crawler.find_by_date(start_date, end_date)
    13.     comment_crawler.crawl_comment_info()
    14.  
    15.  
    16. def comment_thread_id(stock_symbol, start_id, end_id):
    17.     comment_crawler = CommentCrawler(stock_symbol)
    18.     comment_crawler.find_by_id(start_id, end_id)
    19.     comment_crawler.crawl_comment_info()
    20.  
    21.  
    22. if __name__ == "__main__":
    23.     thread1 = threading.Thread(target=post_thread, args=('zssh000001', 5835, 5875))
    24.     thread1.start()
    25.     thread1.join()
    26.     print(f"成功爬取评论数据!")

    输出结果展示：

           成功爬取了495775条评论数据，运行了14个小时，实属不易。后续将会对这个数据集，进行深度的分析。

    需要数据集的家人们可以去百度网盘（永久有效）获取：

    链接：https://pan.baidu.com/s/16Pp57kAbC3xAqPylyfQziA?pwd=2138
    提取码：2138 

    更多优质内容持续发布中，请移步主页查看。

       点赞+关注,下次不迷路！

  • 相关阅读:
    Spring源码级笔记(一)
    [附源码]Python计算机毕业设计Django网文论坛管理系统
    Hystrix面试题大全
    CASIO虚线替换部分及切线支距法
    【python基础】循环语句-break关键字
    SSM - Springboot - MyBatis-Plus 全栈体系（二十七）
    【LeetCode】29. 两数相除
    python+django协同过滤算法的音乐推荐系统研究vue
    阿里云主要产品有哪些？有什么用？
    Python_数据容器_元组tuple
  • 原文地址：https://blog.csdn.net/qq_45856698/article/details/140042040