• python七大爬虫程序

    一，爬取豆瓣电影信息

    1. import random
    2. import urllib.request
    3. from bs4 import BeautifulSoup
    4. import codecs
    5. from time import sleep
    6.  
    7. def main(url, headers):
    8.     # 发送请求
    9.     page = urllib.request.Request(url, headers=headers)
    10.     page = urllib.request.urlopen(page)
    11.     contents = page.read()
    12.     # 用BeautifulSoup解析网页
    13.     soup = BeautifulSoup(contents, "html.parser")
    14.     infofile.write("")
    15.     print('爬取豆瓣电影250: \n')
    16.  
    17.     for tag in soup.find_all(attrs={"class": "item"}):
    18.         # 爬取序号
    19.         num = tag.find('em').get_text()
    20.         print(num)
    21.         infofile.write(num + "\r\n")
    22.         # 电影名称
    23.         name = tag.find_all(attrs={"class": "title"})
    24.         zwname = name[0].get_text()
    25.         print('[中文名称]', zwname)
    26.         infofile.write("[中文名称]" + zwname + "\r\n")
    27.         # 网页链接
    28.         url_movie = tag.find(attrs={"class": "hd"}).a
    29.         urls = url_movie.attrs['href']
    30.         print('[网页链接]', urls)
    31.         infofile.write("[网页链接]" + urls + "\r\n")
    32.         # 爬取评分和评论数
    33.         info = tag.find(attrs={"class": "star"}).get_text()
    34.         info = info.replace('\n', ' ')
    35.         info = info.lstrip()
    36.         print('[评分评论]', info)
    37.         # 获取评语
    38.         info = tag.find(attrs={"class": "inq"})
    39.         if (info):  # 避免没有影评调用get_text()报错
    40.             content = info.get_text()
    41.             print('[影评]', content)
    42.             infofile.write(u"[影评]" + content + "\r\n")
    43.         print('')
    44.  
    45.  
    46. if __name__ == '__main__':
    47.     # 存储文件
    48.     infofile = codecs.open("豆瓣电影信息.txt", 'a', 'utf-8')
    49.     # 消息头
    50.     headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
    51.     # 翻页
    52.     i = 0
    53.     while i < 10:
    54.         print('页码', (i + 1))
    55.         num = i * 25  # 每次显示25部 URL序号按25增加
    56.         url = 'https://movie.douban.com/top250?start=' + str(num) + '&filter='
    57.         main(url, headers)
    58.         sleep(5 + random.random())
    59.         infofile.write("\r\n\r\n")
    60.         i = i + 1
    61.     infofile.close()

    二，爬取知乎网页内容

    1. import csv
    2. import requests
    3. import re
    4. import time
    5.  
    6. def main(page):
    7.     url = f'https://tieba.baidu.com/p/7882177660?pn={page}'
    8.     headers = {
    9.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
    10.     }
    11.     resp = requests.get(url,headers=headers)
    12.     html = resp.text
    13.     # 评论内容
    14.     comments = re.findall('style="display:;">                    (.*?)
      ',html)
  • # 评论用户
  • users = re.findall('class="p_author_name j_user_card" href=".*?" target="_blank">(.*?)',html)
  • # 评论时间
  • comment_times = re.findall('楼(.*?),html)
  • for u,c,t in zip(users,comments,comment_times):
  • # 筛选数据,过滤掉异常数据
  • if 'img' in c or 'div' in c or len(u)>50:
  • continue
  • csvwriter.writerow((u,t,c))
  • print(u,t,c)
  • print(f'第{page}页爬取完毕')
  • if __name__ == '__main__':
  • with open('01.csv','a',encoding='utf-8')as f:
  • csvwriter = csv.writer(f)
  • csvwriter.writerow(('评论用户','评论时间','评论内容'))
  • for page in range(1,8): # 爬取前7页的内容
  • main(page)
  • time.sleep(2)

    • 三，爬起天气预报

    1. import requests
    2. from bs4 import BeautifulSoup
    3. import urllib.request
    4. import random
    5. # 设置header 防止产生403forbidden
    6. my_headers = [
    7.     "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    8.     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    9.     "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
    10.     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    11.     "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
    12.     'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
    13.     'Opera/9.25 (Windows NT 5.1; U; en)',
    14.     'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
    15.     'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
    16.     'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
    17.     'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
    18.     "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
    19.     "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "
    20. ]
    21. # 抓取网页信息
    22. def get_content(url, headers):
    23.     random_header = random.choice(headers)
    24.     req = urllib.request.Request(url)
    25.     req.add_header("User-Agent", random_header)
    26.     req.add_header("Host", "lishi.tianqi.com")
    27.     req.add_header("Referer", "http://lishi.tianqi.com/")
    28.     req.add_header("GET", url)
    29.  
    30.     content = urllib.request.urlopen(req).read()
    31.     return content
    32.     # 三个月份天气的链接
    33. urls = ["http://lishi.tianqi.com/wuhan/202210.html",
    34.         "http://lishi.tianqi.com/wuhan/202211.html",
    35.         "http://lishi.tianqi.com/wuhan/202212.html"]
    36.  
    37. file = open('weather.csv', 'w')
    38. for url in urls:
    39.     response = get_content(url, my_headers)
    40.     soup = BeautifulSoup(response, 'html.parser')
    41.     weather_list = soup.select('ul[class="thrui"]')
    42.  
    43.     for weather in weather_list:
    44.         ul_list = weather.select('li')
    45.         for ul in ul_list:
    46.             li_list = ul.select('div')
    47.             str = ""
    48.             for li in li_list:
    49.                 str += li.string + ','
    50.             file.write(str + '\n')
    51. file.close()

    四，爬取网页标题

    1. import requests
    2. from bs4 import BeautifulSoup
    3. url = "http://project.webcat.top/bx/80607/24411"
    4. # 发送请求
    5. response = requests.get(url)
    6. # 使用BeautifulSoup解析HTML内容
    7. soup = BeautifulSoup(response.content,'html.parser')
    8. # 获取网站标题
    9. title = soup.title.string
    10. print("网站标题:", title)

    五，爬取网页所有链接

    1. import requests
    2. from bs4 import BeautifulSoup
    3.  
    4. # 发送HTTP请求获取网页内容
    5. url = 'https://www.python.org/'
    6. response = requests.get(url)
    7. html_content = response.text
    8.  
    9. # 使用BeautifulSoup解析网页内容
    10. soup = BeautifulSoup(html_content, 'html.parser')
    11.  
    12. # 提取需要的数据
    13. # 这里以提取网页中的所有链接为例
    14. links = soup.find_all('a')
    15. for link in links:
    16.     print(link.get('href'))

    六，爬取网页图片

    1. import requests
    2. from bs4 import BeautifulSoup
    3. import urllib
    4.  
    5. # 爬取网页数据并解析数据
    6. url = 'http://vip.1905.com/m/play/1655899.shtml'  # 替换为你要爬取的网页地址
    7. response = requests.get(url)
    8. soup = BeautifulSoup(response.text, 'html.parser')
    9. # 解析数据并获取影视图片的URL
    10. image_urls = []
    11. images = soup.find_all('img')
    12. for image in images:
    13.     image_url = image['src']
    14.     image_urls.append(image_url)
    15.  
    16. # 下载图片并保存到本地文件
    17. for image_url in image_urls:
    18.     urllib.request.urlretrieve(image_url, 'rrr.jpg')  # 替换为你要保存的文件名和路径

    七，爬取网页完整文本

    1. import requests
    2. from bs4 import BeautifulSoup
    3.  
    4. def scrape_html(url):
    5.     # 发送HTTP请求
    6.     response = requests.get(url)
    7.     if response.status_code == 200:
    8.         soup = BeautifulSoup(response.text, 'html.parser')
    9.         # 找到并打印所有的段落标签（
      ）的内容
    10.         for p_tag in soup.find_all('p'):
    11.             print(p_tag.get_text())
    12.     else:
    13.         print(f"Error: {response.status_code} when fetching {url}")
    14.  
    15. # 测试函数
    16. scrape_html('https://www.bafangwy.com/')  # 替换为你要爬取的网址
  • 相关阅读:
    python调用GPT实现：智能用例生成工具
    ARM32开发--RTC内置实时时钟
    ChatGpt的初步认知(认知搬运工)
    Springboot垃圾分类回收系统的设计与实现8t69x计算机毕业设计-课程设计-期末作业-毕设程序代做
    微软将 AI 工具和软件的研发工作外包给 OpenAI
    Python 使用Scapy构造特殊数据包
    LLM 模型量化推理速度评测
    扬帆际海—shopee跨境店和本土店谁更有优势？
    U_boot介绍
    【DesignMode】组合模式（composite mode）
  • 原文地址：https://blog.csdn.net/2301_80124151/article/details/136414227