• 爬取某网站计算机类图书

    网页链接：

    https://www.ptpress.com.cn/shopping/search?tag=search&orderstr=hot&leve11-75424c57-6dd7-4d1f-b6b9-8e95773c0593

    一、为了完成爬取数据，需要进行以下步骤

    1.在浏览器中打开页面，选择"计算机"

    2.可以看到大量的"计算机"相关书籍，右键点击"检查"

    3.刷新页面，点击下一页，查看url

    4.点击"Response",查看json格式中的信息，发现与要爬取的书籍信息一致

    5.划到最低端可以发现计算机类图书查看页数（570页）

    6.查看Date格式

    7.根据书籍详情来爬取相关信息

    二、代码部分

    1.将爬取内容放入打印并放入excel表格中
    1. import requests
    2. import re
    3. import datetime
    4. from time import sleep
    5. import pandas as pd
    6.  
    7. S = "bookLink"
    8.  
    9. headers = {
    10.     'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
    11.                  '(KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.44',
    12. }
    13. url = 'https://www.ptpress.com.cn/bookinfo/getBookListForEBTag'
    14. book_info = []
    15. for page in range(1,571):
    16.     data = {
    17.         'bookTagId':'a15a734f-0ae9-41d7-9012-6ef9de2e71c8',
    18.         'page':f'{page}',
    19.         'rows':'18',
    20.         'orderStr':'publish'
    21.     }
    22.     response = requests.post(url,data=data,headers=headers)
    23.     data = response.json()
    24.  
    25.     author = data["data"]["data"][0]["author"]
    26.     isbn = data["data"]["data"][0]["isbn"]
    27.     publish = datetime.datetime.strptime(
    28.         data["data"]["data"][0]["publishDate"],"%Y%m"
    29.     )
    30.     discountPrice = data["data"]["data"][0]["discountPrice"]
    31.     bookDiscount = data["data"]["data"][0]["bookDiscount"]
    32.     price = data["data"]["data"][0]["price"]
    33.     bookId = data["data"]["data"][0]["bookId"]
    34.     executiveEditor = data["data"]["data"][0]["executiveEditor"]
    35.     bookName = data["data"]["data"][0]["bookName"]
    36.     picPath = data["data"]["data"][0]["picPath"]
    37.     bookLink = "https://www.ptpress.com.cn/shopping/buy?bookId=" + bookId
    38.  
    39.     book_info.append({
    40.         "author":author,
    41.         "isbn":isbn,
    42.         "publish":publish,
    43.         "discountPrice":discountPrice,
    44.         "bookDiscount":bookDiscount,
    45.         "price":price,
    46.         "bookId":bookId,
    47.         "executiveEditor":executiveEditor,
    48.         "bookName":bookName,
    49.         "picPath":picPath,
    50.         "bookLink":bookLink
    51.     })
    52.     print(f"第{page}页爬取成功！")
    53.     sleep(1)
    54.  
    55. print(book_info)
    56.  
    57. # 将数据保存到Excel文件中
    58. df = pd.DataFrame(book_info)
    59. df.to_excel("book_info.xlsx", index=False)
    爬取结果：

    2.将爬取内容放入打印并放入csv文件中
    1. import requests
    2. import re
    3. import datetime
    4. from time import sleep
    5. import pandas as pd
    6. import csv
    7.  
    8. S = "bookLink"
    9.  
    10. headers = {
    11.     'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
    12.                  '(KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.44',
    13. }
    14. url = 'https://www.ptpress.com.cn/bookinfo/getBookListForEBTag'
    15. book_info = []
    16. for page in range(1,571):
    17.     data = {
    18.         'bookTagId':'a15a734f-0ae9-41d7-9012-6ef9de2e71c8',
    19.         'page':f'{page}',
    20.         'rows':'18',
    21.         'orderStr':'publish'
    22.     }
    23.     response = requests.post(url,data=data,headers=headers)
    24.     data = response.json()
    25.  
    26.     author = data["data"]["data"][0]["author"]
    27.     isbn = data["data"]["data"][0]["isbn"]
    28.     publish = datetime.datetime.strptime(
    29.         data["data"]["data"][0]["publishDate"],"%Y%m"
    30.     )
    31.     discountPrice = data["data"]["data"][0]["discountPrice"]
    32.     bookDiscount = data["data"]["data"][0]["bookDiscount"]
    33.     price = data["data"]["data"][0]["price"]
    34.     bookId = data["data"]["data"][0]["bookId"]
    35.     executiveEditor = data["data"]["data"][0]["executiveEditor"]
    36.     bookName = data["data"]["data"][0]["bookName"]
    37.     picPath = data["data"]["data"][0]["picPath"]
    38.     bookLink = "https://www.ptpress.com.cn/shopping/buy?bookId=" + bookId
    39.  
    40.     book_info.append({
    41.         "author":author,
    42.         "isbn":isbn,
    43.         "publish":publish,
    44.         "discountPrice":discountPrice,
    45.         "bookDiscount":bookDiscount,
    46.         "price":price,
    47.         "bookId":bookId,
    48.         "executiveEditor":executiveEditor,
    49.         "bookName":bookName,
    50.         "picPath":picPath,
    51.         "bookLink":bookLink
    52.     })
    53.     print(f"第{page}页爬取成功！")
    54.     sleep(1)
    55.  
    56. print(book_info)
    57.  
    58. # 将数据保存到csv文件中
    59. df = pd.DataFrame(book_info)
    60. df.to_csv("人民邮电计算机书本信息.csv", index=False)
    爬取结果：

    3.将爬取内容放入打印并放入MySQL数据库中
    1. import requests
    2. import re
    3. import datetime
    4. from time import sleep
    5. import pandas as pd
    6. import csv
    7. import pymysql
    8.  
    9. S = "bookLink"
    10.  
    11. headers = {
    12.     'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
    13.                  '(KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.44',
    14. }
    15. url = 'https://www.ptpress.com.cn/bookinfo/getBookListForEBTag'
    16. book_info = []
    17. for page in range(1,571):
    18.     data = {
    19.         'bookTagId':'a15a734f-0ae9-41d7-9012-6ef9de2e71c8',
    20.         'page':f'{page}',
    21.         'rows':'18',
    22.         'orderStr':'publish'
    23.     }
    24.     response = requests.post(url,data=data,headers=headers)
    25.     data = response.json()
    26.  
    27.     author = data["data"]["data"][0]["author"]
    28.     isbn = data["data"]["data"][0]["isbn"]
    29.     publish = datetime.datetime.strptime(
    30.         data["data"]["data"][0]["publishDate"],"%Y%m"
    31.     )
    32.     discountPrice = data["data"]["data"][0]["discountPrice"]
    33.     bookDiscount = data["data"]["data"][0]["bookDiscount"]
    34.     price = data["data"]["data"][0]["price"]
    35.     bookId = data["data"]["data"][0]["bookId"]
    36.     executiveEditor = data["data"]["data"][0]["executiveEditor"]
    37.     bookName = data["data"]["data"][0]["bookName"]
    38.     picPath = data["data"]["data"][0]["picPath"]
    39.     bookLink = "https://www.ptpress.com.cn/shopping/buy?bookId=" + bookId
    40.  
    41.     book_info.append({
    42.         "author":author,
    43.         "isbn":isbn,
    44.         "publish":publish,
    45.         "discountPrice":discountPrice,
    46.         "bookDiscount":bookDiscount,
    47.         "price":price,
    48.         "bookId":bookId,
    49.         "executiveEditor":executiveEditor,
    50.         "bookName":bookName,
    51.         "picPath":picPath,
    52.         "bookLink":bookLink
    53.     })
    54.     print(f"第{page}页爬取成功！")
    55.     sleep(1)
    56.  
    57. print(book_info)
    58.  
    59. # 将数据保存到MySQL数据库中
    60. conn = pymysql.connect(host='localhost', user='root', password='your_password', db='your_database', charset='utf8')
    61. cursor = conn.cursor()
    62.  
    63. # 创建表格booklist
    64. cursor.execute('CREATE TABLE IF NOT EXISTS booklist (author VARCHAR(255), isbn VARCHAR(255), publish DATE, discountPrice FLOAT, bookDiscount FLOAT, price FLOAT, bookId VARCHAR(255), executiveEditor VARCHAR(255), bookName VARCHAR(255), picPath VARCHAR(255), bookLink VARCHAR(255))')
    65.  
    66. # 将数据插入到表格booklist中
    67. for book in book_info:
    68.     sql = f"INSERT INTO booklist (author, isbn, publish, discountPrice, bookDiscount, price, bookId, executiveEditor, bookName, picPath, bookLink) VALUES ('{book['author']}', '{book['isbn']}', '{book['publish'].strftime('%Y-%m-%d')}', {book['discountPrice']}, {book['bookDiscount']}, {book['price']}, '{book['bookId']}', '{book['executiveEditor']}', '{book['bookName']}', '{book['picPath']}', '{book['bookLink']}')"
    69.     cursor.execute(sql)
    70.  
    71. # 提交事务
    72. conn.commit()
    73.  
    74. # 关闭连接
    75. cursor.close()
    76. conn.close()
    爬取结果：

  • 相关阅读:
    title标签和meta标签怎样设置？有什么含义？
    leetcode刷题日记:141. Linked List Cycle（环形链表）
    渗透测试突破口——未授权访问
    Java之HikariCP数据库连接池浅入浅出
    PAC工单科目金额明细
    一、认识STM32
    Windows简单安装redis
    图文详解线性回归与局部加权线性回归+房价预测实例
    冲刺第十五届蓝桥杯P0003倍数问题
    在配置文件“tsconfig.json”中找不到任何输入。指定的 “include“ 路径为“[“**/*“]”，“exclude“ 路径为[]
  • 原文地址：https://blog.csdn.net/m0_74972727/article/details/133892237