豆瓣图书爬虫学习笔记（附源码）

从豆瓣图书按标签分类爬取图书信息，保存到CSV文件并存储到MySQL数据库。

依赖

1
2
3
4
5
6
7
8
9
import csv
import json
import os
import time
import pandas as pd  # 数据处理
import pymysql       # MySQL操作
import requests      # HTTP请求
import re            # 正则表达式
from lxml import etree  # XML/HTML解析

Spider类

1
2
3
4
5
6
7
class spider(object):
    def __init__(self, page, tag):
        # 构造目标URL: https://book.douban.com/tag/{tag}?start={page}&type=T
        self.page = page * 20
        self.tag = tag
        self.url = f"https://book.douban.com/tag/{self.tag}?start={self.page}&type=T"
        self.headers = {...}  # 包含cookie的请求头

对每个详情页解析:

  • 标题：//span[@property="v:itemreviewed"]/text()
  • 作者：//div[@id="info"]/span[1]/a/text()
  • 封面：//img[@rel="v:photo"]/@src
  • 出版年份：使用正则(\d{4}-\d{1,2})
  • 价格：正则匹配(\d+)\.\d+元?
  • 评分：//strong[@property="v:average"]/text()
  • 评论：JSON格式存储评论时间和内容

Data类 - 数据库操作

1
2
3
4
5
6
7
8
9
class data(object):
    def __init__(self):
        # MySQL连接配置
        self.conn = pymysql.connect(
            host='127.0.0.1',
            user='root',
            password='1111',
            database='spider'
        )

主程序流程

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
if __name__=='__main__':
    tags = ['小说', '文学']  # 目标标签
    page = 2               # 每个标签爬取2页
    
    # 创建爬虫实例
    newspider = spider(0,'')
    
    # 分标签分页爬取
    for tag in tags:
        time.sleep(10)  # 防封策略
        for page_num in range(0, page+1):
            spider_ins = spider(page_num, tag)
            spider_ins.csv_init()
            spider_ins.get_detail(tag)
    
    # 写入数据库
    db = data()
    db.write_to_database('./temp.csv')

数据库设计

1
2
3
4
5
6
7
8
9
10
11
12
CREATE TABLE IF NOT EXISTS book (
    b_id INT(10) AUTO_INCREMENT PRIMARY KEY,
    tag VARCHAR(20),
    title VARCHAR(200),
    author VARCHAR(100),
    year CHAR(10),
    cover VARCHAR(300),
    rate FLOAT,
    price VARCHAR(20),
    comment JSON,  -- 存储JSON格式评论数据
    crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

详细代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
import csv
import json
import os
import time
import pandas as pd
import pymysql
import requests
import re

from lxml import etree


class spider(object):
    def __init__(self,page,tag):
        self.page=page*20
        self.tag=tag
        self.url=f"https://book.douban.com/tag/{self.tag}?start={self.page}&type=T"
        self.headers = {
            'Host': 'book.douban.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Connection': 'keep-alive',
            'Referer': 'https://book.douban.com/tag/',
            'Cache-Control': 'max-age=0',
            'cookie':'bid=Cd8aG6FKDYM; _vwo_uuid_v2=D697E510D4963B643F816125CE52A87EA|d0a2a804f89c858482dcd73706529ff7; _pk_id.100001.3ac3=6738a9674717d69b.1750754636.; __yadk_uid=V4gvX8vwQveX2ZOOr2SkQbbjK47buWzx; dbcl2="289542484:ynYE5qHg9kg"; push_noty_num=0; push_doumail_num=0; ct=y; __utmv=30149280.28954; ll="118137"; ck=ZOht; ap_v=0,6.0; _pk_ref.100001.3ac3=%5B%22%22%2C%22%22%2C1750813492%2C%22https%3A%2F%2Fcn.bing.com%2F%22%5D; _pk_ses.100001.3ac3=1; __utma=30149280.1340771599.1750754635.1750767699.1750813493.4; __utmc=30149280; __utmz=30149280.1750813493.4.2.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt_douban=1; __utma=81379588.1482404396.1750754635.1750767699.1750813493.4; __utmc=81379588; __utmz=81379588.1750813493.4.2.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt=1; __utmb=30149280.3.10.1750813493; __utmb=81379588.3.10.1750813493'
        }
    def get_detail(self,tag):
        try:
            req = requests.get(self.url, headers=self.headers)
            print('爬取的页面：' + self.url)
            req_xpath = etree.HTML(req.text)
            li_xpath = req_xpath.xpath('//ul[@class="subject-list"]/li/div[@class="info"]/h2/a/@href')
            href = []  # 详情链接
            for i in li_xpath:
                href.append(i)

            for i in href:
                req_detail = requests.get(i, headers=self.headers)
                req_detailxpath = etree.HTML(req_detail.text)

                title = req_detailxpath.xpath('//span[@property="v:itemreviewed"]/text()')[0]
                # / text()：提取所选 < span > 标签内部的文本内容
                author = req_detailxpath.xpath('//div[@id="info"]/span[1]/a/text()')[0]
                cover = req_detailxpath.xpath('//img[@rel="v:photo"]/@src')[0]
                info_text = ''.join(req_detailxpath.xpath('//div[@id="info"]/text()'))
                year_match = re.search(r'(\d{4}-\d{1,2})', info_text)
                year = year_match.group(1) if year_match else "未知年份"

                # 安全获取价格 - 更健壮的正则表达式
                price_match = re.search(r'(\d+)\.\d+元?', info_text)
                price = price_match.group(1) if price_match else "0"
                # 评分
                rate_match = req_detailxpath.xpath('//strong[@property="v:average"]/text()')[0].strip()
                rate = rate_match if rate_match else "无"
                comment = []
                for path in req_detailxpath.xpath('//ul/li[@class="comment-item"]'):
                    comment_content = path.xpath('.//p[@class="comment-content"]/span[1]/text()')
                    # // 表示任意深度的后代节点
                    comment_time = path.xpath('.//span[@class="comment-info"]//a[@class="comment-time"]/text()')

                    comment.append({
                        "comment_time": comment_time,
                        "comment_content": comment_content
                    })
                comment = json.dumps(comment)
                self.write_to_csv([tag,title, author, year, cover, rate, price, comment])
                print( {
                    'tag':tag if tag else "未知标签",
                    'title': title if title else "未知题目",
                    'author': author if author else "未知作者",
                    'year': year if year else "未知年份",
                    'cover': cover if cover else "未知封面",
                    'rate': rate if rate else "未知评分",
                    'price': price if price else "未知价格",
                    'comment': comment if comment else "未知评论"
                })
        except Exception as e:
            print(f"爬取失败：{e}")
            return None

# GEI后就存csv,然后存到数据库
    def write_to_csv(self,data):
        with open('./temp.csv','a',newline='',encoding='utf-8') as f:
            writer=csv.writer(f)
            writer.writerow(data)

    def csv_init(self):
        if not os.path.exists('./temp.csv'):
            with open('./temp.csv','a',newline='',encoding='utf-8')as f:
                writer=csv.writer(f)
                writer.writerow(['tag','title','author','year','cover','rate','price','comment'])


class data(object):
    def __init__(self):
        self.conn = pymysql.connect(host='127.0.0.1', user='root', passwd='1111', port=3306, database='spider')

    def write_to_database(self,csv):
        df = pd.read_csv(csv)
        df = df.where(df.notnull(), None)
        try:
            with self.conn.cursor() as cursor:
                insert_sql = "insert into book(tag,title,author,year,cover,rate,price,comment) " \
                             "values (%(tag)s,%(title)s, %(author)s, %(year)s, %(cover)s, %(rate)s, %(price)s, %(comment)s)"
                insert_data = []
                for _, row in df.iterrows():
                    # if pd.isna(row['tag']|row['title']|row['author']|row['year']|row['cover']|row['rate']|row['price']|row['comment']):
                        # 处理缺失值
                        # insert_data.append(None)
                    insert_data.append({
                        'tag': row['tag'] if row['tag'] else None,
                        'title': row['title'] if row['title'] else None,
                        'author': row['author'] if row['author'] else None,
                        'year': row['year'] if row['year'] else None,
                        'cover': row['cover'] if row['cover'] else None,
                        'rate': row['rate'] if row['rate'] else None,
                        'price': row['price'] if row['price'] else None,
                        'comment': row['comment'] if row['comment'] else None
                    })

                batch_size = 40
                total_rows = len(insert_data)

                for i in range(0, total_rows, batch_size):
                    batch = insert_data[i:i + batch_size]
                    cursor.executemany(insert_sql, batch)
                    self.conn.commit()
                self.conn.commit()
        except Exception as e:
            print(f"数据库错误：{e}")




if __name__=='__main__':
    newspider=spider(0,'')
    tags = ['小说', '文学']
    page=2
    for i in tags:
        time.sleep(10)  # 每爬取一段时间数据休息 30 s， 防止被爬虫系统检测到
        for j in range(0,page+1):
            newspider=spider(j,i)
            newspider.csv_init()
            newspider.get_detail(i)
    db=data()
    db.write_to_database('./temp.csv')


# CREATE TABLE IF NOT EXISTS book (
# b_id  int(10) AUTO_INCREMENT PRIMARY KEY,
#                     tag VARCHAR(20),
#                     title VARCHAR(200),
#                     author VARCHAR(100),
#                     year CHAR(10),
#                     cover VARCHAR(300),
#                     rate FLOAT,
#                     price VARCHAR(20),
#                     comment JSON,
#                     crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
#                 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
# delete from book;

学的b站视频：

https://www.bilibili.com/video/BV1ta1yYfEis/?buvid=XXE052DEF87809FC60A50A244DE5EFCCF7B29&is_story_h5=false&mid=Dz9tedHPLI0nxRpU5K2V9g%3D%3D&p=8&plat_id=168&share_from=ugc&share_medium=android&share_plat=android&share_session_id=e230cb0a-a5f4-441c-8c32-d0712aec35c2&share_source=WEIXIN&share_tag=s_i&spmid=main.my-history-search-result.option-more.0&timestamp=1750752022&unique_k=c1vPdbt&up_id=384650805&vd_source=a3d4488494e8456375bd6d1e86850642