feat(projects): 新增Scrapy综合案例示例代码

projects/crawling/bilibili_crawler/bilibili_crawler/helper.py

@@ -0,0 +1,49 @@
+import re
+from datetime import datetime
+
+pats = dict(
+    rank=re.compile(r"(\d+)"),
+    length=re.compile(r"(?P<minute>\d{2}):(?P<second>\d{2})"),
+)
+
+
+def parse_rank(value):
+    matched = pats["rank"].search(value.strip())
+    if matched:
+        number = matched.group()
+        return int(number)
+    return 0
+
+
+def parse_ops(value):
+
+    if not value:
+        return 0
+
+    value = value.strip()
+
+    if "万" in value:
+        digits = float(value.replace("万", "")) * 10000
+        return int(digits)
+    else:
+        return int(value)
+
+
+def parse_length(value):
+    length = value.strip()
+    pat = pats["length"]
+    matched = pat.search(length).groupdict()
+    if matched:
+        minute = int(matched["minute"])
+        second = int(matched["second"])
+
+        total = minute * 60 + second
+    else:
+        total = 0
+
+    return total
+
+
+def parse_timestamp(value):
+    timestamp = int(value)
+    return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S")

projects/crawling/bilibili_crawler/bilibili_crawler/items.py

@@ -0,0 +1,38 @@
+from itemloaders import ItemLoader
+from itemloaders.processors import Compose, TakeFirst
+from scrapy import Field, Item
+
+# isort:skip_file
+from bilibili_crawler.helper import (
+    parse_length,
+    parse_rank,
+    parse_ops,
+    parse_timestamp,
+)
+
+
+class DefaultLoader(ItemLoader):
+
+    default_output_processor = TakeFirst()
+
+
+class APIData(Item):
+
+    title = Field()
+    play = Field()
+    comment = Field()
+    typeid = Field()
+    author = Field()
+    mid = Field()
+    created = Field(output_processor=Compose(TakeFirst(), parse_timestamp))
+    length = Field(output_processor=Compose(TakeFirst(), parse_length))
+    bvid = Field()
+
+
+class VideoData(Item):
+
+    rank = Field(output_processor=Compose(TakeFirst(), parse_rank))
+    like = Field(output_processor=Compose(TakeFirst(), parse_ops))
+    coin = Field(output_processor=Compose(TakeFirst(), parse_ops))
+    collect = Field(output_processor=Compose(TakeFirst(), parse_ops))
+    share = Field(output_processor=Compose(TakeFirst(), parse_ops))

projects/crawling/bilibili_crawler/bilibili_crawler/middlewares.py

@@ -0,0 +1,103 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+
+
+class BilibiliCrawlerSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class BilibiliCrawlerDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

projects/crawling/bilibili_crawler/bilibili_crawler/pipelines.py

@@ -0,0 +1,13 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class BilibiliCrawlerPipeline:
+    def process_item(self, item, spider):
+        return item

projects/crawling/bilibili_crawler/bilibili_crawler/settings.py

@@ -0,0 +1,98 @@
+# Scrapy settings for bilibili_crawler project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = "bilibili_crawler"
+
+SPIDER_MODULES = ["bilibili_crawler.spiders"]
+NEWSPIDER_MODULE = "bilibili_crawler.spiders"
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+CONCURRENT_REQUESTS = 10
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+# DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+# CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+# TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+
+DEFAULT_REQUEST_HEADERS = {
+    "accept": "*/*",
+    "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
+    "accept-encoding": "gzip, deflate, br",
+    "origin": "https://www.bilibili.com",
+    "user-agent": " ".join(
+        [
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
+            "AppleWebKit/537.36 (KHTML, like Gecko)",
+            "Chrome/101.0.4951.64",
+            "Safari/537.36",
+        ]
+    ),
+}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+# SPIDER_MIDDLEWARES = {
+#    'bilibili_crawler.middlewares.BilibiliCrawlerSpiderMiddleware': 543,
+# }
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+# DOWNLOADER_MIDDLEWARES = {
+#    'bilibili_crawler.middlewares.BilibiliCrawlerDownloaderMiddleware': 543,
+# }
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+# EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+# }
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+# ITEM_PIPELINES = {
+#    'bilibili_crawler.pipelines.BilibiliCrawlerPipeline': 300,
+# }
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+# AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+# AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+AUTOTHROTTLE_DEBUG = True
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+# HTTPCACHE_ENABLED = True
+# HTTPCACHE_EXPIRATION_SECS = 0
+# HTTPCACHE_DIR = 'httpcache'
+# HTTPCACHE_IGNORE_HTTP_CODES = []
+# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

projects/crawling/bilibili_crawler/bilibili_crawler/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

projects/crawling/bilibili_crawler/bilibili_crawler/spiders/bilibili.py

@@ -0,0 +1,69 @@
+import math
+from urllib.parse import urlencode
+
+import scrapy
+from bilibili_crawler.items import APIData, DefaultLoader, VideoData
+
+VIDEO_PAGE_URL = "https://www.bilibili.com/video/{bvid}"
+API_URL = "https://api.bilibili.com/x/space/arc/search?"
+
+API_QUERY_PARAMS = {
+    "mid": "533459953",
+    "ps": "30",
+    "tid": "0",
+    "order": "pubdate",
+}
+
+
+class BilibiliSpider(scrapy.Spider):
+    name = "bilibili"
+    start_urls = [
+        API_URL + urlencode({"pn": "1", **API_QUERY_PARAMS}),
+    ]
+
+    def parse(self, response, **kwargs):
+        jsons = response.json()
+        count = jsons["data"]["page"]["count"]
+        total = math.ceil(int(count) / 30)
+
+        for page in range(1, total + 1):
+            url = API_URL + urlencode({"pn": str(page), **API_QUERY_PARAMS})
+            yield scrapy.Request(url=url, callback=self.parse_api)
+
+    def parse_api(self, response, **kwargs):
+        jsons = response.json()
+        api_data = jsons["data"]["list"]["vlist"]
+        for data in api_data:
+            bvid = data["bvid"]
+            loader = DefaultLoader(item=APIData())
+            for k in APIData.fields.keys():
+                loader.add_value(k, data[k])
+
+            yield scrapy.Request(
+                url=VIDEO_PAGE_URL.format(bvid=bvid),
+                callback=self.parse_video_data,
+                cb_kwargs={"api_data": loader.load_item()},
+            )
+
+    def parse_video_data(self, response, api_data, **kwargs):
+        loader = DefaultLoader(
+            item=VideoData(), response=response, selector=response.selector
+        )
+        loader.add_css("like", ".ops .like::text")
+        loader.add_css("coin", ".ops .coin::text")
+        loader.add_css("collect", ".ops .collect::text")
+        loader.add_css("share", ".ops .share::text")
+        loader.add_css("rank", ".video-data .rank::text")
+        video_data = loader.load_item()
+
+        data = dict(**api_data, **video_data)
+
+        yield data
+
+
+if __name__ == "__main__":
+    from scrapy.crawler import CrawlerProcess
+
+    process = CrawlerProcess()
+    process.crawl(BilibiliSpider)
+    process.start()

projects/crawling/bilibili_crawler/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = bilibili_crawler.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = bilibili_crawler