lostecho/sspai-100-hours-series-python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hotfix(crawler): 调整B站一件三连数据CSS选择器规则

Browse Source
This commit is contained in:
100gle
2022-08-24 16:02:17 +08:00
parent c72ca6f39c
commit 968768cc8c
2 changed files with 9 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
code/01/bilibili.py
View File

@@ -108,10 +108,10 @@ async def fetch_stats(bvid: str, asess) -> APIData:
info["rank"] = has_rank.text.strip() info["rank"] = has_rank.text.strip()
try: try:
info["like"] = html.find(".ops .like", first=True).text.strip() info["like"] = html.find(".toolbar-left .like", first=True).text.strip()
info["coin"] = html.find(".ops .coin", first=True).text.strip() info["coin"] = html.find(".toolbar-left .coin", first=True).text.strip()
info["collect"] = html.find(".ops .collect", first=True).text.strip() info["collect"] = html.find(".toolbar-left .collect", first=True).text.strip()
info["share"] = html.find(".ops .share", first=True).text.strip() info["share"] = html.find(".toolbar-left .share", first=True).text.strip()
except AttributeError: except AttributeError:
log.warning(f"cant' get stats from '{bvid}', use default.") log.warning(f"cant' get stats from '{bvid}', use default.")
return {k: "" for k in stats} return {k: "" for k in stats}

11
projects/crawling/bilibili_crawler/bilibili_crawler/spiders/bilibili.py
View File

@@ -49,15 +49,14 @@ class BilibiliSpider(scrapy.Spider):
loader = DefaultLoader( loader = DefaultLoader(
item=VideoData(), response=response, selector=response.selector item=VideoData(), response=response, selector=response.selector
) )
loader.add_css("like", ".ops .like::text") loader.add_css("like", ".toolbar-left .like .info-text::text")
loader.add_css("coin", ".ops .coin::text") loader.add_css("coin", ".toolbar-left .coin .info-text::text")
loader.add_css("collect", ".ops .collect::text") loader.add_css("collect", ".toolbar-left .collect .info-text::text")
loader.add_css("share", ".ops .share::text") loader.add_css("share", ".toolbar-left .share .info-text::text")
loader.add_css("rank", ".video-data .rank::text") loader.add_css("rank", ".video-data .honor-text::text")
video_data = loader.load_item() video_data = loader.load_item()
data = dict(**api_data, **video_data) data = dict(**api_data, **video_data)
yield data yield data