add songyi course list parse

2023-08-01 18:26:00 +08:00
parent 681dd449cd
commit 4f9f64821e
6 changed files with 114 additions and 94 deletions
--- a/parse-json-file.py
+++ b/parse-json-file.py
--- a/requests_get_data.ipynb
+++ b/requests_get_data.ipynb
@@ -1,94 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {
-    "collapsed": true,
-    "ExecuteTime": {
-     "end_time": "2023-07-21T14:15:19.727737100Z",
-     "start_time": "2023-07-21T14:15:19.256727800Z"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "200\n",
-      "{'id': 7899, 'title': '1.1-思考的真相-定义', 'description': '', 'topic': '', 'speaker_ids': ['b76e0418-2d94-4b36-a6dd-497933b79923'], 'is_published': True, 'is_open': False, 'is_charge': False, 'poster_url': '', 'started_at': '2023-07-21T05:04:41.357289Z', 'finished_at': '2023-07-21T05:04:41.357289Z', 'record_type': 'prerecorded', 'open_comment': False, 'tags': [], 'speakers': [{'user_id': 'b76e0418-2d94-4b36-a6dd-497933b79923', 'group_id': '7000102069', 'version': 1485, 'full_name': '笑来', 'bio': '', 'identity_number': 26806, 'avatar_url': 'https://mixin-images.zeromesh.net/3kCJbGHrViT3ARf3VVYvKEiOao5Kyh8cW3XaJa_GlHYNE2bqvzDlpbExFC_lkNGQNaX3scVaRMQeJelHQT4Qb2pU=s256', 'state': 'paid', 'blocked': 0, 'active_at': '2023-07-21T09:38:05.392813Z', 'subscribed_at': '2020-12-03T03:14:33.146439Z', 'expired_at': '2123-07-03T12:34:56.0001Z', 'created_at': '2019-06-24T11:15:22Z', 'uiam_id': 37498, 'joined_at': '2019-07-21T04:28:46.952Z', 'divided_to': '7000103435', 'divided_at': '2020-06-10T20:15:19.641593Z', 'role': '', 'token': '', 'refresh_token': '', 'provider': '', 'policies': None, 'privacy_user': None}], 'message_count': 0, 'group_id': '7000102069', 'product_id': 0, 'course_albums': [], 'is_favored': False}\n"
-     ]
-    }
-   ],
-   "source": [
-    "import requests\n",
-    "import json\n",
-    "\n",
-    "headers = {\n",
-    "    'authority': 'xuexi-courses-api.songy.info',\n",
-    "    'accept': 'application/json, text/plain, */*',\n",
-    "    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',\n",
-    "    'authorization': 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiI5MTU3NSIsImV4cCI6MTY5MDU1MDIzNywianRpIjoiOWJhNzZjZDgtZTgxMi00YzA2LWE2Y2UtMTIxY2IxNmYyYjMyIiwiaWF0IjoxNjg5OTQ1NDM3LCJpc3MiOiIzMDAyMiIsInN1YiI6IjcwMDAxMDIwNjkiLCJ1aWQiOiIyOTFmZjQzNC1iOGQyLTQyNzUtYWUzZi1lNzNlYWEwMzU0ODQiLCJtb2RlIjoiY2xpZW50IiwidHlwIjoiYXBwdXNlciIsIm9hcCI6Im1peGluIiwic2lkIjoiYTU0YmRhOGQyNDIwNDczZTgwOGEzM2ZlZmQ0Y2U1ZjIifQ.m4hjlbZz7a--rwRfn_vRNU6TTuXg-_r0c6N6eKiika0',\n",
-    "    'dnt': '1',\n",
-    "    'origin': 'https://xuexi-courses.firesbox.com',\n",
-    "    'referer': 'https://xuexi-courses.firesbox.com/',\n",
-    "    'sec-ch-ua': '\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Microsoft Edge\";v=\"114\"',\n",
-    "    'sec-ch-ua-mobile': '?0',\n",
-    "    'sec-ch-ua-platform': '\"Windows\"',\n",
-    "    'sec-fetch-dest': 'empty',\n",
-    "    'sec-fetch-mode': 'cors',\n",
-    "    'sec-fetch-site': 'cross-site',\n",
-    "    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.82',\n",
-    "}\n",
-    "\n",
-    "params = {\n",
-    "    'nonce': 'af674e65-49d8-4c2c-8437-a4053a7c7b47',\n",
-    "    'q': '',\n",
-    "    'offset': '0',\n",
-    "    'limit': '20',\n",
-    "    'sort': 'newest-first',\n",
-    "}\n",
-    "\n",
-    "response = requests.get('https://xuexi-courses-api.songy.info/v1/courses/', params=params, headers=headers)\n",
-    "\n",
-    "print(response.status_code)\n",
-    "# print(response.text)\n",
-    "\n",
-    "courses_date = json.loads(response.text)\n",
-    "\n",
-    "courses = courses_date['courses']\n",
-    "print(courses[1])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [],
-   "metadata": {
-    "collapsed": false
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
--- a/songyi/courses.json
+++ b/songyi/courses.json
--- a/songyi/parse-course-list.py
+++ b/songyi/parse-course-list.py
@@ -0,0 +1,59 @@
+# 爬取学升中的课程目录
+
+import json
+import os
+import random
+import time
+
+import requests
+
+# 使用BearerToken认证
+BearerToken = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMDAwMDgzNDciLCJleHAiOjE2OTMwNjMxNTEsImp0aSI6ImUxMGYxMDEyLTY3MDYtNGJiYS1iMTg2LWVmNGViZmVhZDVkMCIsImlhdCI6MTY5MDM4NDc1MSwiaXNzIjoiYXBwdXNlciIsInVpZCI6ImJlMmViOGIyLTFhOTItNGVmMC05ZDAwLTA1YTlkN2E2OWRiMiIsInNjaGVtZSI6Imp3dGhzIiwic2lkIjoiMWI4ZjE1ZTItYjQ5ZC00MmRmLWEwNDUtZmQxYTUwNzI5ZjkxIn0.9TAgxEBSg_Ckbyif8IsC18MMqb9I2qhSwcPbNFk7JhE'
+
+# 请求头
+headers = {
+    'authority': 'bandu-api.songy.info',
+    'accept': '*/*',
+    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+    'authorization': 'Bearer ' + BearerToken,
+    'dnt': '1',
+    'origin': 'https://webapp.songy.info',
+    'referer': 'https://webapp.songy.info/',
+    'sec-ch-ua': '"Not/A)Brand";v="99", "Microsoft Edge";v="115", "Chromium";v="115"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"Windows"',
+    'sec-fetch-dest': 'empty',
+    'sec-fetch-mode': 'cors',
+    'sec-fetch-site': 'same-site',
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.188',
+}
+
+# 请求参数
+params = {
+    'q': '',
+    'offset': '0',
+    'limit': '20',
+    # 'sort': 'oldest-first',
+}
+
+has_more = True
+offset = 0
+# data_item = []
+
+while(has_more):
+
+    time.sleep(1 + random.uniform(0.1, 1.2))
+    response = requests.get('https://bandu-api.songy.info/v1/communities/80000002/courses', params=params, headers=headers)
+    data = json.loads(response.text);
+    # data_item += data['data']['items']
+    has_more = data['data']['has_more']
+    offset += 20
+    print(offset)
+    params['offset'] = str(offset)
+    with open(os.path.join('courses.json'), 'a') as file:
+        file.write(response.text)
+    # if offset == 40:
+    #     break
+
+
+
--- a/songyi/parse-course-message.py
+++ b/songyi/parse-course-message.py
@@ -0,0 +1,27 @@
+import requests
+
+BearerToken = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMDAwMDgzNDciLCJleHAiOjE2OTMwNjMxNTEsImp0aSI6ImUxMGYxMDEyLTY3MDYtNGJiYS1iMTg2LWVmNGViZmVhZDVkMCIsImlhdCI6MTY5MDM4NDc1MSwiaXNzIjoiYXBwdXNlciIsInVpZCI6ImJlMmViOGIyLTFhOTItNGVmMC05ZDAwLTA1YTlkN2E2OWRiMiIsInNjaGVtZSI6Imp3dGhzIiwic2lkIjoiMWI4ZjE1ZTItYjQ5ZC00MmRmLWEwNDUtZmQxYTUwNzI5ZjkxIn0.9TAgxEBSg_Ckbyif8IsC18MMqb9I2qhSwcPbNFk7JhE'
+
+headers = {
+    'authority': 'bandu-api.songy.info',
+    'accept': '*/*',
+    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+    'authorization': 'Bearer ' + BearerToken,
+    'dnt': '1',
+    'origin': 'https://webapp.songy.info',
+    'referer': 'https://webapp.songy.info/',
+    'sec-ch-ua': '"Not/A)Brand";v="99", "Microsoft Edge";v="115", "Chromium";v="115"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"Windows"',
+    'sec-fetch-dest': 'empty',
+    'sec-fetch-mode': 'cors',
+    'sec-fetch-site': 'same-site',
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.188',
+}
+
+class_id = '80000002'
+course_id = '6770'
+
+response = requests.get('https://bandu-api.songy.info/v1/communities/'+ class_id +'/courses/' + course_id + '/messages', headers=headers)
+
+print(response.json())
--- a/songyi/read_list.py
+++ b/songyi/read_list.py
@@ -0,0 +1,16 @@
+# 获取课程列表中的id以及描述信息
+
+import json
+
+with open('courses.json', encoding="UTF-8") as file:
+    str_data = file.readline()
+    courses = json.loads(str_data)
+    # print(type(courses))
+
+    # 获取课程序号及名称存入list中
+    courses_list = []
+    for item in courses['data']['items']:
+        courses_dict = {'id': item['id'], 'title': item['title'], 'description': item['description']}
+        courses_list.append(courses_dict)
+
+    print(courses_list)