lostecho/dt_audio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update json parse files

Browse Source
This commit is contained in:
lostecho
2024-05-30 00:04:46 +08:00
parent ea6a207131
commit a4c35ef4d8
4 changed files with 234 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
courses/course_database.db
View File

Binary file not shown.

224
courses/json_parser_songyi.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

12
courses/parse_course.py
View File

@@ -128,14 +128,15 @@ def query_course_by_id(course_id):
# return re.sub(r'[\r\n]', '', str(all_course_json[0]))
return all_course_json[0][0]
# 保存课程json数据文件到数据库
def save_course_json(ids):
conn = sqlite3.connect('course_database.db')
print("数据库打开成功")
c = conn.cursor()
for id in ids:
if id > 7:
continue
# if id > 7:
# continue
token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMDAwMDgzNDciLCJleHAiOjE3MTkxODk0ODQsImp0aSI6IjU3ZTJhMzdmLTMyZGEtNGQ2My1hZjQxLTY5NTRlNmU1OTg2MiIsImlhdCI6MTcxNjUxMTA4NCwiaXNzIjoiYXBwdXNlciIsInVpZCI6ImJlMmViOGIyLTFhOTItNGVmMC05ZDAwLTA1YTlkN2E2OWRiMiIsInNjaGVtZSI6Imp3dGhzIiwic2lkIjoiMWI4ZjE1ZTItYjQ5ZC00MmRmLWEwNDUtZmQxYTUwNzI5ZjkxIn0.IO7C2gtsi8lMdrOgWGNuxK-t2zzmDPvmI4BqISHeZEI"
json_data = request_date(id, token)
title = json_data["data"]["title"].replace(".", "_").replace("/", "_")
@@ -143,7 +144,8 @@ def save_course_json(ids):
updated_at = datetime.fromisoformat(json_data["data"]["updated_at"].replace('Z', '+00:00'))
# 插入JSON字符串到SQLite表中
c.execute("INSERT OR IGNORE INTO JSON_DATA (ID,JSON,TYPE,REMARK,CREATED_AT,UPDATED_AT) VALUES (?,?,?,?,?,?)", (id,json.dumps(json_data),"COURSE",title,created_at,updated_at))
c.execute("INSERT OR IGNORE INTO JSON_DATA (ID,JSON,TYPE,REMARK,CREATED_AT,UPDATED_AT) VALUES (?,?,?,?,?,?)",
(id, json.dumps(json_data), "COURSE", title, created_at, updated_at))
conn.commit()
secs = random.normalvariate(1, 0.4)
if secs <= 0:
@@ -151,12 +153,14 @@ def save_course_json(ids):
sleep(secs)
conn.close()
if __name__ == '__main__':
ids = get_course_id('all/course.json')
# ids = get_course_id('all/course.json')
token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMDAwMDgzNDciLCJleHAiOjE3MTkxODk0ODQsImp0aSI6IjU3ZTJhMzdmLTMyZGEtNGQ2My1hZjQxLTY5NTRlNmU1OTg2MiIsImlhdCI6MTcxNjUxMTA4NCwiaXNzIjoiYXBwdXNlciIsInVpZCI6ImJlMmViOGIyLTFhOTItNGVmMC05ZDAwLTA1YTlkN2E2OWRiMiIsInNjaGVtZSI6Imp3dGhzIiwic2lkIjoiMWI4ZjE1ZTItYjQ5ZC00MmRmLWEwNDUtZmQxYTUwNzI5ZjkxIn0.IO7C2gtsi8lMdrOgWGNuxK-t2zzmDPvmI4BqISHeZEI"
# json_data = request_date(ids[0], token)
# json_data = query_course_by_id(488)
# get_audio(json_data)
# print(json_data)
# get_all_attachments(json_data)
ids = [489, 490, 491]
save_course_json(ids)

3
requirements.txt
View File

@@ -1,4 +1,5 @@
requests>=2.31.0
ipython~=8.24.0
Scrapy~=2.11.2
pydub~=0.25.1
pydub~=0.25.1
tqdm~=4.66.4