lostecho/dt_audio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update json parse files

Browse Source
This commit is contained in:
lostecho
2024-05-30 00:04:46 +08:00
parent ea6a207131
commit a4c35ef4d8
4 changed files with 234 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
courses/course_database.db
View File

Binary file not shown.

224
courses/json_parser_songyi.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

12
courses/parse_course.py
View File

@@ -128,14 +128,15 @@ def query_course_by_id(course_id):
# return re.sub(r'[\r\n]', '', str(all_course_json[0])) # return re.sub(r'[\r\n]', '', str(all_course_json[0]))
return all_course_json[0][0] return all_course_json[0][0]
# 保存课程json数据文件到数据库 # 保存课程json数据文件到数据库
def save_course_json(ids): def save_course_json(ids):
conn = sqlite3.connect('course_database.db') conn = sqlite3.connect('course_database.db')
print("数据库打开成功") print("数据库打开成功")
c = conn.cursor() c = conn.cursor()
for id in ids: for id in ids:
if id > 7: # if id > 7:
continue # continue
token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMDAwMDgzNDciLCJleHAiOjE3MTkxODk0ODQsImp0aSI6IjU3ZTJhMzdmLTMyZGEtNGQ2My1hZjQxLTY5NTRlNmU1OTg2MiIsImlhdCI6MTcxNjUxMTA4NCwiaXNzIjoiYXBwdXNlciIsInVpZCI6ImJlMmViOGIyLTFhOTItNGVmMC05ZDAwLTA1YTlkN2E2OWRiMiIsInNjaGVtZSI6Imp3dGhzIiwic2lkIjoiMWI4ZjE1ZTItYjQ5ZC00MmRmLWEwNDUtZmQxYTUwNzI5ZjkxIn0.IO7C2gtsi8lMdrOgWGNuxK-t2zzmDPvmI4BqISHeZEI" token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMDAwMDgzNDciLCJleHAiOjE3MTkxODk0ODQsImp0aSI6IjU3ZTJhMzdmLTMyZGEtNGQ2My1hZjQxLTY5NTRlNmU1OTg2MiIsImlhdCI6MTcxNjUxMTA4NCwiaXNzIjoiYXBwdXNlciIsInVpZCI6ImJlMmViOGIyLTFhOTItNGVmMC05ZDAwLTA1YTlkN2E2OWRiMiIsInNjaGVtZSI6Imp3dGhzIiwic2lkIjoiMWI4ZjE1ZTItYjQ5ZC00MmRmLWEwNDUtZmQxYTUwNzI5ZjkxIn0.IO7C2gtsi8lMdrOgWGNuxK-t2zzmDPvmI4BqISHeZEI"
json_data = request_date(id, token) json_data = request_date(id, token)
title = json_data["data"]["title"].replace(".", "_").replace("/", "_") title = json_data["data"]["title"].replace(".", "_").replace("/", "_")
@@ -143,7 +144,8 @@ def save_course_json(ids):
updated_at = datetime.fromisoformat(json_data["data"]["updated_at"].replace('Z', '+00:00')) updated_at = datetime.fromisoformat(json_data["data"]["updated_at"].replace('Z', '+00:00'))
# 插入JSON字符串到SQLite表中 # 插入JSON字符串到SQLite表中
c.execute("INSERT OR IGNORE INTO JSON_DATA (ID,JSON,TYPE,REMARK,CREATED_AT,UPDATED_AT) VALUES (?,?,?,?,?,?)", (id,json.dumps(json_data),"COURSE",title,created_at,updated_at)) c.execute("INSERT OR IGNORE INTO JSON_DATA (ID,JSON,TYPE,REMARK,CREATED_AT,UPDATED_AT) VALUES (?,?,?,?,?,?)",
(id, json.dumps(json_data), "COURSE", title, created_at, updated_at))
conn.commit() conn.commit()
secs = random.normalvariate(1, 0.4) secs = random.normalvariate(1, 0.4)
if secs <= 0: if secs <= 0:
@@ -151,12 +153,14 @@ def save_course_json(ids):
sleep(secs) sleep(secs)
conn.close() conn.close()
if __name__ == '__main__': if __name__ == '__main__':
ids = get_course_id('all/course.json') # ids = get_course_id('all/course.json')
token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMDAwMDgzNDciLCJleHAiOjE3MTkxODk0ODQsImp0aSI6IjU3ZTJhMzdmLTMyZGEtNGQ2My1hZjQxLTY5NTRlNmU1OTg2MiIsImlhdCI6MTcxNjUxMTA4NCwiaXNzIjoiYXBwdXNlciIsInVpZCI6ImJlMmViOGIyLTFhOTItNGVmMC05ZDAwLTA1YTlkN2E2OWRiMiIsInNjaGVtZSI6Imp3dGhzIiwic2lkIjoiMWI4ZjE1ZTItYjQ5ZC00MmRmLWEwNDUtZmQxYTUwNzI5ZjkxIn0.IO7C2gtsi8lMdrOgWGNuxK-t2zzmDPvmI4BqISHeZEI" token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiIxMDAwMDgzNDciLCJleHAiOjE3MTkxODk0ODQsImp0aSI6IjU3ZTJhMzdmLTMyZGEtNGQ2My1hZjQxLTY5NTRlNmU1OTg2MiIsImlhdCI6MTcxNjUxMTA4NCwiaXNzIjoiYXBwdXNlciIsInVpZCI6ImJlMmViOGIyLTFhOTItNGVmMC05ZDAwLTA1YTlkN2E2OWRiMiIsInNjaGVtZSI6Imp3dGhzIiwic2lkIjoiMWI4ZjE1ZTItYjQ5ZC00MmRmLWEwNDUtZmQxYTUwNzI5ZjkxIn0.IO7C2gtsi8lMdrOgWGNuxK-t2zzmDPvmI4BqISHeZEI"
# json_data = request_date(ids[0], token) # json_data = request_date(ids[0], token)
# json_data = query_course_by_id(488) # json_data = query_course_by_id(488)
# get_audio(json_data) # get_audio(json_data)
# print(json_data) # print(json_data)
# get_all_attachments(json_data) # get_all_attachments(json_data)
ids = [489, 490, 491]
save_course_json(ids) save_course_json(ids)

1
requirements.txt
View File

@@ -2,3 +2,4 @@ requests>=2.31.0
ipython~=8.24.0 ipython~=8.24.0
Scrapy~=2.11.2 Scrapy~=2.11.2
pydub~=0.25.1 pydub~=0.25.1
tqdm~=4.66.4