research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-04-25 10:30:25 +08:00
parent 806e2d98c2
commit 4c80e72a98
2 changed files with 28 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
javdb/src/fetch.py
View File

@ -97,7 +97,9 @@ def fetch_movies_by_maker():
if debug:
url_list = db_tools.query_maker_hrefs(name='muramura')
for url in url_list:
for row in url_list:
url = row['href']
row_id = row['id']
# 去掉可下载的标志（如果有）
next_url = utils.remove_url_query(url)
while next_url:
@ -107,7 +109,7 @@ def fetch_movies_by_maker():
list_data, next_url = scraper.parse_maker_detail(soup, next_url)
if list_data:
for movie in list_data:
tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], from_movie_makers=1)
tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], from_movie_makers=1, maker_id=row_id)
if tmp_id:
logging.debug(f'insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}')
else:
@ -132,7 +134,9 @@ def fetch_movies_by_series():
if debug:
url_list = db_tools.query_series_hrefs(name='10musume')
for url in url_list:
for row in url_list:
url = row['href']
row_id = row['id']
# 去掉可下载的标志（如果有）
next_url = utils.remove_url_query(url)
while next_url:
@ -142,7 +146,7 @@ def fetch_movies_by_series():
list_data, next_url = scraper.parse_series_detail(soup, next_url)
if list_data:
for movie in list_data:
tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], from_movie_series=1)
tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], from_movie_series=1, series_id=row_id)
if tmp_id:
logging.debug(f'insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}')
else:
@ -317,14 +321,14 @@ function_map = {
}
# 主函数
def main(cmd):
def main(cmd, args):
# 开启任务
task_id = db_tools.insert_task_log()
if task_id is None:
logging.warning(f'insert task log error.')
return None
logging.info(f'running task. id: {task_id}, debug: {debug}, force: {force}, skip_local: {skip_local}, cmd: {cmd}')
logging.info(f"running task. id: {task_id}, args: {args}")
# 执行指定的函数
if cmd:
@ -388,4 +392,4 @@ if __name__ == "__main__":
args = parser.parse_args()
set_env(args)
main(args.cmd)
main(args.cmd, args)