research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-04-26 17:44:21 +08:00
parent 275252e97d
commit a7b04ad56d
1 changed files with 34 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
javdb/src/fetch.py
View File

@ -14,7 +14,7 @@ config.setup_logging()
debug = False
skip_local = False
fast_mode = False
scan_mode = 0
update_mode = 0
# 获取演员列表
@ -91,8 +91,10 @@ def fetch_movies_by_maker():
if debug:
url_list = db_tools.query_maker_hrefs(name='muramura')
else:
if fast_mode:
if scan_mode==1:
url_list = db_tools.query_maker_hrefs(from_list=1)
elif scan_mode==0:
url_list = db_tools.query_maker_hrefs(from_list=0)
else:
url_list = db_tools.query_maker_hrefs()
@ -130,8 +132,10 @@ def fetch_movies_by_series():
if debug:
url_list = db_tools.query_series_hrefs(name='10musume')
else:
if fast_mode:
if scan_mode == 1:
url_list = db_tools.query_series_hrefs(from_list=1)
elif scan_mode == 0:
url_list = db_tools.query_series_hrefs(from_list=0)
else:
url_list = db_tools.query_series_hrefs()
@ -168,8 +172,10 @@ def fetch_movies_by_publishers():
if debug:
url_list = db_tools.query_publishers_hrefs(limit=1)
else:
if fast_mode:
if scan_mode == 1:
url_list = db_tools.query_publishers_hrefs(from_list=1)
elif scan_mode == 0:
url_list = db_tools.query_publishers_hrefs(from_list=0)
else:
url_list = db_tools.query_publishers_hrefs()
@ -209,8 +215,12 @@ def fetch_performers_detail():
abnormal_codes = [scraper.http_code_404, scraper.http_code_login]
def get_performers(**kwargs):
if fast_mode:
if scan_mode == 1:
kwargs["from_actor_list"] = 1
elif scan_mode == 0:
kwargs["from_actor_list"] = 0
else:
logging.debug(f"scan all records")
kwargs["order_by"] = 'id asc'
return db_tools.query_actors(limit=limit_count, **kwargs)
@ -296,8 +306,12 @@ def fetch_movies_detail():
abnormal_codes = [scraper.http_code_404, scraper.http_code_login]
def get_movies(**kwargs):
if fast_mode:
if scan_mode == 1:
kwargs["uncensored"] = 1
elif scan_mode == 0:
kwargs["uncensored"] = 0
else:
logging.debug(f"scan all records.")
kwargs["order_by"] = 'id asc'
return db_tools.query_movie_hrefs(limit=limit_count, **kwargs)
@ -415,8 +429,8 @@ def set_env(args):
global skip_local
skip_local = args.skip_local
global fast_mode
fast_mode = args.fast_mode
global scan_mode
scan_mode = args.scan_mode
global update_mode
if args.update:
@ -428,11 +442,21 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description='fetch javdb data.')
parser.add_argument("--cmd", type=str, help=f"Comma-separated list of function shortcuts: {keys_str}")
parser.add_argument('--update', type=int, choices=[0, 1, 2, 3, 4], default=0, help='0-只遍历is_full_data=0, 1-只遍历is_full_data=1, 2-遍历is_full_data<=1, 3-只遍历is_full_data>1(异常数据), 4-遍历所有')
parser.add_argument('--fast_mode', action='store_true', help='只遍历所有 uncensored 的 makers/series/actors/movies')
parser.add_argument('--update', type=int, choices=[0, 1, 2, 3, 4], default=0, help='0-只遍历is_full_data=0(默认), 1-只遍历is_full_data=1, 2-遍历is_full_data<=1, 3-只遍历is_full_data>1(异常数据), 4-遍历所有')
parser.add_argument('--scan_mode', type=int, choices=[0, 1, 2], default=2, help='0-只遍历所有 uncensored 的 makers/series/actors/movies, 1-与前者相反, 2-全量(默认)')
parser.add_argument('--skip_local', action='store_true', help='如果本地缓存了页面，则跳过数据库操作')
parser.add_argument('--debug', action='store_true', help='Enable debug mode (limit records)')
args = parser.parse_args()
set_env(args)
main(args.cmd, args)
'''
python3 ./fetch # 遍历新增的所有记录
python3 ./fetch --scan_mode=0 # 遍历新增的 uncensored 记录(无码片)
python3 ./fetch --scan_mode=1 # 遍历新增的 非uncensored 记录(有码片)
python3 ./fetch --update=4 # 遍历全量的记录
python3 ./fetch --update=4 --scan_mode=0 # 遍历全量的 uncensored 记录(无码片)
python3 ./fetch --update=4 --scan_mode=1 # 遍历全量的 非uncensored 记录(有码片)
'''