From a7b04ad56d52e6b71ecc57e7b711b7dc18a09b96 Mon Sep 17 00:00:00 2001 From: oscarz Date: Sat, 26 Apr 2025 17:44:21 +0800 Subject: [PATCH] modify scripts --- javdb/src/fetch.py | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/javdb/src/fetch.py b/javdb/src/fetch.py index ea103c5..9ca5fb8 100644 --- a/javdb/src/fetch.py +++ b/javdb/src/fetch.py @@ -14,7 +14,7 @@ config.setup_logging() debug = False skip_local = False -fast_mode = False +scan_mode = 0 update_mode = 0 # 获取演员列表 @@ -91,8 +91,10 @@ def fetch_movies_by_maker(): if debug: url_list = db_tools.query_maker_hrefs(name='muramura') else: - if fast_mode: + if scan_mode==1: url_list = db_tools.query_maker_hrefs(from_list=1) + elif scan_mode==0: + url_list = db_tools.query_maker_hrefs(from_list=0) else: url_list = db_tools.query_maker_hrefs() @@ -130,8 +132,10 @@ def fetch_movies_by_series(): if debug: url_list = db_tools.query_series_hrefs(name='10musume') else: - if fast_mode: + if scan_mode == 1: url_list = db_tools.query_series_hrefs(from_list=1) + elif scan_mode == 0: + url_list = db_tools.query_series_hrefs(from_list=0) else: url_list = db_tools.query_series_hrefs() @@ -168,8 +172,10 @@ def fetch_movies_by_publishers(): if debug: url_list = db_tools.query_publishers_hrefs(limit=1) else: - if fast_mode: + if scan_mode == 1: url_list = db_tools.query_publishers_hrefs(from_list=1) + elif scan_mode == 0: + url_list = db_tools.query_publishers_hrefs(from_list=0) else: url_list = db_tools.query_publishers_hrefs() @@ -209,8 +215,12 @@ def fetch_performers_detail(): abnormal_codes = [scraper.http_code_404, scraper.http_code_login] def get_performers(**kwargs): - if fast_mode: + if scan_mode == 1: kwargs["from_actor_list"] = 1 + elif scan_mode == 0: + kwargs["from_actor_list"] = 0 + else: + logging.debug(f"scan all records") kwargs["order_by"] = 'id asc' return db_tools.query_actors(limit=limit_count, **kwargs) @@ -296,8 +306,12 @@ def fetch_movies_detail(): abnormal_codes = [scraper.http_code_404, scraper.http_code_login] def get_movies(**kwargs): - if fast_mode: + if scan_mode == 1: kwargs["uncensored"] = 1 + elif scan_mode == 0: + kwargs["uncensored"] = 0 + else: + logging.debug(f"scan all records.") kwargs["order_by"] = 'id asc' return db_tools.query_movie_hrefs(limit=limit_count, **kwargs) @@ -415,8 +429,8 @@ def set_env(args): global skip_local skip_local = args.skip_local - global fast_mode - fast_mode = args.fast_mode + global scan_mode + scan_mode = args.scan_mode global update_mode if args.update: @@ -428,11 +442,21 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='fetch javdb data.') parser.add_argument("--cmd", type=str, help=f"Comma-separated list of function shortcuts: {keys_str}") - parser.add_argument('--update', type=int, choices=[0, 1, 2, 3, 4], default=0, help='0-只遍历is_full_data=0, 1-只遍历is_full_data=1, 2-遍历is_full_data<=1, 3-只遍历is_full_data>1(异常数据), 4-遍历所有') - parser.add_argument('--fast_mode', action='store_true', help='只遍历所有 uncensored 的 makers/series/actors/movies') + parser.add_argument('--update', type=int, choices=[0, 1, 2, 3, 4], default=0, help='0-只遍历is_full_data=0(默认), 1-只遍历is_full_data=1, 2-遍历is_full_data<=1, 3-只遍历is_full_data>1(异常数据), 4-遍历所有') + parser.add_argument('--scan_mode', type=int, choices=[0, 1, 2], default=2, help='0-只遍历所有 uncensored 的 makers/series/actors/movies, 1-与前者相反, 2-全量(默认)') parser.add_argument('--skip_local', action='store_true', help='如果本地缓存了页面，则跳过数据库操作') parser.add_argument('--debug', action='store_true', help='Enable debug mode (limit records)') args = parser.parse_args() set_env(args) main(args.cmd, args) + +''' +python3 ./fetch # 遍历新增的所有记录 +python3 ./fetch --scan_mode=0 # 遍历新增的 uncensored 记录(无码片) +python3 ./fetch --scan_mode=1 # 遍历新增的 非uncensored 记录(有码片) + +python3 ./fetch --update=4 # 遍历全量的记录 +python3 ./fetch --update=4 --scan_mode=0 # 遍历全量的 uncensored 记录(无码片) +python3 ./fetch --update=4 --scan_mode=1 # 遍历全量的 非uncensored 记录(有码片) +''' \ No newline at end of file