research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-04-23 17:13:35 +08:00
parent f42fd2177b
commit f6385b83e4
4 changed files with 296 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
javdb/src/fetch.py
View File

@ -148,10 +148,15 @@ def fetch_movies_by_series():
# 更新演员信息
def fetch_performers_detail():
limit_count = 5 if debug else 100
perfomers_list = []
last_perfomer_id = 0
while True:
# 每次从数据库中取一部分，避免一次全量获取
perfomers_list = db_tools.query_actors(is_full_data=0, limit=100)
# 每次从数据库中取一部分，避免一次全量获取
if force: # 从头逐个遍历
perfomers_list = db_tools.query_actors(start_id=last_perfomer_id, is_full_data_not_in=[2,3], order_by='id asc', limit=limit_count, from_actor_list=1)
else: # 只做更新
perfomers_list = db_tools.query_actors(is_full_data=0, limit=limit_count)
if len(perfomers_list) < 1:
logging.info(f'all performers fetched.')
break
@ -189,21 +194,27 @@ def fetch_performers_detail():
})
if performer_id:
logging.info(f'insert one person, id: {performer_id}, person: ({person}), url: {url}')
last_perfomer_id = performer_id
else:
logging.warning(f'insert person: ({person}) {url} failed.')
time.sleep(0.5)
# 调试break
if debug:
return True
# 更新影片信息
def fetch_movies_detail():
limit_count = 10 if debug else 100
movies_list = []
while True:
movies_list = db_tools.query_movie_hrefs(is_full_data=0, limit=100)
last_movie_id = 0
while True:
if force: # 从头逐个遍历
movies_list = db_tools.query_movie_hrefs(start_id=last_movie_id, is_full_data_not_in=[2,3], order_by='id asc', limit=limit_count, from_actor_list=1)
else: # 只做更新
movies_list = db_tools.query_movie_hrefs(is_full_data=0, limit=limit_count)
if len(movies_list) < 1:
logging.info(f'all movies fetched.')
break
last_movie_id = 0
succ_count = 0
for movie in movies_list:
url = movie['href']
@ -231,7 +242,7 @@ def fetch_movies_detail():
logging.warning(f'401 page(need login). id: {movie_id}, title: ({title}), url: {url}, Skiping...')
else:
logging.warning(f'fetch_page error. url: {url}')
time.sleep(1)
time.sleep(0.5)
logging.info(f'total request: {len(movies_list)}, succ: {succ_count}. last movie id: {last_movie_id}')
# 调试增加break
if debug:
@ -253,6 +264,9 @@ function_map = {
def main(cmd, args_debug, args_force):
global debug
debug = args_debug
if debug:
logger = logging.getLogger()
#logger.setLevel(logging.DEBUG)
global force
force = args_force