research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-06-27 09:20:47 +08:00
parent d91ba1cd17
commit ac7cff9454
4 changed files with 39 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/javbus/fetch.py
View File

@ -315,7 +315,8 @@ def fetch_performers_detail():
uncensored = int(performer['uncensored'])
avatar = None
if not utils.is_valid_url(url):
logging.info(f'invalid url ({url}), name: {person}. skipping...')
actor_id = db_tools.update_actor_detail_404({'href': url, 'is_full_data': craw.http_code_404})
logging.info(f'invalid url ({url}), id: {actor_id}, name: {person}. skipping...')
continue
next_url = url
@ -331,14 +332,9 @@ def fetch_performers_detail():
avatar = data.get('avatar')
all_movies.extend(data.get('movies', []))
elif status_code and status_code == craw.http_code_404:
actor_id = db_tools.update_actor_detail_404({'href': url, 'is_full_data': craw.http_code_404})
logging.warning(f'404 page. id: {actor_id}, name: ({person}), url: {url}, Skiping...')
need_insert = False
break
elif status_code and status_code == craw.http_code_redirect:
actor_id = db_tools.update_actor_detail_404({'href': url, 'is_full_data': craw.http_code_redirect})
logging.warning(f'401 page(need login). id: {actor_id}, name: ({person}), url: {url}, Skiping...')
elif status_code and status_code in [craw.http_code_404, craw.http_code_403, craw.http_code_redirect]:
actor_id = db_tools.update_actor_detail_404({'href': url, 'is_full_data': status_code})
logging.warning(f'get page http code {status_code}. id: {actor_id}, name: ({person}), url: {url}, Skiping...')
need_insert = False
break
else:
@ -414,6 +410,7 @@ def fetch_movies_detail():
curr_id = movie['id']
uncensored = int(movie['uncensored'])
if not utils.is_valid_url(url):
movie_id = db_tools.insert_or_update_movie_404({'href': url, 'is_full_data': craw.http_code_404})
logging.info(f'invalid url ({url}), row id: {curr_id}. skipping...')
continue
@ -442,12 +439,9 @@ def fetch_movies_detail():
logging.warning(f'parse_page_movie error. url: {url}')
time.sleep(2)
elif status_code and status_code == craw.http_code_404:
movie_id = db_tools.insert_or_update_movie_404({'href': url, 'is_full_data': craw.http_code_404})
logging.warning(f'404 page. id: {movie_id}, title: ({title}), url: {url}, Skiping...')
elif status_code and status_code == craw.http_code_redirect:
movie_id = db_tools.insert_or_update_movie_404({'href': url, 'is_full_data': craw.http_code_redirect})
logging.warning(f'401 page(need login). id: {movie_id}, title: ({title}), url: {url}, Skiping...')
elif status_code and status_code in [craw.http_code_404, craw.http_code_403, craw.http_code_redirect]:
movie_id = db_tools.insert_or_update_movie_404({'href': url, 'is_full_data': status_code})
logging.warning(f'get page http code {status_code}. id: {movie_id}, title: ({title}), url: {url}, Skiping...')
else:
logging.warning(f'fetch_page error. url: {url}')
time.sleep(0.5)
@ -456,7 +450,10 @@ def fetch_movies_detail():
if debug:
return True
# 重置 movies 表的 uncensored 标志位
def reset_movies_uncensored():
db_tools.reset_movies_uncensored(check_and_do=0 if debug else 1)
# 建立缩写到函数的映射
function_map = {
"actor_list": fetch_actor_list,
@ -467,6 +464,7 @@ function_map = {
"movies" : fetch_movies_detail,
"langs" : update_multi_langs,
"tags" : update_multilang_tags,
"reset_un" : reset_movies_uncensored
}
# 主函数