research/stock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscar
2025-03-15 08:02:25 +08:00
parent af92229a3e
commit 3b76c00500
4 changed files with 89 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/iafd/src/fetch.py
View File

@ -233,6 +233,7 @@ def fetch_movies_by_stu():
# 更新演员信息，单次循环
def fetch_performers_detail_once(perfomers_list):
last_performer_id = 0
for performer in perfomers_list:
url = performer['href']
person = performer['name']
@ -247,7 +248,8 @@ def fetch_performers_detail_once(perfomers_list):
**data
})
if performer_id:
logging.info(f'insert one person, id: {performer_id}, person: ({person}), url: {url}')
logging.debug(f'insert one person, id: {performer_id}, person: ({person}), url: {url}')
last_performer_id = performer_id
else:
logging.warning(f'insert person: ({person}) {url} failed.')
@ -264,10 +266,11 @@ def fetch_performers_detail_once(perfomers_list):
else:
logging.warning(f'fetch_page error. person: ({person}), url: {url}')
time.sleep(1)
return last_performer_id
# 更新演员信息
def fetch_performers_detail():
limit_count = 5 if debug else 1000
limit_count = 5 if debug else 100
perfomers_list = []
# 获取新演员的列表
@ -276,7 +279,8 @@ def fetch_performers_detail():
if len(perfomers_list) < 1:
logging.info(f'all new performers fetched. ')
break
fetch_performers_detail_once(perfomers_list)
last_perfomer_id = fetch_performers_detail_once(perfomers_list)
logging.info(f'insert {len(perfomers_list)} person. last performer id: {last_perfomer_id}')
if debug:
break
@ -286,19 +290,21 @@ def fetch_performers_detail():
if len(perfomers_list) < 1:
logging.info(f'all existed performers updated. ')
break
fetch_performers_detail_once(perfomers_list)
last_perfomer_id = fetch_performers_detail_once(perfomers_list)
logging.info(f'insert {len(perfomers_list)} person. last performer id: {last_perfomer_id}')
if debug:
break
# 更新影片信息
def fetch_movies_detail():
limit_count = 10 if debug else 1000
limit_count = 10 if debug else 100
movies_list = []
while True:
movies_list = db_tools.query_movie_hrefs(is_full_data=0, limit=limit_count)
if len(movies_list) < 1:
logging.info(f'all movies fetched.')
break
last_movie_id = 0
for movie in movies_list:
url = movie['href']
title = movie['title']
@ -314,7 +320,8 @@ def fetch_movies_detail():
movie_data['StudioHref'] = utils.dist_stu_href_rewrite(movie_data['StudioHref'].lower())
movie_id = db_tools.insert_or_update_movie(movie_data)
if movie_id:
logging.info(f'insert one movie, id: {movie_id}, title: ({title}) url: {url}')
logging.debug(f'insert one movie, id: {movie_id}, title: ({title}) url: {url}')
last_movie_id = movie_id
else:
logging.warning(f'insert movie {url} failed.')
@ -327,6 +334,7 @@ def fetch_movies_detail():
else:
logging.warning(f'fetch_page error. url: {url}')
time.sleep(1)
logging.info(f'insert {len(movies_list)} movies. last movie id: {last_movie_id}')
# 调试增加break
if debug:
return True