research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-07-01 10:45:31 +08:00
parent 1731f46006
commit 7fe40c7f6b
1 changed files with 13 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
iafd/src/fetch.py
View File

@ -30,9 +30,9 @@ def fetch_performers_by_astro():
# 写入演员数据表 # 写入演员数据表
perfomer_id = db_tools.insert_performer_index(name=row['person'], href=row.get('href', '').lower(), from_astro_list=1) perfomer_id = db_tools.insert_performer_index(name=row['person'], href=row.get('href', '').lower(), from_astro_list=1)
if perfomer_id: if perfomer_id:
logging.debug(f'insert performer index to db. performer_id:{perfomer_id}, name: {row['person']}, href:{row['href']}') logging.debug(f"insert performer index to db. performer_id:{perfomer_id}, name: {row['person']}, href:{row['href']}")
else: else:
logging.warning(f'insert performer index failed. name: {row['person']}, href:{row['href']}') logging.warning(f"insert performer index failed. name: {row['person']}, href:{row['href']}")
else: else:
logging.warning(f'fetch astro error. {url} ...') logging.warning(f'fetch astro error. {url} ...')
@ -60,9 +60,9 @@ def fetch_performers_by_birth():
# 写入演员数据表 # 写入演员数据表
perfomer_id = db_tools.insert_performer_index(name=row['person'], href=row.get('href', '').lower(), from_birth_list=1) perfomer_id = db_tools.insert_performer_index(name=row['person'], href=row.get('href', '').lower(), from_birth_list=1)
if perfomer_id: if perfomer_id:
logging.debug(f'insert performer index to db. performer_id:{perfomer_id}, name: {row['person']}, href:{row['href']}') logging.debug(f"insert performer index to db. performer_id:{perfomer_id}, name: {row['person']}, href:{row['href']}")
else: else:
logging.warning(f'insert performer index failed. name: {row['person']}, href:{row['href']}') logging.warning(f"insert performer index failed. name: {row['person']}, href:{row['href']}")
else: else:
logging.warning(f'fetch astro error. {url} ...') logging.warning(f'fetch astro error. {url} ...')
elif status_code and status_code == 404: elif status_code and status_code == 404:
@ -85,7 +85,7 @@ def fetch_ethic_list():
for row in list_data : for row in list_data :
dist_id = db_tools.insert_or_update_ethnic({'name': row['name'], 'href': row.get('href', '')}) dist_id = db_tools.insert_or_update_ethnic({'name': row['name'], 'href': row.get('href', '')})
if dist_id: if dist_id:
logging.debug(f'insert one record into ethnic table. id:{dist_id}, name: {row['name']}, href:{row.get('href', '')}') logging.debug(f"insert one record into ethnic table. id:{dist_id}, name: {row['name']}, href:{row.get('href', '')}")
else: else:
logging.warning(f'fetch ethnic error. {url} ...') logging.warning(f'fetch ethnic error. {url} ...')
elif status_code and status_code == 404: elif status_code and status_code == 404:
@ -118,9 +118,9 @@ def fetch_performers_by_ethnic():
perfomer_id = db_tools.insert_performer_index(name=row['person'], href=row.get('href', '').lower(), from_ethnic_list=1) perfomer_id = db_tools.insert_performer_index(name=row['person'], href=row.get('href', '').lower(), from_ethnic_list=1)
if perfomer_id: if perfomer_id:
count += 1 count += 1
logging.debug(f'insert performer index to db. performer_id:{perfomer_id}, name: {row['person']}, href:{row['href']}') logging.debug("'insert performer index to db. performer_id:{perfomer_id}, name: {row['person']}, href:{row['href']}")
else: else:
logging.warning(f'insert performer index failed. name: {row['person']}, href:{row['href']}') logging.warning("'insert performer index failed. name: {row['person']}, href:{row['href']}")
else: else:
logging.warning(f'fetch astro error. {next_url} ...') logging.warning(f'fetch astro error. {next_url} ...')
elif status_code and status_code == 404: elif status_code and status_code == 404:
@ -148,7 +148,7 @@ def fetch_distributors_list():
dis_url = scraper.distributors_base_url + row['href'] dis_url = scraper.distributors_base_url + row['href']
dist_id = db_tools.insert_or_update_distributor({'name': row['name'], 'href': dis_url}) dist_id = db_tools.insert_or_update_distributor({'name': row['name'], 'href': dis_url})
if dist_id: if dist_id:
logging.debug(f'insert one record into distributors table. id:{dist_id}, name: {row['name']}, href:{dis_url}') logging.debug(f"insert one record into distributors table. id:{dist_id}, name: {row['name']}, href:{dis_url}")
else: else:
logging.warning(f'fetch astro error. {url} ...') logging.warning(f'fetch astro error. {url} ...')
elif status_code and status_code == 404: elif status_code and status_code == 404:
@ -168,7 +168,7 @@ def fetch_studios_list():
stu_url = scraper.studios_base_url + row['href'] stu_url = scraper.studios_base_url + row['href']
stu_id = db_tools.insert_or_update_studio({'name': row['name'], 'href': stu_url}) stu_id = db_tools.insert_or_update_studio({'name': row['name'], 'href': stu_url})
if stu_id: if stu_id:
logging.debug(f'insert one record into studios table. id:{stu_id}, name: {row['name']}, href:{stu_url}') logging.debug(f"insert one record into studios table. id:{stu_id}, name: {row['name']}, href:{stu_url}")
else: else:
logging.warning(f'fetch astro error. {url} ...') logging.warning(f'fetch astro error. {url} ...')
elif status_code and status_code == 404: elif status_code and status_code == 404:
@ -194,9 +194,9 @@ def fetch_movies_by_dist():
for movie in list_data: for movie in list_data:
tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], release_year=utils.to_number(movie['year']), from_dist_list=1) tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], release_year=utils.to_number(movie['year']), from_dist_list=1)
if tmp_id: if tmp_id:
logging.debug(f'insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}') logging.debug(f"insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}")
else: else:
logging.warning(f'insert movie index failed. title: {movie['title']}, href: {movie['href']}') logging.warning(f"insert movie index failed. title: {movie['title']}, href: {movie['href']}")
else : else :
logging.warning(f'parse_page_movie error. url: {url}') logging.warning(f'parse_page_movie error. url: {url}')
elif status_code and status_code == 404: elif status_code and status_code == 404:
@ -224,9 +224,9 @@ def fetch_movies_by_stu():
for movie in list_data: for movie in list_data:
tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], release_year=utils.to_number(movie['year']), from_stu_list=1) tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], release_year=utils.to_number(movie['year']), from_stu_list=1)
if tmp_id: if tmp_id:
logging.debug(f'insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}') logging.debug(f"insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}")
else: else:
logging.warning(f'insert movie index failed. title: {movie['title']}, href: {movie['href']}') logging.warning(f"insert movie index failed. title: {movie['title']}, href: {movie['href']}")
else : else :
logging.warning(f'parse_page_movie error. url: {url}') logging.warning(f'parse_page_movie error. url: {url}')
elif status_code and status_code == 404: elif status_code and status_code == 404: