add some scripts.

This commit is contained in:
2025-02-27 10:31:28 +08:00
parent 37fc083a17
commit f482a3353f
3 changed files with 296 additions and 2 deletions

View File

@ -102,7 +102,11 @@ def fetch_and_parse_page(url, scraper):
blacked_cnt = 0
tushy_cnt = 0
x_art_cnt = 0
role = 'personal'
table = soup.find('table', id='personal')
if table is None:
table = soup.find('table', id='directoral')
role = 'directoral'
if table:
# 找到thead并跳过
thead = table.find('thead')
@ -174,7 +178,7 @@ def fetch_and_parse_page(url, scraper):
data['tushy_cnt'] = tushy_cnt
data['x_art_cnt'] = x_art_cnt
return data, movies
return data, {'role': role, 'movies' : movies}
except RequestException as e:
logging.error(f"Error fetching {url}: {e}")
return None, None
@ -312,7 +316,7 @@ def main():
'href': href,
'person': person,
**data,
'movies': movies if movies else []
'credits': movies if movies else {}
}
write_person_json(person.strip(), href, full_data)