add some scripts.
This commit is contained in:
@ -102,7 +102,11 @@ def fetch_and_parse_page(url, scraper):
|
||||
blacked_cnt = 0
|
||||
tushy_cnt = 0
|
||||
x_art_cnt = 0
|
||||
role = 'personal'
|
||||
table = soup.find('table', id='personal')
|
||||
if table is None:
|
||||
table = soup.find('table', id='directoral')
|
||||
role = 'directoral'
|
||||
if table:
|
||||
# 找到thead并跳过
|
||||
thead = table.find('thead')
|
||||
@ -174,7 +178,7 @@ def fetch_and_parse_page(url, scraper):
|
||||
data['tushy_cnt'] = tushy_cnt
|
||||
data['x_art_cnt'] = x_art_cnt
|
||||
|
||||
return data, movies
|
||||
return data, {'role': role, 'movies' : movies}
|
||||
except RequestException as e:
|
||||
logging.error(f"Error fetching {url}: {e}")
|
||||
return None, None
|
||||
@ -312,7 +316,7 @@ def main():
|
||||
'href': href,
|
||||
'person': person,
|
||||
**data,
|
||||
'movies': movies if movies else []
|
||||
'credits': movies if movies else {}
|
||||
}
|
||||
write_person_json(person.strip(), href, full_data)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user