research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-03-18 17:59:22 +08:00
parent a4ea79d4db
commit ea3e62c0d3
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
aabook/src/fetch.py
View File

@ -27,9 +27,9 @@ def fetch_book_list():
for item in list_data: for item in list_data:
row_id = db_tools.insert_or_update_common(item, db_tools.tbl_name_books) row_id = db_tools.insert_or_update_common(item, db_tools.tbl_name_books)
if row_id: if row_id:
logging.debug(f'insert one book. row_id: {row_id}, name: {item['name']}') logging.debug(f"insert one book. row_id: {row_id}, name: {item['name']}")
else: else:
logging.warning(f'insert book error. name: {item['name']}, href: {item['href']}') logging.warning(f"insert book error. name: {item['name']}, href: {item['href']}")
if next_url is None: if next_url is None:
logging.info(f'get all pages.') logging.info(f'get all pages.')
return True return True
@ -69,13 +69,13 @@ def fetch_chapter_content(url):
if contents: if contents:
chapter_data['contents'] = contents chapter_data['contents'] = contents
else: else:
logging.warning(f'fetching real content faild. url: {data['content_url']}') logging.warning(f"fetching real content faild. url: {data['content_url']}")
return None, None return None, None
else: else:
logging.warning(f'fetch chapter page no data. url: {url}') logging.warning(f'fetch chapter page no data. url: {url}')
return None, None return None, None
else: else:
logging.warning(f'fetch chapter page error. url: {url}, status_code: {status_code}') logging.warning(f"fetch chapter page error. url: {url}, status_code: {status_code}")
return None, None return None, None
return chapter_data, next_url return chapter_data, next_url

2
aabook/src/scraper.py
View File

@ -180,7 +180,7 @@ def pase_chapter_list(soup, url):
section_titles = div_table_of_contents.find_all('p', class_='section_title') section_titles = div_table_of_contents.find_all('p', class_='section_title')
sections = div_table_of_contents.find_all('ul', class_='section_list') sections = div_table_of_contents.find_all('ul', class_='section_list')
if len(sections) > len(section_titles): # 一般是 后者比前者多1个，最后一个是广告 if len(sections) > len(section_titles): # 一般是 后者比前者多1个，最后一个是广告
logging.warning(f'sections not matched titles, url: {url}, titles: {len(section_titles)}, sections: {len(sections)}') logging.warning(f"sections not matched titles, url: {url}, titles: {len(section_titles)}, sections: {len(sections)}")
return None return None
else: else:
for i in range(len(sections)): for i in range(len(sections)):