research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-03-18 17:59:22 +08:00
parent a4ea79d4db
commit ea3e62c0d3
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
aabook/src/fetch.py
View File

@ -27,9 +27,9 @@ def fetch_book_list():
for item in list_data:
row_id = db_tools.insert_or_update_common(item, db_tools.tbl_name_books)
if row_id:
logging.debug(f'insert one book. row_id: {row_id}, name: {item['name']}')
logging.debug(f"insert one book. row_id: {row_id}, name: {item['name']}")
else:
logging.warning(f'insert book error. name: {item['name']}, href: {item['href']}')
logging.warning(f"insert book error. name: {item['name']}, href: {item['href']}")
if next_url is None:
logging.info(f'get all pages.')
return True
@ -69,13 +69,13 @@ def fetch_chapter_content(url):
if contents:
chapter_data['contents'] = contents
else:
logging.warning(f'fetching real content faild. url: {data['content_url']}')
logging.warning(f"fetching real content faild. url: {data['content_url']}")
return None, None
else:
logging.warning(f'fetch chapter page no data. url: {url}')
return None, None
else:
logging.warning(f'fetch chapter page error. url: {url}, status_code: {status_code}')
logging.warning(f"fetch chapter page error. url: {url}, status_code: {status_code}")
return None, None
return chapter_data, next_url

2
aabook/src/scraper.py
View File

@ -180,7 +180,7 @@ def pase_chapter_list(soup, url):
section_titles = div_table_of_contents.find_all('p', class_='section_title')
sections = div_table_of_contents.find_all('ul', class_='section_list')
if len(sections) > len(section_titles): # 一般是 后者比前者多1个，最后一个是广告
logging.warning(f'sections not matched titles, url: {url}, titles: {len(section_titles)}, sections: {len(sections)}')
logging.warning(f"sections not matched titles, url: {url}, titles: {len(section_titles)}, sections: {len(sections)}")
return None
else:
for i in range(len(sections)):