research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscar
2025-03-16 15:51:17 +08:00
parent 6a89b25792
commit f45ad35c6a
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/iafd/src/iafd_scraper.py
View File

@ -47,14 +47,14 @@ def fetch_page(url, validator, max_retries=3, parser="html.parser", preprocessor
# 处理 HTTP 状态码 # 处理 HTTP 状态码
if response.status_code == 404: if response.status_code == 404:
logging.warning(f"Page not found (404): {url}") logging.debug(f"Page not found (404): {url}")
return None, 404 # 直接返回 404，调用方可以跳过 return None, 404 # 直接返回 404，调用方可以跳过
response.raise_for_status() # 处理 HTTP 错误 response.raise_for_status() # 处理 HTTP 错误
# 过期的网页，与404相同处理 # 过期的网页，与404相同处理
if "invalid or outdated page" in response.text.lower(): if "invalid or outdated page" in response.text.lower():
logging.warning(f"invalid or outdated page: {url}") logging.debug(f"invalid or outdated page: {url}")
return None, 404 # 直接返回 404，调用方可以跳过 return None, 404 # 直接返回 404，调用方可以跳过
# 预处理 HTML（如果提供了 preprocessor） # 预处理 HTML（如果提供了 preprocessor）