research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
sophon
2025-07-30 14:49:14 +08:00
parent 73e8833bd0
commit c3f765f6c7
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy_proj/scrapy_proj/settings.py
View File

@ -86,7 +86,14 @@ STATS_EXPORT_SCRIPT = 'scrapy_proj/extensions/push_to_wecom.sh' # 本地shell
TWISTED_REACTOR = 'twisted.internet.epollreactor.EPollReactor' # 适用于Linux
# 允许 404 状态码被 Spider 处理
HTTPERROR_ALLOWED_CODES = [404]
#HTTPERROR_ALLOWED_CODES = [404]
# 允许 Spider 接收所有非 200 响应
HTTPERROR_ALLOW_ALL = True
RETRY_ENABLED = True
RETRY_TIMES = 5
RETRY_HTTP_CODES = [500, 502, 503, 504, 521, 522, 524]
RETRY_BACKOFF_BASE = 3
RETRY_BACKOFF_MAX = 60
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = "scrapy_proj (+http://www.yourdomain.com)"