research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
sophon
2025-07-30 10:43:47 +08:00
parent a377e361da
commit 73e8833bd0
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scrapy_proj/scrapy_proj/extensions/failure_monitor.py
View File

@ -47,7 +47,8 @@ class FailureMonitorExtension:
# 从request.meta中获取丢弃理由 # 从request.meta中获取丢弃理由
drop_reason = reason if reason else request.meta.get('_dropreason', '未知原因') drop_reason = reason if reason else request.meta.get('_dropreason', '未知原因')
spider.logger.warning(f"request_dropped on url: {request.url} | 原因: {drop_reason}") spider.logger.warning(f"request_dropped on url: {request.url} | 原因: {drop_reason}")
self.calculate_failure(spider) if 'duplicate' not in drop_reason: # 重复请求不计入
self.calculate_failure(spider)
''' '''
Sent when a spider callback generates an error (i.e. raises an exception). Sent when a spider callback generates an error (i.e. raises an exception).