modify scripts

scrapy_proj/scrapy_proj/extensions/failure_monitor.py

@@ -47,7 +47,8 @@ class FailureMonitorExtension:
         # 从request.meta中获取丢弃理由
         drop_reason = reason if reason else request.meta.get('_dropreason', '未知原因')
         spider.logger.warning(f"request_dropped on url: {request.url} | 原因: {drop_reason}")
-        self.calculate_failure(spider)
+        if 'duplicate' not in drop_reason:  # 重复请求不计入
+            self.calculate_failure(spider)
     
     '''
         Sent when a spider callback generates an error (i.e. raises an exception).