research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-07-05 12:52:28 +08:00
parent 00d8ac08f4
commit 95b4d8b414
2 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy_proj/scrapy_proj/spiders/sis_spider.py
View File

@ -1,3 +1,4 @@
from datetime import datetime
import scrapy
from urllib.parse import urljoin
from scrapy_proj.spiders.base_spider import BaseSpider, extract_text_from_element
@ -110,7 +111,8 @@ class Sis001Spider(BaseSpider):
# 判断是否还要翻页，只有满足所有页面的数据，日期均小于开始日期时，停止翻页
up_date = parse_date_to_datetime(item['update_date'])
if up_date and self.begin and up_date < self.begin :
self.logger.debug(f"url: {response.url} update: {up_date}, begin: {self.begin}, now: {datetime.now()}")
if up_date and self.begin and (up_date < self.begin or up_date>datetime.now()) :
self.logger.debug(f"find early data.")
else:
need_next = True

6
scrapy_proj/scrapy_proj/spiders/u3c3_spider.py
View File

@ -1,3 +1,4 @@
from datetime import datetime
import scrapy
from scrapy_proj.spiders.base_spider import BaseSpider, extract_text_from_element
from scrapy_proj.items import U001Item
@ -11,7 +12,7 @@ class U001Spider(BaseSpider):
def __init__(self, debug='False', begin=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.debug = True if (str(debug).lower() == 'true' or str(debug).lower() == '1') else False
self.logger.info(f"debug mod: {self.debug}")
self.logger.info(f"debug mod: {self.debug}, begin: {begin}")
self.begin = parse_date_to_datetime(begin) if begin else None
@ -36,7 +37,8 @@ class U001Spider(BaseSpider):
# 判断是否还要翻页，只有满足所有页面的数据，日期均小于开始日期时，停止翻页
up_date = parse_date_to_datetime(item['update_date'])
if up_date and self.begin and up_date < self.begin :
self.logger.debug(f"url: {response.url} update: {up_date}, begin: {self.begin}, now: {datetime.now()}")
if up_date and self.begin and (up_date < self.begin or up_date>datetime.now()):
self.logger.debug(f"find early data.")
else:
need_next = True