research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
sophon
2025-07-20 11:19:55 +08:00
parent c81d4c49c7
commit 8bf2db4229
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy_proj/scrapy_proj/spiders/clm_spider.py
View File

@ -186,7 +186,7 @@ class ClmSpider(BaseSpider):
if next_page_href and next_page_href != '#': if next_page_href and next_page_href != '#':
# 拼接完整URL（相对路径转绝对路径） # 拼接完整URL（相对路径转绝对路径）
next_page_url = response.urljoin(next_page_href) next_page_url = response.urljoin(next_page_href)
self.logger.info(f'{total_text}, 发现下一页：{next_page_url}') self.logger.debug(f'{total_text}, 发现下一页：{next_page_url}')
# 递归请求下一页 # 递归请求下一页
yield scrapy.Request( yield scrapy.Request(
url=next_page_url, url=next_page_url,