modify scripts

scrapy_proj/scrapy_proj/db_wapper/spider_db_handler.py

@@ -358,7 +358,7 @@ class IAFDDBHandler(SQLiteDBHandler):
                 placeholders=', '.join(['?'] * len(names))
             )
 
-            logging.info(f'{insert_sql}')
+            logging.debug(f'{insert_sql}')
 
             self.cursor.execute(insert_sql, names)
             self.conn.commit()  # 提交临时表数据

scrapy_proj/scrapy_proj/spiders/clm_spider.py

@@ -1,41 +1,52 @@
 from datetime import datetime
 import scrapy
+import sys
 from urllib.parse import urljoin, quote_plus
-from scrapy_proj.utils.utils import parse_size, parse_date_to_datetime
+from scrapy_proj.utils.utils import parse_size, parse_date_to_datetime, load_json_file
 from scrapy_proj.spiders.base_spider import BaseSpider, extract_text_from_element
 from scrapy_proj.items import ClmIndexItem, ClmKeyWordsItem
 from scrapy_proj.comm.comm_def import SPIDER_NAME_CLM, ITEM_TYPE_CLM_INDEX, ITEM_TYPE_CLM_KEYWORDS
 from scrapy_proj.db_wapper.spider_db_handler import IAFDDBHandler, ClmDBHandler
 
 db_clm = ClmDBHandler()
+default_keywords_file = 'scrapy_proj/data/clm_keywords.json'
 
 class ClmSpider(BaseSpider):
     name = SPIDER_NAME_CLM
     allowed_domains = ["clmclm.com"]
     search_url = 'https://www.clmclm.com/search'
 
-    def __init__(self, debug='False', keywords=None, min_size=None, *args, **kwargs):
+    def __init__(self, debug='False', min_size=None, begin=None, mod='all', *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.debug = True if (str(debug).lower() == 'true' or str(debug).lower() == '1') else False
-        self.logger.info(f"debug mod: {self.debug}")
 
-        self.keywords = keywords
+        self.debug = True if (str(debug).lower() == 'true' or str(debug).lower() == '1') else False
-        self.min_size = float(min_size) if min_size else 1.0
+        self.update_mod = False
         self.run_task = True
         self.fix_title = False
 
-        # 增加一个暗号
+        # 增加一个更新模式，需要传入 mod == update 并且有 开始时间
-        if keywords and keywords.lower() == 'reload' :
+        self.begin = parse_date_to_datetime(begin) if begin else None
-            self.initDB()
+        self.min_size = float(min_size) if min_size else 1.0
-            self.init_load_actors_from_others()
+        self.keywords_file = kwargs.get('file_path') if kwargs.get('file_path') else default_keywords_file
-            self.run_task = False
+
-            self.logger.info(f"reload keywords db succ!")
+        self.logger.info(f"RUN CMD: {' '.join(sys.argv)}")
 
         # 增加一个暗号
-        if keywords and keywords.lower() == 'fix' :
+        if mod.lower() == 'update' and self.begin:
+            self.update_mod = True
+        elif mod.lower() == 'reload' :
+            self.reload_keywords()
+            self.run_task = False
+        elif mod.lower() == 'fix' :
             self.fix_title = True
             self.run_task = False
 
+    # 重新加载关键词
+    def reload_keywords(self):
+        self.load_keywords_from_file()
+        self.load_keywords_from_db()
+        self.logger.info(f"reload keywords db succ!")
+
     # 入口函数，由基类的方法触发
     def custom_start_requests(self):
         if self.fix_title:
@@ -54,10 +65,12 @@ class ClmSpider(BaseSpider):
             else:
                 self.logger.warning(f"no data.")
 
+        # 一些初始化等等的任务，不需要继续执行
         if not self.run_task:
             return 
 
-        tmp_query_str = f" groups='actress' and tags not like '%vixen%' "
+        #tmp_query_str = f" groups='actress' and tags not like '%vixen%' "
+        tmp_query_str = f" 1=1 "
         if self.debug:
             keywords = db_clm.get_key_words(limit =5, query_str = tmp_query_str)
         else:
@@ -97,9 +110,41 @@ class ClmSpider(BaseSpider):
 
         # 转换为字符串并处理编码
         result_url = location.decode('utf-8')
-        self.logger.info(f"重定向到结果页: {result_url}")
 
-        # 访问重定向后的结果页面，使用之前的解析方法
+        # 访问重定向后的结果页面，如果要求按时间排序，需要中转一下链接
+        if self.update_mod and self.begin :
+            self.logger.info(f"重定向到结果页: {result_url}, 继续寻找按时间排序的链接")
+            yield scrapy.Request(
+                url=result_url,
+                #headers=self._get_headers(),
+                callback=self.parse_page_by_date,
+                meta={'words_id': response.meta.get('words_id', 0), 'words': response.meta.get('words', '')}
+            )
+        else:
+            self.logger.info(f"重定向到结果页: {result_url}, 全量拉取")
+            yield scrapy.Request(
+                url=result_url,
+                #headers=self._get_headers(),
+                callback=self.parse_page_common,
+                meta={'words_id': response.meta.get('words_id', 0), 'words': response.meta.get('words', '')}
+            )
+
+    def parse_page_by_date(self, response):
+        # 解析"添加时间"对应的链接
+        # 使用CSS选择器定位包含"添加时间"文本的a标签
+        add_time_link = response.css(
+            'div.sortbar a:contains("添加时间")::attr(href)'
+        ).get()
+
+        result_url = response.url
+        if add_time_link:
+            # 拼接完整URL（response.url为当前页面URL，用于补全相对路径）
+            result_url = urljoin(response.url, add_time_link)
+            self.logger.info(f"获取到按时间排序的结果页，开始请求: {result_url}")
+            
+        else:
+            self.logger.warning(f"未找到添加时间对应的链接，使用原来的地址: {result_url}")
+
         yield scrapy.Request(
             url=result_url,
             #headers=self._get_headers(),
@@ -107,12 +152,13 @@ class ClmSpider(BaseSpider):
             meta={'words_id': response.meta.get('words_id', 0), 'words': response.meta.get('words', '')}
         )
 
-
     def parse_page_common(self, response):
-
         need_next = False
         # 提取所有 ssbox 节点（每个 ssbox 对应一条数据）
         ssboxes = response.xpath('//div[@class="ssbox"]')
+        if not ssboxes:
+            self.logger.warning(f"无法解析页面。 url: {response.url}")
+            return
         
         for ssbox in ssboxes:
             # 1. 提取 h3 中的链接和文本
@@ -143,15 +189,12 @@ class ClmSpider(BaseSpider):
             add_time = sbar.xpath('.//span[contains(text(), "添加时间:")]/b/text()').get() if sbar else None
             # 大小（如 "39.5 GB"）
             size = sbar.xpath('.//span[contains(text(), "大小:")]/b/text()').get() if sbar else None
+            size_gb = parse_size(size)
             # 热度（如 "435"）
             heat = sbar.xpath('.//span[contains(text(), "热度:")]/b/text()').get() if sbar else None
             # 最近下载时间（可选提取）
             last_download = sbar.xpath('.//span[contains(text(), "最近下载:")]/b/text()').get() if sbar else None
 
-            size_gb = parse_size(size)
-            if size_gb < self.min_size:
-                continue
-
             item = ClmIndexItem()
             item['item_type'] = ITEM_TYPE_CLM_INDEX
             item['category'] = category
@@ -167,38 +210,51 @@ class ClmSpider(BaseSpider):
             item['key_words'] = response.meta.get('words', '')
             item['is_update'] = False
 
+            # 判断是否还要翻页，只有满足所有页面的数据，日期均小于开始日期时，停止翻页
+            up_date = parse_date_to_datetime(item['add_date'])
+            #self.logger.debug(f"url: {response.url} update: {up_date}, begin: {self.begin}, now: {datetime.now()}")
+            if up_date and self.begin and (up_date < self.begin or up_date>datetime.now()):
+                pass
+            else:
+                need_next = True
+
+            # 太小的文件不要
+            if size_gb < self.min_size:
+                continue
+
             yield item
 
-        if self.debug:
-            return
-
         # 解析下一页链接
+        next_page_url = None
+        total_text = ''
         pager = response.xpath('//div[@class="pager"]')
         if pager:
             total_text = pager.xpath('.//span[contains(text(), "共")]/text()').get() 
-
             # 定位“下一页”的a标签（通过文本定位，避免混淆其他a标签）
             next_page_a = pager.xpath('.//a[contains(text(), "下一页")]').get()
-            # 提取href属性
             next_page_href = pager.xpath('.//a[contains(text(), "下一页")]/@href').get()
-
             # 判断是否还有下一页
             if next_page_href and next_page_href != '#':
                 # 拼接完整URL（相对路径转绝对路径）
-                next_page_url = response.urljoin(next_page_href)
+                next_page_url = response.urljoin(next_page_href)  
-                self.logger.debug(f'{total_text}, 发现下一页：{next_page_url}')
+
-                # 递归请求下一页
+        if self.debug:
-                yield scrapy.Request(
+            self.logger.info(f'debug模式下停止翻页. {total_text}. url: {response.url}')  
-                    url=next_page_url,
+            return
-                    callback=self.parse_page_common,
+        elif not need_next or not next_page_url: 
-                    meta={'words_id': response.meta.get('words_id', 0), 'words': response.meta.get('words', '')},
+            total_rows = db_clm.get_count_by_keywords_id(response.meta.get('words_id', 0))
-                    dont_filter=True  # 允许重复请求（防止因URL参数被过滤）
+            curr_words = response.meta.get('words', '')
-                )
+            self.logger.info(f'停止翻页. 更新模式: {self.update_mod}. {total_text}, 共 {total_rows} 条记录。 key words: ({curr_words}), url: {response.url}')  
-            else:
+            return
-                # 当href为#或不存在时，说明已无下一页
+
-                total_rows = db_clm.get_count_by_keywords_id(response.meta.get('words_id', 0))
+        self.logger.debug(f'{total_text}, 发现下一页：{next_page_url}')
-                curr_words = response.meta.get('words', '')
+        # 递归请求下一页
-                self.logger.info(f'已获取完所有页面，停止翻页. {total_text}, 共 {total_rows} 条记录。 key words: ({curr_words}), url: {response.url}')    
+        yield scrapy.Request(
+            url=next_page_url,
+            callback=self.parse_page_common,
+            meta={'words_id': response.meta.get('words_id', 0), 'words': response.meta.get('words', '')},
+            dont_filter=True  # 允许重复请求（防止因URL参数被过滤）
+        )
 
     def parse_page_detail(self, response):
         # 匹配 class 为 'bt_title' 的 div 下的 h2 标签文本
@@ -214,48 +270,31 @@ class ClmSpider(BaseSpider):
 
 
     # 指定的关键词，导入到数据库
-    def initDB(self):
+    def load_keywords_from_file(self):
-        default_keywords = [
+        self.logger.info(f"load keywords from file: {self.keywords_file}")
-            {'vixen group' : ['vixen', 'tushy', 'tushyraw', 'blacked', 'blackedraw', 'deeper']},
+        json_data, err = load_json_file(self.keywords_file)
-            {'VIP 4K' : ['Cuck4K', 'Daddy4k', 'Loan4k', 'Dyke4K', 'Rim4k', 'Pie4k', 'Ignore4K', 'Daddy4k', 'Stuck4k', 'Tutor4k']},
+        if not json_data:
-            {'Teen Mega World' : ['anal-angels', 'Anal-Beauty', 'Beauty4k', 'creampie-angels', 'Beauty-Angels', 'FirstBGG', 'FuckStudies', 'OhMyHoles', 'X-Angels']},
+            self.logger.warning(f"load file error. {err}")
-            {'Fuck You Cash' : ['BBCPie', 'Tiny4k', 'Cum4K', 'Anal4K', 'Exotic4K', 'Facials4k', 'Holed', 'Lubed', 'Mom4K', 'passion hd']},
+            return 
-            {'Naughty America (Network)' : ['Naughty Office', 'Naughty Americans', 'Naughty America', 'Naughty Weddings']},
+
-            {'Nubiles Porn (Network)' : ['MyFamilyPies', 'StepSiblingsCaught',  'nubilesporn']},
+        total_lines = 0
-            {'Brazzers' : ['Real Wife Stories', 'brazzers']},
+        for group, items in json_data.items():
-            {'TeamSkeet (Network)' : ['teenpies', 'shoplyfter']},
+            total_lines += len(items)
-            {'BangBros' : ['BangBus', 'BangBros']},
+            self.logger.info(f"load group ({group}), {len(items)} items.")
-            {'Nubile Films' : ['nfbusty', 'NubileFilms']},
+            for item in items:
-            {'DDF Network' : ['DDFBusty']},
+                words_item = ClmKeyWordsItem()
-            {'Adult Time (Network)' : ['AdultTime', 'BurningAngel']},
+                words_item['item_type'] = ITEM_TYPE_CLM_KEYWORDS
-            {'Anal Vids' : ['AnalVids']},
+                words_item['words'] = item
-            {'LegalPorno' : ['LegalPorno']},
+                words_item['groups'] = group
-            {'Pornbox' : ['Pornworld']},
+                words_item['tags'] = ''
-            {'Wow (Network)' : ['WowGirls']},
+                words_item['index_count'] = 0
-            {'Malibu Media' : ['x-art']},
+                db_clm.insert_item(words_item)
-            {'VIPissy Cash' : ['VIPissy']},
+                self.logger.debug(f"insert item: {item}: {group}")
-            {'japan Blu-Ray' : ['Japan AV Blu-Ray']},
+        
-            {'siterip' : ['siterip']},
+        self.logger.info(f"load {self.keywords_file} succ, total lines: {total_lines}")
-            {'Brazil' : ['NewMFX']},
-            {'Wicked' : ['Wicked']},
-            {'Sticky Dollars' : ['Swallowed']},
-            {'ManyVids' : ['ManyVids']},
-            {'PervCity' : ['AnalOverdose']}
-        ]
-        for row in default_keywords:
-            for group, items in row.items():
-                for item in items:
-                    words_item = ClmKeyWordsItem()
-                    words_item['item_type'] = ITEM_TYPE_CLM_KEYWORDS
-                    words_item['words'] = item
-                    words_item['groups'] = group
-                    words_item['tags'] = ''
-                    words_item['index_count'] = 0
-                    db_clm.insert_item(words_item)
-                    self.logger.debug(f"insert item: {item}: {group}")
 
     # 从其他数据源获取到演员列表，导入到数据库
-    def init_load_actors_from_others(self):
+    def load_keywords_from_db(self):
         db_comm = IAFDDBHandler()
         all_likes = {
             'vixen' : ['vixen.com', 'Vixen Video'],
@@ -292,7 +331,7 @@ class ClmSpider(BaseSpider):
         # 查询另一个数据表，获取结果
         load_results = db_comm.get_lord_actors()
         if load_results:
-            self.logger.info(f"total actors in lord: {len(load_results)}")
+            self.logger.info(f"total actors in thelordofporn: {len(load_results)}")
             for row in load_results:
                 actor_name = row['name']
                 if actor_name not in actor_tags:
@@ -302,7 +341,7 @@ class ClmSpider(BaseSpider):
         self.logger.info(f"after merge, total actors: {len(actor_tags)}")
         for actor, tags_set in actor_tags.items():
             tag_str = ','.join(tags_set)  # set直接支持迭代，无需额外转换
-            self.logger.info(f"actor: {actor}, tags: {tag_str}")
+            self.logger.debug(f"actor: {actor}, tags: {tag_str}")
             words_item = ClmKeyWordsItem()
             words_item['item_type'] = ITEM_TYPE_CLM_KEYWORDS
             words_item['words'] = actor
@@ -310,4 +349,4 @@ class ClmSpider(BaseSpider):
             words_item['tags'] = tag_str
             words_item['index_count'] = 0
             db_clm.insert_item(words_item)
-            #self.logger.debug(f"insert item: {words_item}")
+            self.logger.debug(f"insert item: {words_item}")

scrapy_proj/scrapy_proj/utils/utils.py

@@ -1,6 +1,35 @@
 import re
+import json
+import os
 from datetime import datetime, timezone
 
+def load_json_file(file_path):
+    # 检查文件是否存在
+    if not os.path.exists(file_path):
+        return None, f"{file_path} not exists."
+    
+    # 检查是否是文件（避免目录路径）
+    if not os.path.isfile(file_path):
+        return None, f"{file_path} is not file."
+    
+    try:
+        # 读取文件内容
+        with open(file_path, 'r', encoding='utf-8') as f:
+            try:
+                # 解析JSON数据
+                json_data = json.load(f)
+                return json_data, None
+            except json.JSONDecodeError as e:
+                return None, f"JSON格式解析失败 - {e}"
+            except Exception as e:
+                return None, f"读取文件内容时发生异常 - {e}"
+    except PermissionError:
+        return None, f"错误: 没有权限读取文件 - {file_path}"
+    except Exception as e:
+        return None, f"错误: 打开文件时发生异常 - {e}"
+
+    return None, "未知错误"
+
 '''timestamp(ms) 转为日期'''
 def format_timestamp(ts, is_ms=True):
     if not ts: