modify scripts

iafd/src/fetch.py

@@ -244,6 +244,7 @@ def fetch_performers_detail_once(perfomers_list):
         url = performer['href']
         person = performer['name']
         curr_id = performer['id']
+        movies_cnt = performer['movies_cnt']
         logging.debug(f"Fetching data for performer ({person}), url {url} ...")
         soup, status_code = scraper.fetch_page(url, partial(scraper.generic_validator, tag="div", identifier="headshot", attr_type="id"))
         # 从本地读取的文件，忽略
@@ -253,6 +254,14 @@ def fetch_performers_detail_once(perfomers_list):
         if soup:
             data = scraper.parse_page_performer(soup, url)
             if data:
+                # 检查影片数量是否有更新
+                page_movies_cnt = int(data.get('movies_cnt', '0'))
+                if page_movies_cnt <= movies_cnt:
+                    if not force:
+                        logging.info(f"actor already update. skipping... person: ({person}), url: {url}")
+                        last_performer_id = curr_id
+                        continue
+
                 performer_id = db_tools.insert_or_update_performer({
                     'href': url,
                     'person': person,
@@ -377,6 +386,8 @@ def fetch_movies_detail():
         if debug:
             return True
 
+def reset_actor_movie_cnt():
+    db_tools.reset_actor_movies()
 
 # 建立缩写到函数的映射
 function_map = {
@@ -387,6 +398,7 @@ function_map = {
     "stu" : fetch_movies_by_stu,
     "performers": fetch_performers_detail,
     "movies" : fetch_movies_detail,
+    "reset_mv" : reset_actor_movie_cnt,
 }   
 
 # 主函数

iafd/src/sqlite_utils.py

@@ -329,7 +329,7 @@ def query_performer(identifier):
 # 按条件查询 href 列表 
 def query_performer_hrefs(**filters):
     try:
-        sql = "SELECT href, name, id FROM iafd_performers WHERE 1=1"
+        sql = "SELECT href, name, id, movies_cnt FROM iafd_performers WHERE 1=1"
         params = []
 
         if "id" in filters:
@@ -375,7 +375,7 @@ def query_performer_hrefs(**filters):
         logging.debug(f"query sql: {sql}")
         cursor.execute(sql, params)
         #return [row[0].lower() for row in cursor.fetchall()]   # 返回小写
-        return [{'href': row[0], 'name': row[1], 'id':row[2]} for row in cursor.fetchall()]
+        return [{'href': row[0], 'name': row[1], 'id':row[2], 'movies_cnt':row[3]} for row in cursor.fetchall()]
 
     except sqlite3.Error as e:
         logging.error(f"查询 href 失败: {e}")
@@ -905,6 +905,62 @@ def check_and_create_stat_table(taskid = 0):
         logging.warning(f"An error occurred: {e}")
 
 
+# 处理影片的 无码 字段
+def reset_actor_movies(check_and_do = 0):
+    try:
+        # 检查表中是否已存在movies_cnt列
+        cursor.execute(f"PRAGMA table_info(iafd_performers);")
+        columns = [row[1] for row in cursor.fetchall()]
+        
+        if 'movies_cnt' not in columns:
+            # 列不存在，添加新列
+            add_field_sql = f"""
+            ALTER TABLE iafd_performers ADD COLUMN movies_cnt INTEGER DEFAULT 0 NOT NULL;
+            """
+            cursor.execute(add_field_sql)
+            logging.info("成功添加movies_cnt字段")
+        else:
+            logging.info("movies_cnt字段已存在，跳过添加")
+
+        # 确保关联表有索引
+        cursor.execute(f"""
+            CREATE INDEX IF NOT EXISTS idx_iafd_performers_movies_performer_id
+            ON iafd_performers_movies(performer_id);
+        """)
+
+        # 创建临时表存储统计结果
+        cursor.execute(f"""
+            CREATE TEMPORARY TABLE temp_actor_counts AS
+            SELECT performer_id, COUNT(movie_id) AS cnt
+            FROM iafd_performers_movies
+            GROUP BY performer_id;
+        """)
+        
+        # 为临时表添加索引
+        cursor.execute("CREATE INDEX idx_temp_performer_id ON temp_actor_counts(performer_id);")
+        
+        # 更新主表
+        cursor.execute(f"""
+            UPDATE iafd_performers
+            SET movies_cnt = COALESCE((
+                SELECT cnt FROM temp_actor_counts 
+                WHERE performer_id = iafd_performers.id
+            ), 0);  -- 使用COALESCE处理没有影片的演员
+        """)
+
+        updated_rows = cursor.rowcount
+        logging.info(f"成功更新{updated_rows}个演员的影片数量")
+        
+        # 清理资源
+        cursor.execute("DROP TABLE IF EXISTS temp_actor_counts;")
+        conn.commit()
+
+        logging.info("任务执行完成！")
+
+    except sqlite3.Error as e:
+        conn.rollback()
+        logging.error("Error updating actor movie_cnt: %s", e)
+
 
 # 插入一条任务日志
 def insert_task_log():