modify some scripts.

scripts/iafd/src/fetch.py

@@ -13,6 +13,7 @@ import utils
 config.setup_logging()
 
 debug = False
+force = False
 
 # 按星座获取演员列表，无翻页
 def fetch_performers_by_astro(existed_performer_hrefs):    
@@ -352,12 +353,24 @@ def load_data():
     if performers_data is None:
         print(f'read file error.')
         performers_data = []
+
+    # 读取数据库中的演员列表
+    existed_performer_hrefs = db_tools.query_performer_hrefs()
+    if not existed_performer_hrefs:
+        logging.warning(f'get existed performers from db error.')
+        return None
+    
     for person in performers_data:
+        # 非强制更新下，跳过已有数据
+        if not force and person['href'] in existed_performer_hrefs:
+            continue
+
         performer_id = db_tools.insert_or_update_performer(person)
         if performer_id:
             logging.info(f'insert one person, id: {performer_id}, person: {person['person']}, url: {person['href']}')
         else:
             logging.warning(f'insert person: {person['person']}, {person['href']} failed.')
+    logging.info(f'all performers loaded in database.')
 
     # 导入已经在本地的 movies 数据
     movies_file = '../result/movie_details.json'
@@ -381,12 +394,17 @@ def load_data():
     
 
 # 主函数
-def main(task, args_debug):
+def main(task, args_debug, args_force):
     global debug
     debug = args_debug
     if debug:
         logging.info('Debug mode enabled.')
 
+    global force
+    force = args_force
+    if force:
+        logging.info('force update for all data.')
+
     if task == 'fetch':
         check_update()
     elif task == 'load':
@@ -400,6 +418,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='fetch iafd data.')
     parser.add_argument('--task', type=str, default='fetch', help='fetch from iafd.com or load from local data ... (fetch , load)')
     parser.add_argument('--debug', action='store_true', help='Enable debug mode (limit records)')
+    parser.add_argument('--force', action='store_true', help='force update (true for rewrite all)')
     args = parser.parse_args()
     
-    main(args.task, args.debug)
+    main(args.task, args.debug, args.force)

scripts/iafd/src/sqlite_utils.py

@@ -61,9 +61,10 @@ def insert_or_update_performer(data):
         cursor.execute("DELETE FROM performer_aliases WHERE performer_id = ?", (performer_id,))
 
         # 插入新的 alias
-        for alias in data.get("performer_aka", []):
+        #for alias in data.get("performer_aka", []):
+        for alias in data.get("performer_aka") or []:
             if alias.lower() != "no known aliases":
-                cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?)", (performer_id, alias))
+                cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?) ON CONFLICT(performer_id, alias) DO NOTHING ", (performer_id, alias))
 
         conn.commit()
         logging.debug(f"成功插入/更新演员: {data['person']}")