modify some scripts.

scripts/iafd/result/detail.csv

@@ -1,8 +0,0 @@
-person,href,performer_aka,birthday,astrology,birthplace,gender,years_active,ethnicity,nationality,hair_colors,eye_color,height,weight,measurements,tattoos,piercings,movies_cnt,vixen_cnt,blacked_cnt,tushy_cnt,x_art_cnt
-Aaliyah,https://www.iafd.com/person.rme/id=50caabcd-c270-4d4a-b80f-2429201f6ff5,No known aliases,01/01/????,Capricorn,US,Trans woman,2006-2007,Black,American,Black/Dark Brown,Unknown,"5 feet, 5 inches (165 cm)",123 lbs (56 kg),No data,No data,No data,3,0,0,0,0
-Amanda Addams,https://www.iafd.com/person.rme/id=67bdb3b8-adf6-4ce5-9278-73a5777b459e,Amanda Adams|Amanda Stone|Camile|Camille|Emily Moore,01/01/????,Capricorn,"New Port Richey, FL",Woman,1990-1997,Caucasian,American,Brown/Red,Unknown,No data,No data,42DD-??-??,Flower with stem and leaves right breast,None,74,0,0,0,0
-Ashley,https://www.iafd.com/person.rme/id=44d0ecd8-c519-40b4-83aa-cdfe9670d238,No known aliases,01/01/????,Capricorn,No data,Trans woman,,Latin,Mexican,,Unknown,No data,No data,No data,No data,No data,1,0,0,0,0
-Daniela Canalis,https://www.iafd.com/person.rme/id=3b2728e0-6eac-4620-8e3f-5086d3cbaa24,Daniele|Danyelle Comini|Isabella Bittencour,01/01/????,Capricorn,"Belo Horizonte, Brazil",Trans woman,2003-2006,Caucasian,Brazilian,,Blue,"5 feet, 5 inches (165 cm)",No data,No data,No data,No data,11,0,0,0,0
-Dream Doll,https://www.iafd.com/person.rme/id=481fd985-9740-4c1b-b794-4620d6528cc1,No known aliases,01/01/????,Capricorn,USA,Trans woman,,Black,American,,Unknown,No data,No data,No data,"""Theresa"" right front shoulder; Stars right lower arm; ""Locked Loyalty"" inner left lower arm; Flower right hip",Right cheekbone,2,0,0,0,0
-Gianna,https://www.iafd.com/person.rme/id=8e22a8e0-e28e-4767-ad74-cae7a1ef6f15,No known aliases,01/01/????,Capricorn,No data,Trans woman,2006-2009,Caucasian,No data,,Unknown,No data,No data,No data,No data,No data,5,0,0,0,0
-Kirsten,https://www.iafd.com/person.rme/id=ca699282-1b57-4ce7-9bcc-d7799a292e34,Kirsten Claudia|Trish,01/01/????,Capricorn,No data,Trans woman,,Latin,No data,Black/Brown/Light Brown,Unknown,No data,No data,36DD-24-35,No data,No data,3,0,0,0,0

scripts/iafd/result/detail.csv

@@ -0,0 +1 @@
+/root/hostdir/scripts_data/detail_birth_astro.csv

scripts/iafd/result/detail.json

@@ -1,184 +0,0 @@
-[
-    {
-        "href": "https://www.iafd.com/person.rme/id=50caabcd-c270-4d4a-b80f-2429201f6ff5",
-        "person": "Aaliyah",
-        "performer_aka": [
-            "No known aliases"
-        ],
-        "birthday": "01/01/????",
-        "astrology": "Capricorn",
-        "birthplace": "US",
-        "gender": "Trans woman",
-        "years_active": "2006-2007",
-        "ethnicity": "Black",
-        "nationality": "American",
-        "hair_colors": "Black/Dark Brown",
-        "eye_color": "Unknown",
-        "height": "5 feet, 5 inches (165 cm)",
-        "weight": "123 lbs (56 kg)",
-        "measurements": "No data",
-        "tattoos": "No data",
-        "piercings": "No data",
-        "movies_cnt": 3,
-        "vixen_cnt": 0,
-        "blacked_cnt": 0,
-        "tushy_cnt": 0,
-        "x_art_cnt": 0
-    },
-    {
-        "href": "https://www.iafd.com/person.rme/id=67bdb3b8-adf6-4ce5-9278-73a5777b459e",
-        "person": "Amanda Addams",
-        "performer_aka": [
-            "Amanda Adams",
-            "Amanda Stone",
-            "Camile",
-            "Camille",
-            "Emily Moore"
-        ],
-        "birthday": "01/01/????",
-        "astrology": "Capricorn",
-        "birthplace": "New Port Richey, FL",
-        "gender": "Woman",
-        "years_active": "1990-1997",
-        "ethnicity": "Caucasian",
-        "nationality": "American",
-        "hair_colors": "Brown/Red",
-        "eye_color": "Unknown",
-        "height": "No data",
-        "weight": "No data",
-        "measurements": "42DD-??-??",
-        "tattoos": "Flower with stem and leaves right breast",
-        "piercings": "None",
-        "movies_cnt": 74,
-        "vixen_cnt": 0,
-        "blacked_cnt": 0,
-        "tushy_cnt": 0,
-        "x_art_cnt": 0
-    },
-    {
-        "href": "https://www.iafd.com/person.rme/id=44d0ecd8-c519-40b4-83aa-cdfe9670d238",
-        "person": "Ashley",
-        "performer_aka": [
-            "No known aliases"
-        ],
-        "birthday": "01/01/????",
-        "astrology": "Capricorn",
-        "birthplace": "No data",
-        "gender": "Trans woman",
-        "ethnicity": "Latin",
-        "nationality": "Mexican",
-        "eye_color": "Unknown",
-        "height": "No data",
-        "weight": "No data",
-        "measurements": "No data",
-        "tattoos": "No data",
-        "piercings": "No data",
-        "movies_cnt": 1,
-        "vixen_cnt": 0,
-        "blacked_cnt": 0,
-        "tushy_cnt": 0,
-        "x_art_cnt": 0
-    },
-    {
-        "href": "https://www.iafd.com/person.rme/id=3b2728e0-6eac-4620-8e3f-5086d3cbaa24",
-        "person": "Daniela Canalis",
-        "performer_aka": [
-            "Daniele",
-            "Danyelle Comini",
-            "Isabella Bittencour"
-        ],
-        "birthday": "01/01/????",
-        "astrology": "Capricorn",
-        "birthplace": "Belo Horizonte, Brazil",
-        "gender": "Trans woman",
-        "years_active": "2003-2006",
-        "ethnicity": "Caucasian",
-        "nationality": "Brazilian",
-        "eye_color": "Blue",
-        "height": "5 feet, 5 inches (165 cm)",
-        "weight": "No data",
-        "measurements": "No data",
-        "tattoos": "No data",
-        "piercings": "No data",
-        "movies_cnt": 11,
-        "vixen_cnt": 0,
-        "blacked_cnt": 0,
-        "tushy_cnt": 0,
-        "x_art_cnt": 0
-    },
-    {
-        "href": "https://www.iafd.com/person.rme/id=481fd985-9740-4c1b-b794-4620d6528cc1",
-        "person": "Dream Doll",
-        "performer_aka": [
-            "No known aliases"
-        ],
-        "birthday": "01/01/????",
-        "astrology": "Capricorn",
-        "birthplace": "USA",
-        "gender": "Trans woman",
-        "ethnicity": "Black",
-        "nationality": "American",
-        "eye_color": "Unknown",
-        "height": "No data",
-        "weight": "No data",
-        "measurements": "No data",
-        "tattoos": "\"Theresa\" right front shoulder; Stars right lower arm; \"Locked Loyalty\" inner left lower arm; Flower right hip",
-        "piercings": "Right cheekbone",
-        "movies_cnt": 2,
-        "vixen_cnt": 0,
-        "blacked_cnt": 0,
-        "tushy_cnt": 0,
-        "x_art_cnt": 0
-    },
-    {
-        "href": "https://www.iafd.com/person.rme/id=8e22a8e0-e28e-4767-ad74-cae7a1ef6f15",
-        "person": "Gianna",
-        "performer_aka": [
-            "No known aliases"
-        ],
-        "birthday": "01/01/????",
-        "astrology": "Capricorn",
-        "birthplace": "No data",
-        "gender": "Trans woman",
-        "years_active": "2006-2009",
-        "ethnicity": "Caucasian",
-        "nationality": "No data",
-        "eye_color": "Unknown",
-        "height": "No data",
-        "weight": "No data",
-        "measurements": "No data",
-        "tattoos": "No data",
-        "piercings": "No data",
-        "movies_cnt": 5,
-        "vixen_cnt": 0,
-        "blacked_cnt": 0,
-        "tushy_cnt": 0,
-        "x_art_cnt": 0
-    },
-    {
-        "href": "https://www.iafd.com/person.rme/id=ca699282-1b57-4ce7-9bcc-d7799a292e34",
-        "person": "Kirsten",
-        "performer_aka": [
-            "Kirsten Claudia",
-            "Trish"
-        ],
-        "birthday": "01/01/????",
-        "astrology": "Capricorn",
-        "birthplace": "No data",
-        "gender": "Trans woman",
-        "ethnicity": "Latin",
-        "nationality": "No data",
-        "hair_colors": "Black/Brown/Light Brown",
-        "eye_color": "Unknown",
-        "height": "No data",
-        "weight": "No data",
-        "measurements": "36DD-24-35",
-        "tattoos": "No data",
-        "piercings": "No data",
-        "movies_cnt": 3,
-        "vixen_cnt": 0,
-        "blacked_cnt": 0,
-        "tushy_cnt": 0,
-        "x_art_cnt": 0
-    }
-]

scripts/iafd/result/detail.json

@@ -0,0 +1 @@
+/root/hostdir/scripts_data/detail_birth_astro.json

scripts/iafd/src/fetch.py

@@ -13,6 +13,7 @@ import utils
 config.setup_logging()
 
 debug = False
+force = False
 
 # 按星座获取演员列表，无翻页
 def fetch_performers_by_astro(existed_performer_hrefs):    
@@ -352,12 +353,24 @@ def load_data():
     if performers_data is None:
         print(f'read file error.')
         performers_data = []
+
+    # 读取数据库中的演员列表
+    existed_performer_hrefs = db_tools.query_performer_hrefs()
+    if not existed_performer_hrefs:
+        logging.warning(f'get existed performers from db error.')
+        return None
+    
     for person in performers_data:
+        # 非强制更新下，跳过已有数据
+        if not force and person['href'] in existed_performer_hrefs:
+            continue
+
         performer_id = db_tools.insert_or_update_performer(person)
         if performer_id:
             logging.info(f'insert one person, id: {performer_id}, person: {person['person']}, url: {person['href']}')
         else:
             logging.warning(f'insert person: {person['person']}, {person['href']} failed.')
+    logging.info(f'all performers loaded in database.')
 
     # 导入已经在本地的 movies 数据
     movies_file = '../result/movie_details.json'
@@ -381,12 +394,17 @@ def load_data():
     
 
 # 主函数
-def main(task, args_debug):
+def main(task, args_debug, args_force):
     global debug
     debug = args_debug
     if debug:
         logging.info('Debug mode enabled.')
 
+    global force
+    force = args_force
+    if force:
+        logging.info('force update for all data.')
+
     if task == 'fetch':
         check_update()
     elif task == 'load':
@@ -400,6 +418,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='fetch iafd data.')
     parser.add_argument('--task', type=str, default='fetch', help='fetch from iafd.com or load from local data ... (fetch , load)')
     parser.add_argument('--debug', action='store_true', help='Enable debug mode (limit records)')
+    parser.add_argument('--force', action='store_true', help='force update (true for rewrite all)')
     args = parser.parse_args()
     
-    main(args.task, args.debug)
+    main(args.task, args.debug, args.force)

scripts/iafd/src/sqlite_utils.py

@@ -61,9 +61,10 @@ def insert_or_update_performer(data):
         cursor.execute("DELETE FROM performer_aliases WHERE performer_id = ?", (performer_id,))
 
         # 插入新的 alias
-        for alias in data.get("performer_aka", []):
+        #for alias in data.get("performer_aka", []):
+        for alias in data.get("performer_aka") or []:
             if alias.lower() != "no known aliases":
-                cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?)", (performer_id, alias))
+                cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?) ON CONFLICT(performer_id, alias) DO NOTHING ", (performer_id, alias))
 
         conn.commit()
         logging.debug(f"成功插入/更新演员: {data['person']}")