From 0741ac94ad23a3d41e914e26c88d33d486d4d57a Mon Sep 17 00:00:00 2001 From: oscar Date: Tue, 4 Mar 2025 10:58:44 +0800 Subject: [PATCH] modify some scripts. --- scripts/iafd/result/detail.csv | 9 +- scripts/iafd/result/detail.json | 185 +------------------------------ scripts/iafd/src/fetch.py | 23 +++- scripts/iafd/src/sqlite_utils.py | 5 +- 4 files changed, 26 insertions(+), 196 deletions(-) mode change 100644 => 120000 scripts/iafd/result/detail.csv mode change 100644 => 120000 scripts/iafd/result/detail.json diff --git a/scripts/iafd/result/detail.csv b/scripts/iafd/result/detail.csv deleted file mode 100644 index 95392d7..0000000 --- a/scripts/iafd/result/detail.csv +++ /dev/null @@ -1,8 +0,0 @@ -person,href,performer_aka,birthday,astrology,birthplace,gender,years_active,ethnicity,nationality,hair_colors,eye_color,height,weight,measurements,tattoos,piercings,movies_cnt,vixen_cnt,blacked_cnt,tushy_cnt,x_art_cnt -Aaliyah,https://www.iafd.com/person.rme/id=50caabcd-c270-4d4a-b80f-2429201f6ff5,No known aliases,01/01/????,Capricorn,US,Trans woman,2006-2007,Black,American,Black/Dark Brown,Unknown,"5 feet, 5 inches (165 cm)",123 lbs (56 kg),No data,No data,No data,3,0,0,0,0 -Amanda Addams,https://www.iafd.com/person.rme/id=67bdb3b8-adf6-4ce5-9278-73a5777b459e,Amanda Adams|Amanda Stone|Camile|Camille|Emily Moore,01/01/????,Capricorn,"New Port Richey, FL",Woman,1990-1997,Caucasian,American,Brown/Red,Unknown,No data,No data,42DD-??-??,Flower with stem and leaves right breast,None,74,0,0,0,0 -Ashley,https://www.iafd.com/person.rme/id=44d0ecd8-c519-40b4-83aa-cdfe9670d238,No known aliases,01/01/????,Capricorn,No data,Trans woman,,Latin,Mexican,,Unknown,No data,No data,No data,No data,No data,1,0,0,0,0 -Daniela Canalis,https://www.iafd.com/person.rme/id=3b2728e0-6eac-4620-8e3f-5086d3cbaa24,Daniele|Danyelle Comini|Isabella Bittencour,01/01/????,Capricorn,"Belo Horizonte, Brazil",Trans woman,2003-2006,Caucasian,Brazilian,,Blue,"5 feet, 5 inches (165 cm)",No data,No data,No data,No data,11,0,0,0,0 -Dream Doll,https://www.iafd.com/person.rme/id=481fd985-9740-4c1b-b794-4620d6528cc1,No known aliases,01/01/????,Capricorn,USA,Trans woman,,Black,American,,Unknown,No data,No data,No data,"""Theresa"" right front shoulder; Stars right lower arm; ""Locked Loyalty"" inner left lower arm; Flower right hip",Right cheekbone,2,0,0,0,0 -Gianna,https://www.iafd.com/person.rme/id=8e22a8e0-e28e-4767-ad74-cae7a1ef6f15,No known aliases,01/01/????,Capricorn,No data,Trans woman,2006-2009,Caucasian,No data,,Unknown,No data,No data,No data,No data,No data,5,0,0,0,0 -Kirsten,https://www.iafd.com/person.rme/id=ca699282-1b57-4ce7-9bcc-d7799a292e34,Kirsten Claudia|Trish,01/01/????,Capricorn,No data,Trans woman,,Latin,No data,Black/Brown/Light Brown,Unknown,No data,No data,36DD-24-35,No data,No data,3,0,0,0,0 diff --git a/scripts/iafd/result/detail.csv b/scripts/iafd/result/detail.csv new file mode 120000 index 0000000..ab209b0 --- /dev/null +++ b/scripts/iafd/result/detail.csv @@ -0,0 +1 @@ +/root/hostdir/scripts_data/detail_birth_astro.csv \ No newline at end of file diff --git a/scripts/iafd/result/detail.json b/scripts/iafd/result/detail.json deleted file mode 100644 index 28aaeff..0000000 --- a/scripts/iafd/result/detail.json +++ /dev/null @@ -1,184 +0,0 @@ -[ - { - "href": "https://www.iafd.com/person.rme/id=50caabcd-c270-4d4a-b80f-2429201f6ff5", - "person": "Aaliyah", - "performer_aka": [ - "No known aliases" - ], - "birthday": "01/01/????", - "astrology": "Capricorn", - "birthplace": "US", - "gender": "Trans woman", - "years_active": "2006-2007", - "ethnicity": "Black", - "nationality": "American", - "hair_colors": "Black/Dark Brown", - "eye_color": "Unknown", - "height": "5 feet, 5 inches (165 cm)", - "weight": "123 lbs (56 kg)", - "measurements": "No data", - "tattoos": "No data", - "piercings": "No data", - "movies_cnt": 3, - "vixen_cnt": 0, - "blacked_cnt": 0, - "tushy_cnt": 0, - "x_art_cnt": 0 - }, - { - "href": "https://www.iafd.com/person.rme/id=67bdb3b8-adf6-4ce5-9278-73a5777b459e", - "person": "Amanda Addams", - "performer_aka": [ - "Amanda Adams", - "Amanda Stone", - "Camile", - "Camille", - "Emily Moore" - ], - "birthday": "01/01/????", - "astrology": "Capricorn", - "birthplace": "New Port Richey, FL", - "gender": "Woman", - "years_active": "1990-1997", - "ethnicity": "Caucasian", - "nationality": "American", - "hair_colors": "Brown/Red", - "eye_color": "Unknown", - "height": "No data", - "weight": "No data", - "measurements": "42DD-??-??", - "tattoos": "Flower with stem and leaves right breast", - "piercings": "None", - "movies_cnt": 74, - "vixen_cnt": 0, - "blacked_cnt": 0, - "tushy_cnt": 0, - "x_art_cnt": 0 - }, - { - "href": "https://www.iafd.com/person.rme/id=44d0ecd8-c519-40b4-83aa-cdfe9670d238", - "person": "Ashley", - "performer_aka": [ - "No known aliases" - ], - "birthday": "01/01/????", - "astrology": "Capricorn", - "birthplace": "No data", - "gender": "Trans woman", - "ethnicity": "Latin", - "nationality": "Mexican", - "eye_color": "Unknown", - "height": "No data", - "weight": "No data", - "measurements": "No data", - "tattoos": "No data", - "piercings": "No data", - "movies_cnt": 1, - "vixen_cnt": 0, - "blacked_cnt": 0, - "tushy_cnt": 0, - "x_art_cnt": 0 - }, - { - "href": "https://www.iafd.com/person.rme/id=3b2728e0-6eac-4620-8e3f-5086d3cbaa24", - "person": "Daniela Canalis", - "performer_aka": [ - "Daniele", - "Danyelle Comini", - "Isabella Bittencour" - ], - "birthday": "01/01/????", - "astrology": "Capricorn", - "birthplace": "Belo Horizonte, Brazil", - "gender": "Trans woman", - "years_active": "2003-2006", - "ethnicity": "Caucasian", - "nationality": "Brazilian", - "eye_color": "Blue", - "height": "5 feet, 5 inches (165 cm)", - "weight": "No data", - "measurements": "No data", - "tattoos": "No data", - "piercings": "No data", - "movies_cnt": 11, - "vixen_cnt": 0, - "blacked_cnt": 0, - "tushy_cnt": 0, - "x_art_cnt": 0 - }, - { - "href": "https://www.iafd.com/person.rme/id=481fd985-9740-4c1b-b794-4620d6528cc1", - "person": "Dream Doll", - "performer_aka": [ - "No known aliases" - ], - "birthday": "01/01/????", - "astrology": "Capricorn", - "birthplace": "USA", - "gender": "Trans woman", - "ethnicity": "Black", - "nationality": "American", - "eye_color": "Unknown", - "height": "No data", - "weight": "No data", - "measurements": "No data", - "tattoos": "\"Theresa\" right front shoulder; Stars right lower arm; \"Locked Loyalty\" inner left lower arm; Flower right hip", - "piercings": "Right cheekbone", - "movies_cnt": 2, - "vixen_cnt": 0, - "blacked_cnt": 0, - "tushy_cnt": 0, - "x_art_cnt": 0 - }, - { - "href": "https://www.iafd.com/person.rme/id=8e22a8e0-e28e-4767-ad74-cae7a1ef6f15", - "person": "Gianna", - "performer_aka": [ - "No known aliases" - ], - "birthday": "01/01/????", - "astrology": "Capricorn", - "birthplace": "No data", - "gender": "Trans woman", - "years_active": "2006-2009", - "ethnicity": "Caucasian", - "nationality": "No data", - "eye_color": "Unknown", - "height": "No data", - "weight": "No data", - "measurements": "No data", - "tattoos": "No data", - "piercings": "No data", - "movies_cnt": 5, - "vixen_cnt": 0, - "blacked_cnt": 0, - "tushy_cnt": 0, - "x_art_cnt": 0 - }, - { - "href": "https://www.iafd.com/person.rme/id=ca699282-1b57-4ce7-9bcc-d7799a292e34", - "person": "Kirsten", - "performer_aka": [ - "Kirsten Claudia", - "Trish" - ], - "birthday": "01/01/????", - "astrology": "Capricorn", - "birthplace": "No data", - "gender": "Trans woman", - "ethnicity": "Latin", - "nationality": "No data", - "hair_colors": "Black/Brown/Light Brown", - "eye_color": "Unknown", - "height": "No data", - "weight": "No data", - "measurements": "36DD-24-35", - "tattoos": "No data", - "piercings": "No data", - "movies_cnt": 3, - "vixen_cnt": 0, - "blacked_cnt": 0, - "tushy_cnt": 0, - "x_art_cnt": 0 - } -] \ No newline at end of file diff --git a/scripts/iafd/result/detail.json b/scripts/iafd/result/detail.json new file mode 120000 index 0000000..754f49e --- /dev/null +++ b/scripts/iafd/result/detail.json @@ -0,0 +1 @@ +/root/hostdir/scripts_data/detail_birth_astro.json \ No newline at end of file diff --git a/scripts/iafd/src/fetch.py b/scripts/iafd/src/fetch.py index 1798b16..d6196dc 100644 --- a/scripts/iafd/src/fetch.py +++ b/scripts/iafd/src/fetch.py @@ -13,6 +13,7 @@ import utils config.setup_logging() debug = False +force = False # 按星座获取演员列表，无翻页 def fetch_performers_by_astro(existed_performer_hrefs): @@ -352,12 +353,24 @@ def load_data(): if performers_data is None: print(f'read file error.') performers_data = [] + + # 读取数据库中的演员列表 + existed_performer_hrefs = db_tools.query_performer_hrefs() + if not existed_performer_hrefs: + logging.warning(f'get existed performers from db error.') + return None + for person in performers_data: + # 非强制更新下，跳过已有数据 + if not force and person['href'] in existed_performer_hrefs: + continue + performer_id = db_tools.insert_or_update_performer(person) if performer_id: logging.info(f'insert one person, id: {performer_id}, person: {person['person']}, url: {person['href']}') else: logging.warning(f'insert person: {person['person']}, {person['href']} failed.') + logging.info(f'all performers loaded in database.') # 导入已经在本地的 movies 数据 movies_file = '../result/movie_details.json' @@ -381,12 +394,17 @@ def load_data(): # 主函数 -def main(task, args_debug): +def main(task, args_debug, args_force): global debug debug = args_debug if debug: logging.info('Debug mode enabled.') + global force + force = args_force + if force: + logging.info('force update for all data.') + if task == 'fetch': check_update() elif task == 'load': @@ -400,6 +418,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='fetch iafd data.') parser.add_argument('--task', type=str, default='fetch', help='fetch from iafd.com or load from local data ... (fetch , load)') parser.add_argument('--debug', action='store_true', help='Enable debug mode (limit records)') + parser.add_argument('--force', action='store_true', help='force update (true for rewrite all)') args = parser.parse_args() - main(args.task, args.debug) + main(args.task, args.debug, args.force) diff --git a/scripts/iafd/src/sqlite_utils.py b/scripts/iafd/src/sqlite_utils.py index 5d052b8..fe54b78 100644 --- a/scripts/iafd/src/sqlite_utils.py +++ b/scripts/iafd/src/sqlite_utils.py @@ -61,9 +61,10 @@ def insert_or_update_performer(data): cursor.execute("DELETE FROM performer_aliases WHERE performer_id = ?", (performer_id,)) # 插入新的 alias - for alias in data.get("performer_aka", []): + #for alias in data.get("performer_aka", []): + for alias in data.get("performer_aka") or []: if alias.lower() != "no known aliases": - cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?)", (performer_id, alias)) + cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?) ON CONFLICT(performer_id, alias) DO NOTHING ", (performer_id, alias)) conn.commit() logging.debug(f"成功插入/更新演员: {data['person']}")