modify some scripts.
This commit is contained in:
@ -1,8 +0,0 @@
|
||||
person,href,performer_aka,birthday,astrology,birthplace,gender,years_active,ethnicity,nationality,hair_colors,eye_color,height,weight,measurements,tattoos,piercings,movies_cnt,vixen_cnt,blacked_cnt,tushy_cnt,x_art_cnt
|
||||
Aaliyah,https://www.iafd.com/person.rme/id=50caabcd-c270-4d4a-b80f-2429201f6ff5,No known aliases,01/01/????,Capricorn,US,Trans woman,2006-2007,Black,American,Black/Dark Brown,Unknown,"5 feet, 5 inches (165 cm)",123 lbs (56 kg),No data,No data,No data,3,0,0,0,0
|
||||
Amanda Addams,https://www.iafd.com/person.rme/id=67bdb3b8-adf6-4ce5-9278-73a5777b459e,Amanda Adams|Amanda Stone|Camile|Camille|Emily Moore,01/01/????,Capricorn,"New Port Richey, FL",Woman,1990-1997,Caucasian,American,Brown/Red,Unknown,No data,No data,42DD-??-??,Flower with stem and leaves right breast,None,74,0,0,0,0
|
||||
Ashley,https://www.iafd.com/person.rme/id=44d0ecd8-c519-40b4-83aa-cdfe9670d238,No known aliases,01/01/????,Capricorn,No data,Trans woman,,Latin,Mexican,,Unknown,No data,No data,No data,No data,No data,1,0,0,0,0
|
||||
Daniela Canalis,https://www.iafd.com/person.rme/id=3b2728e0-6eac-4620-8e3f-5086d3cbaa24,Daniele|Danyelle Comini|Isabella Bittencour,01/01/????,Capricorn,"Belo Horizonte, Brazil",Trans woman,2003-2006,Caucasian,Brazilian,,Blue,"5 feet, 5 inches (165 cm)",No data,No data,No data,No data,11,0,0,0,0
|
||||
Dream Doll,https://www.iafd.com/person.rme/id=481fd985-9740-4c1b-b794-4620d6528cc1,No known aliases,01/01/????,Capricorn,USA,Trans woman,,Black,American,,Unknown,No data,No data,No data,"""Theresa"" right front shoulder; Stars right lower arm; ""Locked Loyalty"" inner left lower arm; Flower right hip",Right cheekbone,2,0,0,0,0
|
||||
Gianna,https://www.iafd.com/person.rme/id=8e22a8e0-e28e-4767-ad74-cae7a1ef6f15,No known aliases,01/01/????,Capricorn,No data,Trans woman,2006-2009,Caucasian,No data,,Unknown,No data,No data,No data,No data,No data,5,0,0,0,0
|
||||
Kirsten,https://www.iafd.com/person.rme/id=ca699282-1b57-4ce7-9bcc-d7799a292e34,Kirsten Claudia|Trish,01/01/????,Capricorn,No data,Trans woman,,Latin,No data,Black/Brown/Light Brown,Unknown,No data,No data,36DD-24-35,No data,No data,3,0,0,0,0
|
||||
|
1
scripts/iafd/result/detail.csv
Symbolic link
1
scripts/iafd/result/detail.csv
Symbolic link
@ -0,0 +1 @@
|
||||
/root/hostdir/scripts_data/detail_birth_astro.csv
|
||||
|
@ -1,184 +0,0 @@
|
||||
[
|
||||
{
|
||||
"href": "https://www.iafd.com/person.rme/id=50caabcd-c270-4d4a-b80f-2429201f6ff5",
|
||||
"person": "Aaliyah",
|
||||
"performer_aka": [
|
||||
"No known aliases"
|
||||
],
|
||||
"birthday": "01/01/????",
|
||||
"astrology": "Capricorn",
|
||||
"birthplace": "US",
|
||||
"gender": "Trans woman",
|
||||
"years_active": "2006-2007",
|
||||
"ethnicity": "Black",
|
||||
"nationality": "American",
|
||||
"hair_colors": "Black/Dark Brown",
|
||||
"eye_color": "Unknown",
|
||||
"height": "5 feet, 5 inches (165 cm)",
|
||||
"weight": "123 lbs (56 kg)",
|
||||
"measurements": "No data",
|
||||
"tattoos": "No data",
|
||||
"piercings": "No data",
|
||||
"movies_cnt": 3,
|
||||
"vixen_cnt": 0,
|
||||
"blacked_cnt": 0,
|
||||
"tushy_cnt": 0,
|
||||
"x_art_cnt": 0
|
||||
},
|
||||
{
|
||||
"href": "https://www.iafd.com/person.rme/id=67bdb3b8-adf6-4ce5-9278-73a5777b459e",
|
||||
"person": "Amanda Addams",
|
||||
"performer_aka": [
|
||||
"Amanda Adams",
|
||||
"Amanda Stone",
|
||||
"Camile",
|
||||
"Camille",
|
||||
"Emily Moore"
|
||||
],
|
||||
"birthday": "01/01/????",
|
||||
"astrology": "Capricorn",
|
||||
"birthplace": "New Port Richey, FL",
|
||||
"gender": "Woman",
|
||||
"years_active": "1990-1997",
|
||||
"ethnicity": "Caucasian",
|
||||
"nationality": "American",
|
||||
"hair_colors": "Brown/Red",
|
||||
"eye_color": "Unknown",
|
||||
"height": "No data",
|
||||
"weight": "No data",
|
||||
"measurements": "42DD-??-??",
|
||||
"tattoos": "Flower with stem and leaves right breast",
|
||||
"piercings": "None",
|
||||
"movies_cnt": 74,
|
||||
"vixen_cnt": 0,
|
||||
"blacked_cnt": 0,
|
||||
"tushy_cnt": 0,
|
||||
"x_art_cnt": 0
|
||||
},
|
||||
{
|
||||
"href": "https://www.iafd.com/person.rme/id=44d0ecd8-c519-40b4-83aa-cdfe9670d238",
|
||||
"person": "Ashley",
|
||||
"performer_aka": [
|
||||
"No known aliases"
|
||||
],
|
||||
"birthday": "01/01/????",
|
||||
"astrology": "Capricorn",
|
||||
"birthplace": "No data",
|
||||
"gender": "Trans woman",
|
||||
"ethnicity": "Latin",
|
||||
"nationality": "Mexican",
|
||||
"eye_color": "Unknown",
|
||||
"height": "No data",
|
||||
"weight": "No data",
|
||||
"measurements": "No data",
|
||||
"tattoos": "No data",
|
||||
"piercings": "No data",
|
||||
"movies_cnt": 1,
|
||||
"vixen_cnt": 0,
|
||||
"blacked_cnt": 0,
|
||||
"tushy_cnt": 0,
|
||||
"x_art_cnt": 0
|
||||
},
|
||||
{
|
||||
"href": "https://www.iafd.com/person.rme/id=3b2728e0-6eac-4620-8e3f-5086d3cbaa24",
|
||||
"person": "Daniela Canalis",
|
||||
"performer_aka": [
|
||||
"Daniele",
|
||||
"Danyelle Comini",
|
||||
"Isabella Bittencour"
|
||||
],
|
||||
"birthday": "01/01/????",
|
||||
"astrology": "Capricorn",
|
||||
"birthplace": "Belo Horizonte, Brazil",
|
||||
"gender": "Trans woman",
|
||||
"years_active": "2003-2006",
|
||||
"ethnicity": "Caucasian",
|
||||
"nationality": "Brazilian",
|
||||
"eye_color": "Blue",
|
||||
"height": "5 feet, 5 inches (165 cm)",
|
||||
"weight": "No data",
|
||||
"measurements": "No data",
|
||||
"tattoos": "No data",
|
||||
"piercings": "No data",
|
||||
"movies_cnt": 11,
|
||||
"vixen_cnt": 0,
|
||||
"blacked_cnt": 0,
|
||||
"tushy_cnt": 0,
|
||||
"x_art_cnt": 0
|
||||
},
|
||||
{
|
||||
"href": "https://www.iafd.com/person.rme/id=481fd985-9740-4c1b-b794-4620d6528cc1",
|
||||
"person": "Dream Doll",
|
||||
"performer_aka": [
|
||||
"No known aliases"
|
||||
],
|
||||
"birthday": "01/01/????",
|
||||
"astrology": "Capricorn",
|
||||
"birthplace": "USA",
|
||||
"gender": "Trans woman",
|
||||
"ethnicity": "Black",
|
||||
"nationality": "American",
|
||||
"eye_color": "Unknown",
|
||||
"height": "No data",
|
||||
"weight": "No data",
|
||||
"measurements": "No data",
|
||||
"tattoos": "\"Theresa\" right front shoulder; Stars right lower arm; \"Locked Loyalty\" inner left lower arm; Flower right hip",
|
||||
"piercings": "Right cheekbone",
|
||||
"movies_cnt": 2,
|
||||
"vixen_cnt": 0,
|
||||
"blacked_cnt": 0,
|
||||
"tushy_cnt": 0,
|
||||
"x_art_cnt": 0
|
||||
},
|
||||
{
|
||||
"href": "https://www.iafd.com/person.rme/id=8e22a8e0-e28e-4767-ad74-cae7a1ef6f15",
|
||||
"person": "Gianna",
|
||||
"performer_aka": [
|
||||
"No known aliases"
|
||||
],
|
||||
"birthday": "01/01/????",
|
||||
"astrology": "Capricorn",
|
||||
"birthplace": "No data",
|
||||
"gender": "Trans woman",
|
||||
"years_active": "2006-2009",
|
||||
"ethnicity": "Caucasian",
|
||||
"nationality": "No data",
|
||||
"eye_color": "Unknown",
|
||||
"height": "No data",
|
||||
"weight": "No data",
|
||||
"measurements": "No data",
|
||||
"tattoos": "No data",
|
||||
"piercings": "No data",
|
||||
"movies_cnt": 5,
|
||||
"vixen_cnt": 0,
|
||||
"blacked_cnt": 0,
|
||||
"tushy_cnt": 0,
|
||||
"x_art_cnt": 0
|
||||
},
|
||||
{
|
||||
"href": "https://www.iafd.com/person.rme/id=ca699282-1b57-4ce7-9bcc-d7799a292e34",
|
||||
"person": "Kirsten",
|
||||
"performer_aka": [
|
||||
"Kirsten Claudia",
|
||||
"Trish"
|
||||
],
|
||||
"birthday": "01/01/????",
|
||||
"astrology": "Capricorn",
|
||||
"birthplace": "No data",
|
||||
"gender": "Trans woman",
|
||||
"ethnicity": "Latin",
|
||||
"nationality": "No data",
|
||||
"hair_colors": "Black/Brown/Light Brown",
|
||||
"eye_color": "Unknown",
|
||||
"height": "No data",
|
||||
"weight": "No data",
|
||||
"measurements": "36DD-24-35",
|
||||
"tattoos": "No data",
|
||||
"piercings": "No data",
|
||||
"movies_cnt": 3,
|
||||
"vixen_cnt": 0,
|
||||
"blacked_cnt": 0,
|
||||
"tushy_cnt": 0,
|
||||
"x_art_cnt": 0
|
||||
}
|
||||
]
|
||||
1
scripts/iafd/result/detail.json
Symbolic link
1
scripts/iafd/result/detail.json
Symbolic link
@ -0,0 +1 @@
|
||||
/root/hostdir/scripts_data/detail_birth_astro.json
|
||||
@ -13,6 +13,7 @@ import utils
|
||||
config.setup_logging()
|
||||
|
||||
debug = False
|
||||
force = False
|
||||
|
||||
# 按星座获取演员列表，无翻页
|
||||
def fetch_performers_by_astro(existed_performer_hrefs):
|
||||
@ -352,12 +353,24 @@ def load_data():
|
||||
if performers_data is None:
|
||||
print(f'read file error.')
|
||||
performers_data = []
|
||||
|
||||
# 读取数据库中的演员列表
|
||||
existed_performer_hrefs = db_tools.query_performer_hrefs()
|
||||
if not existed_performer_hrefs:
|
||||
logging.warning(f'get existed performers from db error.')
|
||||
return None
|
||||
|
||||
for person in performers_data:
|
||||
# 非强制更新下，跳过已有数据
|
||||
if not force and person['href'] in existed_performer_hrefs:
|
||||
continue
|
||||
|
||||
performer_id = db_tools.insert_or_update_performer(person)
|
||||
if performer_id:
|
||||
logging.info(f'insert one person, id: {performer_id}, person: {person['person']}, url: {person['href']}')
|
||||
else:
|
||||
logging.warning(f'insert person: {person['person']}, {person['href']} failed.')
|
||||
logging.info(f'all performers loaded in database.')
|
||||
|
||||
# 导入已经在本地的 movies 数据
|
||||
movies_file = '../result/movie_details.json'
|
||||
@ -381,12 +394,17 @@ def load_data():
|
||||
|
||||
|
||||
# 主函数
|
||||
def main(task, args_debug):
|
||||
def main(task, args_debug, args_force):
|
||||
global debug
|
||||
debug = args_debug
|
||||
if debug:
|
||||
logging.info('Debug mode enabled.')
|
||||
|
||||
global force
|
||||
force = args_force
|
||||
if force:
|
||||
logging.info('force update for all data.')
|
||||
|
||||
if task == 'fetch':
|
||||
check_update()
|
||||
elif task == 'load':
|
||||
@ -400,6 +418,7 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='fetch iafd data.')
|
||||
parser.add_argument('--task', type=str, default='fetch', help='fetch from iafd.com or load from local data ... (fetch , load)')
|
||||
parser.add_argument('--debug', action='store_true', help='Enable debug mode (limit records)')
|
||||
parser.add_argument('--force', action='store_true', help='force update (true for rewrite all)')
|
||||
args = parser.parse_args()
|
||||
|
||||
main(args.task, args.debug)
|
||||
main(args.task, args.debug, args.force)
|
||||
|
||||
@ -61,9 +61,10 @@ def insert_or_update_performer(data):
|
||||
cursor.execute("DELETE FROM performer_aliases WHERE performer_id = ?", (performer_id,))
|
||||
|
||||
# 插入新的 alias
|
||||
for alias in data.get("performer_aka", []):
|
||||
#for alias in data.get("performer_aka", []):
|
||||
for alias in data.get("performer_aka") or []:
|
||||
if alias.lower() != "no known aliases":
|
||||
cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?)", (performer_id, alias))
|
||||
cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?) ON CONFLICT(performer_id, alias) DO NOTHING ", (performer_id, alias))
|
||||
|
||||
conn.commit()
|
||||
logging.debug(f"成功插入/更新演员: {data['person']}")
|
||||
|
||||
Reference in New Issue
Block a user