90 lines
3.8 KiB
Python
90 lines
3.8 KiB
Python
"""
|
||
Script Name:
|
||
Description: 从 https://www.iafd.com 上获取信息。利用cloudscraper绕过cloudflare
|
||
detail_fetch.py 从本地已经保存的列表数据,逐个拉取详情,并输出到文件。
|
||
list_fetch_astro.py 按照星座拉取数据,获得演员的信息列表。数据量适中,各详细字段较全
|
||
list_fetch_birth.py 按照生日拉取数据,获得演员的信息列表。数据量适中,各详细字段较全
|
||
list_fetch_ethnic.py 按照人种拉取数据,获得演员的信息列表。数据量大,但详细字段很多无效的
|
||
list_merge.py 上面三个列表的数据,取交集,得到整体数据。
|
||
iafd_scrape.py 借助 https://github.com/stashapp/CommunityScrapers 实现的脚本,可以输入演员的 iafd链接,获取兼容 stashapp 格式的数据。（作用不大,因为国籍、照片等字段不匹配）
|
||
|
||
html_format.py 负责读取已经保存的html目录, 提取信息,格式化输出。
|
||
data_merge.py 负责合并数据,它把从 iafd, javhd, thelordofporn 以及搭建 stashapp, 从上面更新到的演员数据(需导出)进行合并；
|
||
stashdb_merge.py 负责把从stashapp中导出的单个演员的json文件, 批量合并并输出; 通常我们需要把stashapp中导出的批量文件压缩并传输到data/tmp目录,解压后合并
|
||
从而获取到一份完整的数据列表。
|
||
|
||
Author: [Your Name]
|
||
Created Date: YYYY-MM-DD
|
||
Last Modified: YYYY-MM-DD
|
||
Version: 1.0
|
||
|
||
Modification History:
|
||
- YYYY-MM-DD [Your Name]:
|
||
- YYYY-MM-DD [Your Name]:
|
||
- YYYY-MM-DD [Your Name]:
|
||
"""
|
||
|
||
import os
|
||
import json
|
||
import csv
|
||
import logging
|
||
|
||
# 配置日志
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 输入和输出目录
|
||
input_dir = 'data/tmp' # 假设metadata目录在当前目录下
|
||
output_json_file = 'stashdb.json'
|
||
output_csv_file = 'stashdb.csv'
|
||
|
||
# 用于保存所有的条目
|
||
data_list = []
|
||
|
||
# 遍历metadata文件夹，读取所有json文件
|
||
for filename in os.listdir(input_dir):
|
||
if filename.endswith('.json'):
|
||
file_path = os.path.join(input_dir, filename)
|
||
|
||
try:
|
||
with open(file_path, 'r', encoding='utf-8') as file:
|
||
data = json.load(file)
|
||
|
||
# 提取需要的字段
|
||
person = {
|
||
'name': data.get('name'),
|
||
'gender': data.get('gender'),
|
||
'birthdate': data.get('birthdate'),
|
||
'ethnicity': data.get('ethnicity'),
|
||
'country': data.get('country'),
|
||
'height': data.get('height'),
|
||
'measurements': data.get('measurements'),
|
||
'fake_tits': data.get('fake_tits'),
|
||
'career_length': data.get('career_length'),
|
||
'aliases': ', '.join(data.get('aliases', [])) # 连接aliases数组元素
|
||
}
|
||
|
||
# 将数据添加到列表中
|
||
data_list.append(person)
|
||
logger.info(f"Processed file: {filename}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error processing file {filename}: {e}")
|
||
|
||
# 输出到 JSON 文件
|
||
try:
|
||
with open(output_json_file, 'w', encoding='utf-8') as json_file:
|
||
json.dump(data_list, json_file, ensure_ascii=False, indent=4)
|
||
logger.info(f"Data successfully written to {output_json_file}")
|
||
except Exception as e:
|
||
logger.error(f"Error writing JSON file: {e}")
|
||
|
||
# 输出到 CSV 文件
|
||
try:
|
||
with open(output_csv_file, 'w', newline='', encoding='utf-8') as csv_file:
|
||
writer = csv.DictWriter(csv_file, fieldnames=data_list[0].keys())
|
||
writer.writeheader()
|
||
writer.writerows(data_list)
|
||
logger.info(f"Data successfully written to {output_csv_file}")
|
||
except Exception as e:
|
||
logger.error(f"Error writing CSV file: {e}") |