"""
Script Name: 
Description: 从 javhd.com 上获取女优列表，并逐个获取女优详细信息。
    list_fetch.py 从网站上获取列表, 并以json的形式把结果输出到本地文件, 支持ja,zh,en三种语言可选(一般情况下可以三种全部拉取一遍); 
    list_format.py 则把这些文件读取出来，合并，形成完整的列表, 主要是把三种语言的女优名字拼到一起, 使用处理后的链接地址+图片地址作为判断同一个人的依据;
    model_fetch.py 则把上一步获取到的列表，读取详情页面，合并进来一些详细信息。
    注意: Header部分是从浏览器中抓取的, 时间久了可能要替换。

Author: [Your Name]
Created Date: YYYY-MM-DD
Last Modified: YYYY-MM-DD
Version: 1.0

Modification History:
    - YYYY-MM-DD [Your Name]: 
    - YYYY-MM-DD [Your Name]: 
    - YYYY-MM-DD [Your Name]: 
"""

import os
import re
import json
import glob
import logging
import csv
from collections import defaultdict

# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# 结果目录
RESULT_DIR = "result"
RESULT_TMP_DIR = f'{RESULT_DIR}/tmp'
OUTPUT_JSON = os.path.join(RESULT_DIR, "models.json")
OUTPUT_CSV = os.path.join(RESULT_DIR, "models.csv")

# 可能需要重命名的文件
LANGS = ["ja", "en", "zh"]
for lang in LANGS:
    old_file = os.path.join(RESULT_TMP_DIR, f"{lang}_model.json")
    new_file = os.path.join(RESULT_TMP_DIR, f"{lang}_model_popular_1.json")
    if os.path.exists(old_file):
        logging.info(f"Renaming {old_file} to {new_file}")
        os.rename(old_file, new_file)

# 读取所有匹配的 JSON 文件
file_paths = sorted(glob.glob(os.path.join(RESULT_TMP_DIR, "*_model_popular_*.json")))
pattern = re.compile(r'(\w+)_model_popular_(\d+)\.json')

def normalize_url(url):
    """去掉URL中的 en/ja/zh 子目录"""
    return re.sub(r'/(en|ja|zh)/', '/', url)

# 主处理程序
def main_process():
    models = {}

    for file_path in file_paths:
        match = pattern.search(os.path.basename(file_path))
        if not match:
            continue
        
        lang, num = match.groups()
        num = int(num)
        
        logging.info(f"Processing {file_path} (lang={lang}, num={num})")
        
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                data = json.load(f)
        except Exception as e:
            logging.error(f"Failed to load {file_path}: {e}")
            continue
        
        template = data.get("template", "")
        thumb_components = re.findall(r'<thumb-component[^>]*>', template)
        
        for idx, thumb in enumerate(thumb_components, start=1):
            rank = (num - 1) * 36 + idx
            
            link_content = re.search(r'link-content="(.*?)"', thumb)
            url_thumb = re.search(r'url-thumb="(.*?)"', thumb)
            title = re.search(r'title="(.*?)"', thumb)
            
            if not url_thumb or not title:
                logging.info(f"no countent for rank:{rank} title:{title} url:{url_thumb}  {thumb}")
                continue
            
            pic = url_thumb.group(1)
            name = title.group(1)
            url = link_content.group(1) if link_content and lang == "en" else ""
            norm_url = normalize_url(link_content.group(1))
            
            key = (pic, norm_url)
            if key not in models:
                models[key] = {"rank": rank, "ja_name": "", "zh_name": "", "en_name": "", "url": url, "pic": pic}
            
            models[key][f"{lang}_name"] = name
            if lang == "en" and url:
                models[key]["url"] = url

    # 按 rank 排序后输出 JSON
    sorted_models = sorted(models.values(), key=lambda x: x["rank"])

    with open(OUTPUT_JSON, "w", encoding="utf-8") as f:
        json.dump(sorted_models, f, indent=4, ensure_ascii=False)
    logging.info(f"Saved JSON output to {OUTPUT_JSON}")

    # 输出 CSV 格式
    headers = ["rank", "ja_name", "zh_name", "en_name", "url", "pic"]
    with open(OUTPUT_CSV, "w", encoding="utf-8") as csvfile:    
        writer = csv.DictWriter(csvfile, fieldnames=headers)
        writer.writeheader()
        writer.writerows(sorted_models)
    logging.info(f"Saved TXT output to {OUTPUT_CSV}")


if __name__ == '__main__':
    main_process()