"""
Script Name: 
Description: 从 https://www.iafd.com 上获取信息。利用cloudscraper绕过cloudflare
    detail_fetch.py 从本地已经保存的列表数据,逐个拉取详情,并输出到文件。
    list_fetch_astro.py 按照星座拉取数据,获得演员的信息列表。数据量适中,各详细字段较全
    list_fetch_birth.py 按照生日拉取数据,获得演员的信息列表。数据量适中,各详细字段较全
    list_fetch_ethnic.py 按照人种拉取数据,获得演员的信息列表。数据量大,但详细字段很多无效的
    list_merge.py 上面三个列表的数据,取交集,得到整体数据。
    iafd_scrape.py 借助 https://github.com/stashapp/CommunityScrapers 实现的脚本,可以输入演员的 iafd链接,获取兼容 stashapp 格式的数据。（作用不大,因为国籍、照片等字段不匹配）

    html_format.py 负责读取已经保存的html目录, 提取信息,格式化输出。
    data_merge.py 负责合并数据,它把从 iafd, javhd, thelordofporn 以及搭建 stashapp, 从上面更新到的演员数据(需导出)进行合并；
    stashdb_merge.py 负责把从stashapp中导出的单个演员的json文件, 批量合并并输出; 通常我们需要把stashapp中导出的批量文件压缩并传输到data/tmp目录,解压后合并
    从而获取到一份完整的数据列表。

Author: [Your Name]
Created Date: YYYY-MM-DD
Last Modified: YYYY-MM-DD
Version: 1.0

Modification History:
    - YYYY-MM-DD [Your Name]: 
    - YYYY-MM-DD [Your Name]: 
    - YYYY-MM-DD [Your Name]: 
"""

import cloudscraper
import json
import time
import csv
import argparse
from bs4 import BeautifulSoup
import logging
import config

config.setup_logging()

# 定义基础 URL 和可变参数
host_url = "https://www.iafd.com"
# 结果路径
res_dir = f"{config.global_share_data_dir}/iafd"

fetch_config = {
    'dist': {
        'base_url': f"{host_url}/distrib.rme/distrib=",
        'list_page_url': f"{host_url}/distrib.asp",
        'html_table_id': 'distable',
        'html_select_name': 'Distrib',
        'output_key_id': 'distributors',
        'json_file': f'{res_dir}/distributors.json',
        'csv_file': f'{res_dir}/distributors.csv',
    },
    'stu': {
        'base_url': f"{host_url}/studio.rme/studio=",
        'list_page_url': f"{host_url}/studio.asp",
        'html_table_id': 'studio',
        'html_select_name': 'Studio',
        'output_key_id': 'studios',
        'json_file': f'{res_dir}/studios.json',
        'csv_file': f'{res_dir}/studios.csv',
    }
}

distr_map = {
    6812 : 'nubilefilms.com',
    8563 : 'teenmegaworld network',
    6779 : 'x-art.com',
    7133 : 'tushy.com',
    6496 : 'blacked.com',
    7758 : 'vixen.com',
    6791 : 'teamskeet.com',
    12454: 'vip4k.com',
    13541: 'wow network',
    9702 : 'cum4k.com',
    6778 : 'tiny4k.com',
    12667: 'anal4k.com',
    7419 : 'exotic4k.com',
    13594: 'facials4k.com',
    13633: 'mom4k.com',
    12335: 'slim4k.com',
    16709: 'strippers4k.com',

}
studio_map = {
    6812 : 'nubilefilms.com',
    9811 : 'Teen Mega World',
    6779 : 'x-art.com',
    7133 : 'tushy.com',
    6496 : 'blacked.com',
    7758 : 'vixen.com',
    6791 : 'teamskeet.com',
    8052: 'wowgirls.com',
    9702 : 'cum4k.com',
    6778 : 'tiny4k.com',
    12667: 'anal4k.com',
    7419 : 'exotic4k.com',
    13594: 'facials4k.com',
    13633: 'mom4k.com',
    12335: 'slim4k.com',
    16709: 'strippers4k.com',

}

# 设置 headers 和 scraper
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
scraper = cloudscraper.create_scraper()

all_data = []

# 网络请求并解析 HTML
def fetch_page(url):
    try:
        response = scraper.get(url, headers=headers)
        response.raise_for_status()
        return response.text
    except Exception as e:
        logging.error(f"Failed to fetch {url}: {e}")
        return None

# 解析 HTML 内容，提取需要的数据
def parse_page(html, name, config):
    table_id = config['html_table_id']
    key_id = config['output_key_id']

    soup = BeautifulSoup(html, "html.parser")
    table = soup.find("table", id=table_id)

    if not table:
        logging.warning(f"Warning: No {table_id} table found in {name}")
        return None
    
    # 找到thead并跳过
    thead = table.find('thead')
    if thead:
        thead.decompose()  # 去掉thead部分，不需要解析

    # 现在只剩下tbody部分
    tbody = table.find('tbody')
    rows = tbody.find_all('tr') if tbody else []

    global all_data
    for row in rows:
        cols = row.find_all('td')
        if len(cols) >= 5:
            title = cols[0].text.strip()
            label = cols[1].text.strip()
            year = cols[2].text.strip()
            rev = cols[3].text.strip()
            a_href = cols[0].find('a')
            href = host_url + a_href['href'] if a_href else ''

            all_data.append({
                key_id: name,
                'title': title,
                'label': label,
                'year': year,
                'rev': rev,
                'href': href
            })
    return soup

# 处理翻页,星座的无需翻页
def handle_pagination(soup, astro):
    return None

# 获取列表页
def process_list_gage(config):
    list_page_url=config['list_page_url']
    select_name = config['html_select_name']
    list_map = {}

    logging.info(f"Fetching data for {list_page_url} ...")
    select_element = None
    while True:
        html = fetch_page(list_page_url)
        if html:
            soup = BeautifulSoup(html, "html.parser")
            select_element = soup.find('select', {'name': select_name})
            if select_element :
                break
            else:
                logging.info(f"wrong html content. retring {list_page_url} ...")
        else:
            logging.info(f"wrong html content. retring {list_page_url} ...")

    if not select_element:
        return None
    
    options = select_element.find_all('option')
    for option in options:
        value = option.get('value')  # 获取 value 属性
        text = option.text.strip()   # 获取文本内容
        list_map[int(value)] = text
    logging.info(f'fetch {list_page_url} succ. total lines: {len(list_map)}')
    return list_map

# 主逻辑函数：循环处理每个种族
def process_main_data(list_data, config):
    base_url = config['base_url']

    for key, name in list_data.items():
        url = base_url + str(key)
        next_url = url
        logging.info(f"Fetching data for {name}, url {url} ...")

        while next_url:
            html = fetch_page(next_url)
            if html:
                soup = parse_page(html, name, config)
                if soup:
                    next_url = handle_pagination(soup, name)
                else:
                    logging.info(f"wrong html content. retring {next_url} ...")
                # 定期保存结果
                save_data(config)
                time.sleep(2)  # 控制访问频率
            else:
                logging.info(f"Retrying {next_url} ...")
                time.sleep(5)  # 等待后再重试

# 保存到文件
def save_data(config):    
    with open(config['json_file'], 'w', encoding='utf-8') as json_file:
        json.dump(all_data, json_file, indent=4, ensure_ascii=False)
    
    with open(config['csv_file'], 'w', newline='', encoding='utf-8') as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames=[config['output_key_id'], 'title', 'label', 'year', 'rev', 'href'])
        writer.writeheader()
        writer.writerows(all_data)


# 执行主逻辑
if __name__ == '__main__':
    # 命令行参数处理
    parser = argparse.ArgumentParser(description='fetch movie list from iafd.com')
    parser.add_argument('--type', type=str, default='dist', help='fetch by ... (dist , stu)')
    parser.add_argument('--kind', type=str, default='parts', help='fetch all or parts (parts , all)')
    args = parser.parse_args()
    
    config = fetch_config[args.type]
    if not config:
        logging.warning(f'unkwon type: {args.type} {args.kind}')        
    else:
        list_data = {}
        if args.kind == 'all':
            list_data = process_list_gage(config)
        elif args.type == 'dist':
            list_data = distr_map
        else:
            list_data = studio_map

        process_main_data(list_data, config)
        logging.info("Data fetching and saving completed.")
