1057 lines
41 KiB
Python
1057 lines
41 KiB
Python
import sqlite3
|
||
import json
|
||
import config
|
||
import logging
|
||
from datetime import datetime
|
||
|
||
# 连接 SQLite 数据库
|
||
DB_PATH = f"{config.global_share_data_dir}/sqlite/shared.db" # 替换为你的数据库文件
|
||
conn = sqlite3.connect(DB_PATH, check_same_thread=False)
|
||
cursor = conn.cursor()
|
||
|
||
cached_tags = {}
|
||
|
||
# """从指定表中通过 href 查找 id"""
|
||
def get_id_by_href(table: str, href: str) -> int:
|
||
if href is None:
|
||
return None
|
||
cursor.execute(f"SELECT id FROM {table} WHERE href = ?", (href,))
|
||
row = cursor.fetchone()
|
||
return row[0] if row else None
|
||
|
||
|
||
def insert_actor_index(name, href, from_actor_list=None, from_movie_list=None):
|
||
try:
|
||
# **查询是否已存在该演员**
|
||
cursor.execute("SELECT id, name, from_actor_list, from_movie_list FROM javdb_actors WHERE href = ?", (href,))
|
||
existing_actor = cursor.fetchone()
|
||
|
||
if existing_actor: # **如果演员已存在**
|
||
actor_id, existing_name, existing_actor_list, existing_movie_list = existing_actor
|
||
|
||
# **如果没有传入值，则保持原有值**
|
||
from_actor_list = from_actor_list if from_actor_list is not None else existing_actor_list
|
||
from_movie_list = from_movie_list if from_movie_list is not None else existing_movie_list
|
||
|
||
cursor.execute("""
|
||
UPDATE javdb_actors
|
||
SET name = ?,
|
||
from_actor_list = ?,
|
||
from_movie_list = ?,
|
||
updated_at = datetime('now', 'localtime')
|
||
WHERE href = ?
|
||
""", (name, from_actor_list, from_movie_list, href))
|
||
else: # **如果演员不存在，插入**
|
||
cursor.execute("""
|
||
INSERT INTO javdb_actors (href, name, from_actor_list, from_movie_list)
|
||
VALUES (?, ?, COALESCE(?, 0), COALESCE(?, 0))
|
||
""", (href, name, from_actor_list, from_movie_list))
|
||
|
||
conn.commit()
|
||
|
||
performer_id = get_id_by_href('javdb_actors', href)
|
||
if performer_id:
|
||
logging.debug(f"Inserted/Updated actor index, id: {performer_id}, name: {name}, href: {href}")
|
||
|
||
return performer_id
|
||
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error(f"数据库错误: {e}")
|
||
return None
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logging.error(f"未知错误: {e}")
|
||
return None
|
||
'''
|
||
def insert_movie_index_old(title, href, from_actor_list=None, from_movie_makers=None, from_movie_series=None, maker_id=None, series_id=None, from_movie_publishers=None, pub_id=None):
|
||
try:
|
||
# **先检查数据库中是否已有该电影**
|
||
cursor.execute("SELECT id, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, from_movie_publishers, pub_id FROM javdb_movies WHERE href = ?", (href,))
|
||
existing_movie = cursor.fetchone()
|
||
|
||
if existing_movie: # **如果电影已存在**
|
||
movie_id, existing_actor, existing_maker, existing_series, existing_maker_id, existing_series_id, existing_pub, existing_pub_id = existing_movie
|
||
|
||
# **如果没有传入值，就用原来的值**
|
||
from_actor_list = from_actor_list if from_actor_list is not None else existing_actor
|
||
from_movie_makers = from_movie_makers if from_movie_makers is not None else existing_maker
|
||
from_movie_series = from_movie_series if from_movie_series is not None else existing_series
|
||
maker_id = maker_id if maker_id is not None else existing_maker_id
|
||
series_id = series_id if series_id is not None else existing_series_id
|
||
from_movie_publishers = from_movie_publishers if from_movie_publishers is not None else existing_pub
|
||
pub_id = pub_id if pub_id is not None else existing_pub_id
|
||
|
||
cursor.execute("""
|
||
UPDATE javdb_movies
|
||
SET title = ?,
|
||
from_actor_list = ?,
|
||
from_movie_makers = ?,
|
||
from_movie_series = ?,
|
||
maker_id = ?,
|
||
series_id = ?,
|
||
from_movie_publishers = ?,
|
||
pub_id = ?,
|
||
updated_at = datetime('now', 'localtime')
|
||
WHERE href = ?
|
||
""", (title, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, from_movie_publishers, pub_id, href))
|
||
else: # **如果电影不存在，插入**
|
||
cursor.execute("""
|
||
INSERT INTO javdb_movies (title, href, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, from_movie_publishers, pub_id)
|
||
VALUES (?, ?, COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0))
|
||
""", (title, href, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, from_movie_publishers, pub_id))
|
||
|
||
conn.commit()
|
||
|
||
movie_id = get_id_by_href('javdb_movies', href)
|
||
if movie_id:
|
||
logging.debug(f'Inserted/Updated movie index, id: {movie_id}, title: {title}, href: {href}')
|
||
|
||
return movie_id
|
||
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logging.error(f"Error inserting/updating movie: {e}")
|
||
return None
|
||
'''
|
||
def insert_movie_index(title, href, **kwargs):
|
||
try:
|
||
# 先检查数据库中是否已有该电影
|
||
cursor.execute("SELECT * FROM javdb_movies WHERE href = ?", (href,))
|
||
existing_movie = cursor.fetchone()
|
||
|
||
# 获取列名
|
||
column_names = [description[0] for description in cursor.description]
|
||
|
||
fields = [
|
||
'from_actor_list', 'from_movie_makers', 'from_movie_series', 'from_movie_publishers',
|
||
'maker_id', 'series_id', 'pub_id', 'uncensored'
|
||
]
|
||
|
||
if existing_movie: # 如果电影已存在
|
||
existing_values = dict(zip(column_names, existing_movie))
|
||
movie_id = existing_values['id']
|
||
logging.debug(f"values in db: {existing_values}")
|
||
|
||
# 如果没有传入值，就用原来的值
|
||
for field in fields:
|
||
kwargs[field] = kwargs.get(field) if kwargs.get(field) is not None else existing_values[field]
|
||
|
||
set_clauses = ", ".join([f"{field} = ?" for field in fields])
|
||
sql = f"""
|
||
UPDATE javdb_movies
|
||
SET title = ?, {set_clauses}, updated_at = datetime('now', 'localtime')
|
||
WHERE href = ?
|
||
"""
|
||
values = [title] + [kwargs[field] for field in fields] + [href]
|
||
logging.debug(f"sql: {sql}, values: {values}")
|
||
cursor.execute(sql, values)
|
||
else: # 如果电影不存在，插入
|
||
columns = ', '.join(['title', 'href'] + fields)
|
||
placeholders = ', '.join(['?'] * (len(fields) + 2))
|
||
sql = f"INSERT INTO javdb_movies ({columns}) VALUES ({placeholders})"
|
||
values = [title, href] + [kwargs.get(field, 0) for field in fields]
|
||
logging.debug(f"sql: {sql}, values: {values}")
|
||
cursor.execute(sql, values)
|
||
|
||
conn.commit()
|
||
|
||
movie_id = get_id_by_href('javdb_movies', href)
|
||
if movie_id:
|
||
logging.debug(f'Inserted/Updated movie index, id: {movie_id}, title: {title}, href: {href}')
|
||
|
||
return movie_id
|
||
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logging.error(f"Error inserting/updating movie: {e}")
|
||
return None
|
||
|
||
|
||
# 插入演员和电影的关联数据
|
||
def insert_actor_movie(performer_id, movie_id, tags=''):
|
||
try:
|
||
cursor.execute("""
|
||
INSERT INTO javdb_actors_movies (actor_id, movie_id, tags, updated_at)
|
||
VALUES (?, ?, ?, datetime('now', 'localtime'))
|
||
ON CONFLICT(actor_id, movie_id) DO UPDATE SET tags=excluded.tags, updated_at=datetime('now', 'localtime')
|
||
""",
|
||
(performer_id, movie_id, tags)
|
||
)
|
||
conn.commit()
|
||
|
||
#logging.debug(f'insert one performer_movie, performer_id: {performer_id}, movie_id: {movie_id}')
|
||
|
||
return performer_id
|
||
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logging.error("Error inserting movie: %s", e)
|
||
return None
|
||
|
||
# 插入演员数据
|
||
def insert_or_update_actor(actor):
|
||
try:
|
||
cursor.execute('''
|
||
INSERT INTO javdb_actors (name, href, pic, is_full_data, updated_at)
|
||
VALUES (?, ?, ?, 1, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET name=excluded.name, pic=excluded.pic, is_full_data=1, updated_at=datetime('now', 'localtime')
|
||
''', (actor['name'], actor['href'], actor['pic']))
|
||
|
||
conn.commit()
|
||
|
||
# 查询刚插入的数据
|
||
cursor.execute('SELECT id, from_actor_list FROM javdb_actors WHERE href = ?', (actor['href'],))
|
||
actor_id, uncensored = cursor.fetchone()
|
||
if actor_id is None:
|
||
logging.warning(f'insert data error. name: {actor['name']}, href: {actor['href']}')
|
||
return None
|
||
|
||
logging.debug(f'insert one actor, id: {actor_id}, name: {actor['name']}, href: {actor['href']}')
|
||
|
||
# 插入别名
|
||
for alias in actor.get("alias") or []:
|
||
cursor.execute('''
|
||
INSERT OR IGNORE INTO javdb_actors_alias (actor_id, alias, updated_at)
|
||
VALUES (?, ?, datetime('now', 'localtime'))
|
||
''', (actor_id, alias))
|
||
|
||
conn.commit()
|
||
|
||
# 插入影片列表
|
||
for movie in actor.get("credits") or []:
|
||
# from_actor_list = 1 表示无码影星的，其他不处理
|
||
if uncensored and uncensored > 0:
|
||
movie_id = insert_movie_index(movie['title'], movie['href'], from_actor_list=1, uncensored=uncensored)
|
||
else:
|
||
movie_id = insert_movie_index(movie['title'], movie['href'], from_actor_list=1)
|
||
if movie_id:
|
||
tmp_id = insert_actor_movie(actor_id, movie_id)
|
||
if tmp_id :
|
||
logging.debug(f'insert one performer_movie, performer_id: {actor_id}, movie_id: {movie_id}')
|
||
else:
|
||
logging.warning(f'insert performer_movie failed. performer_id: {actor_id}, moive href: {movie['href']}')
|
||
|
||
return actor_id
|
||
except Exception as e:
|
||
logging.error(f"插入/更新演员 {actor['name']} 失败: {e}")
|
||
conn.rollback()
|
||
|
||
# """插入或更新电影数据(异常url的处理，比如404链接)"""
|
||
def insert_or_update_actor_404(name, href, is_full_data=1):
|
||
try:
|
||
# 插入或更新电影信息
|
||
cursor.execute(
|
||
"""
|
||
INSERT INTO javdb_actors (name, href, is_full_data, updated_at)
|
||
VALUES (?, ?, ?, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
name=excluded.name, is_full_data=excluded.is_full_data, updated_at = datetime('now', 'localtime')
|
||
""",
|
||
(name, href, is_full_data)
|
||
)
|
||
conn.commit()
|
||
|
||
# 获取插入的 movie_id
|
||
actor_id = get_id_by_href('javdb_actors', href)
|
||
if actor_id is None:
|
||
return None
|
||
|
||
return actor_id
|
||
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logging.error("Error inserting movie: %s", e)
|
||
return None
|
||
|
||
|
||
# 删除演员
|
||
def delete_actor_by_href(href):
|
||
try:
|
||
cursor.execute('DELETE FROM javdb_actors WHERE href = ?', (href,))
|
||
conn.commit()
|
||
logging.info(f"成功删除演员: {href}")
|
||
except Exception as e:
|
||
logging.error(f"删除演员 {href} 失败: {e}")
|
||
conn.rollback()
|
||
|
||
# 查询
|
||
def query_actors(**filters):
|
||
try:
|
||
sql = "SELECT href, name FROM javdb_actors WHERE 1=1"
|
||
params = []
|
||
|
||
conditions = {
|
||
"id": " AND id = ?",
|
||
"href": " AND href = ?",
|
||
"name": " AND name LIKE ?",
|
||
"is_full_data": " AND is_full_data = ?",
|
||
"from_actor_list": " AND from_actor_list = ?",
|
||
"before_updated_at": " AND updated_at <= ?",
|
||
"after_updated_at": " AND updated_at >= ?",
|
||
"start_id": " AND id > ?",
|
||
}
|
||
|
||
for key, condition in conditions.items():
|
||
if key in filters:
|
||
sql += condition
|
||
if key == "name":
|
||
params.append(f"%{filters[key]}%")
|
||
else:
|
||
params.append(filters[key])
|
||
|
||
for key in ["is_full_data_in", "is_full_data_not_in"]:
|
||
if key in filters:
|
||
values = filters[key]
|
||
if values:
|
||
placeholders = ", ".join(["?"] * len(values))
|
||
operator = "IN" if key == "is_full_data_in" else "NOT IN"
|
||
sql += f" AND is_full_data {operator} ({placeholders})"
|
||
params.extend(values)
|
||
|
||
if "order_by" in filters:
|
||
# 注意：这里 order by 后面直接跟字段名，不能用占位符，否则会被当作字符串处理
|
||
sql += f" ORDER BY {filters['order_by']} "
|
||
|
||
if 'limit' in filters:
|
||
sql += " LIMIT ?"
|
||
params.append(filters["limit"])
|
||
|
||
cursor.execute(sql, params)
|
||
#return [row[0].lower() for row in cursor.fetchall()] # 返回小写
|
||
return [{'href': row[0], 'name': row[1]} for row in cursor.fetchall()]
|
||
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询 href 失败: {e}")
|
||
return None
|
||
|
||
|
||
# 插入或更新发行商 """
|
||
def insert_or_update_makers(data, caller='list'):
|
||
try:
|
||
if caller == 'list':
|
||
cursor.execute("""
|
||
INSERT INTO javdb_makers (name, href, from_list, updated_at)
|
||
VALUES (?, ? , 1, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
name = excluded.name,
|
||
from_list = 1,
|
||
updated_at = datetime('now', 'localtime')
|
||
""", (data["name"], data["href"]))
|
||
conn.commit()
|
||
elif caller == 'movie':
|
||
cursor.execute("""
|
||
INSERT INTO javdb_makers (name, href, from_movie_list, updated_at)
|
||
VALUES (?, ? , 1, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
name = excluded.name,
|
||
from_movie_list = 1,
|
||
updated_at = datetime('now', 'localtime')
|
||
""", (data["name"], data["href"]))
|
||
conn.commit()
|
||
else:
|
||
logging.warning(f"unexpected caller: {caller}")
|
||
return None
|
||
|
||
# 获取 performer_id
|
||
cursor.execute("SELECT id FROM javdb_makers WHERE href = ?", (data["href"],))
|
||
dist_id = cursor.fetchone()[0]
|
||
if dist_id:
|
||
logging.debug(f"成功插入/更新发行商: {data['name']}")
|
||
return dist_id
|
||
else:
|
||
return None
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error(f"数据库错误: {e}")
|
||
return None
|
||
|
||
# 删除发行商（按 id 或 name） """
|
||
def delete_maker(identifier):
|
||
try:
|
||
if isinstance(identifier, int):
|
||
cursor.execute("DELETE FROM javdb_makers WHERE id = ?", (identifier,))
|
||
elif isinstance(identifier, str):
|
||
cursor.execute("DELETE FROM javdb_makers WHERE name = ?", (identifier,))
|
||
conn.commit()
|
||
logging.info(f"成功删除发行商: {identifier}")
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error(f"删除失败: {e}")
|
||
|
||
# 查询发行商（按 id 或 name） """
|
||
def query_maker(identifier):
|
||
try:
|
||
if isinstance(identifier, int):
|
||
cursor.execute("SELECT * FROM javdb_makers WHERE id = ?", (identifier,))
|
||
else:
|
||
cursor.execute("SELECT * FROM javdb_makers WHERE name LIKE ?", (f"%{identifier}%",))
|
||
|
||
distributor = cursor.fetchone()
|
||
if distributor:
|
||
return dict(zip([desc[0] for desc in cursor.description], distributor))
|
||
else:
|
||
logging.warning(f"未找到发行商: {identifier}")
|
||
return None
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询失败: {e}")
|
||
return None
|
||
|
||
# 按条件查询 href 列表
|
||
def query_maker_hrefs(**filters):
|
||
try:
|
||
sql = "SELECT href, id, from_list FROM javdb_makers WHERE 1=1"
|
||
params = []
|
||
|
||
if "id" in filters:
|
||
sql += " AND id = ?"
|
||
params.append(filters["id"])
|
||
if "from_list" in filters:
|
||
sql += " AND from_list = ?"
|
||
params.append(filters["from_list"])
|
||
if "url" in filters:
|
||
sql += " AND href = ?"
|
||
params.append(filters["href"])
|
||
if "name" in filters:
|
||
sql += " AND name LIKE ?"
|
||
params.append(f"%{filters['name']}%")
|
||
if 'limit' in filters:
|
||
sql += " limit ?"
|
||
params.append(filters["limit"])
|
||
|
||
cursor.execute(sql, params)
|
||
#return [row[0] for row in cursor.fetchall()] # 链接使用小写
|
||
return [{'href': row[0], 'id': row[1], 'from_list':row[2]} for row in cursor.fetchall()]
|
||
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询 href 失败: {e}")
|
||
return None
|
||
|
||
# """ 插入或更新制作公司 """
|
||
def insert_or_update_series(data, caller='list'):
|
||
try:
|
||
if caller == 'list':
|
||
cursor.execute("""
|
||
INSERT INTO javdb_series (name, href, from_list, updated_at)
|
||
VALUES (?, ? , 1, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
name = excluded.name,
|
||
from_list = 1,
|
||
updated_at = datetime('now', 'localtime')
|
||
""", (data["name"], data["href"]))
|
||
conn.commit()
|
||
elif caller == 'movie':
|
||
cursor.execute("""
|
||
INSERT INTO javdb_series (name, href, from_movie_list, updated_at)
|
||
VALUES (?, ? , 1, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
name = excluded.name,
|
||
from_movie_list = 1,
|
||
updated_at = datetime('now', 'localtime')
|
||
""", (data["name"], data["href"]))
|
||
conn.commit()
|
||
else:
|
||
logging.warning(f"unexpected caller: {caller}")
|
||
return None
|
||
|
||
# 获取 performer_id
|
||
cursor.execute("SELECT id FROM javdb_series WHERE href = ?", (data["href"],))
|
||
stu_id = cursor.fetchone()[0]
|
||
if stu_id:
|
||
logging.debug(f"成功插入/更新发行商: {data['name']}")
|
||
return stu_id
|
||
else:
|
||
return None
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error(f"数据库错误: {e}")
|
||
return None
|
||
|
||
# """ 删除制作公司（按 id 或 name） """
|
||
def delete_series(identifier):
|
||
try:
|
||
if isinstance(identifier, int):
|
||
cursor.execute("DELETE FROM javdb_series WHERE id = ?", (identifier,))
|
||
elif isinstance(identifier, str):
|
||
cursor.execute("DELETE FROM javdb_series WHERE name = ?", (identifier,))
|
||
conn.commit()
|
||
logging.info(f"成功删除制作公司: {identifier}")
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error(f"删除失败: {e}")
|
||
|
||
# """ 查询制作公司（按 id 或 name） """
|
||
def query_series(identifier):
|
||
try:
|
||
if isinstance(identifier, int):
|
||
cursor.execute("SELECT * FROM javdb_series WHERE id = ?", (identifier,))
|
||
else:
|
||
cursor.execute("SELECT * FROM javdb_series WHERE name LIKE ?", (f"%{identifier}%",))
|
||
|
||
studio = cursor.fetchone()
|
||
if studio:
|
||
return dict(zip([desc[0] for desc in cursor.description], studio))
|
||
else:
|
||
logging.warning(f"未找到制作公司: {identifier}")
|
||
return None
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询失败: {e}")
|
||
return None
|
||
|
||
# 按条件查询 href 列表
|
||
def query_series_hrefs(**filters):
|
||
try:
|
||
sql = "SELECT href, id, from_list FROM javdb_series WHERE 1=1"
|
||
params = []
|
||
|
||
if "id" in filters:
|
||
sql += " AND id = ?"
|
||
params.append(filters["id"])
|
||
if "from_list" in filters:
|
||
sql += " AND from_list = ?"
|
||
params.append(filters["from_list"])
|
||
if "href" in filters:
|
||
sql += " AND href = ?"
|
||
params.append(filters["href"])
|
||
if "name" in filters:
|
||
sql += " AND name LIKE ?"
|
||
params.append(f"%{filters['name']}%")
|
||
if 'limit' in filters:
|
||
sql += " limit ?"
|
||
params.append(filters["limit"])
|
||
|
||
cursor.execute(sql, params)
|
||
#return [row[0] for row in cursor.fetchall()] # 链接使用小写
|
||
#return [{'href': row[0], 'id': row[1]} for row in cursor.fetchall()]
|
||
return [{'href': row[0], 'id': row[1], 'from_list':row[2]} for row in cursor.fetchall()]
|
||
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询 href 失败: {e}")
|
||
return None
|
||
|
||
# 插入或更新发行商 """
|
||
def insert_or_update_publishers(data, caller='list'):
|
||
try:
|
||
if caller == 'list':
|
||
cursor.execute("""
|
||
INSERT INTO javdb_publishers (name, href, from_list, updated_at)
|
||
VALUES (?, ? , 1, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
name = excluded.name,
|
||
from_list = 1,
|
||
updated_at = datetime('now', 'localtime')
|
||
""", (data["name"], data["href"]))
|
||
conn.commit()
|
||
elif caller == 'movie':
|
||
cursor.execute("""
|
||
INSERT INTO javdb_publishers (name, href, from_movie_list, updated_at)
|
||
VALUES (?, ? , 1, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
name = excluded.name,
|
||
from_movie_list = 1,
|
||
updated_at = datetime('now', 'localtime')
|
||
""", (data["name"], data["href"]))
|
||
conn.commit()
|
||
else:
|
||
logging.warning(f"unexpected caller: {caller}")
|
||
return None
|
||
|
||
# 获取 performer_id
|
||
cursor.execute("SELECT id FROM javdb_publishers WHERE href = ?", (data["href"],))
|
||
dist_id = cursor.fetchone()[0]
|
||
if dist_id:
|
||
logging.debug(f"成功插入/更新发行商: {data['name']}")
|
||
return dist_id
|
||
else:
|
||
return None
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error(f"数据库错误: {e}")
|
||
return None
|
||
|
||
# 删除发行商（按 id 或 name） """
|
||
def delete_publishers(identifier):
|
||
try:
|
||
if isinstance(identifier, int):
|
||
cursor.execute("DELETE FROM javdb_publishers WHERE id = ?", (identifier,))
|
||
elif isinstance(identifier, str):
|
||
cursor.execute("DELETE FROM javdb_publishers WHERE name = ?", (identifier,))
|
||
conn.commit()
|
||
logging.info(f"成功删除发行商: {identifier}")
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error(f"删除失败: {e}")
|
||
|
||
# 查询发行商（按 id 或 name） """
|
||
def query_publishers(identifier):
|
||
try:
|
||
if isinstance(identifier, int):
|
||
cursor.execute("SELECT * FROM javdb_publishers WHERE id = ?", (identifier,))
|
||
else:
|
||
cursor.execute("SELECT * FROM javdb_publishers WHERE name LIKE ?", (f"%{identifier}%",))
|
||
|
||
distributor = cursor.fetchone()
|
||
if distributor:
|
||
return dict(zip([desc[0] for desc in cursor.description], distributor))
|
||
else:
|
||
logging.warning(f"未找到发行商: {identifier}")
|
||
return None
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询失败: {e}")
|
||
return None
|
||
|
||
# 按条件查询 href 列表
|
||
def query_publishers_hrefs(**filters):
|
||
try:
|
||
sql = "SELECT href, id FROM javdb_publishers WHERE 1=1"
|
||
params = []
|
||
|
||
if "id" in filters:
|
||
sql += " AND id = ?"
|
||
params.append(filters["id"])
|
||
if "from_list" in filters:
|
||
sql += " AND from_list = ?"
|
||
params.append(filters["from_list"])
|
||
if "url" in filters:
|
||
sql += " AND href = ?"
|
||
params.append(filters["href"])
|
||
if "name" in filters:
|
||
sql += " AND name LIKE ?"
|
||
params.append(f"%{filters['name']}%")
|
||
if 'limit' in filters:
|
||
sql += " limit ?"
|
||
params.append(filters["limit"])
|
||
|
||
cursor.execute(sql, params)
|
||
#return [row[0] for row in cursor.fetchall()] # 链接使用小写
|
||
return [{'href': row[0], 'id': row[1]} for row in cursor.fetchall()]
|
||
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询 href 失败: {e}")
|
||
return None
|
||
|
||
|
||
# 插入或更新类别 """
|
||
def insert_or_update_tags(name, href):
|
||
try:
|
||
if href in cached_tags:
|
||
return cached_tags[href]['id']
|
||
|
||
cursor.execute("""
|
||
INSERT INTO javdb_tags (name, href, updated_at)
|
||
VALUES (?, ? , datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
name = excluded.name,
|
||
updated_at = datetime('now', 'localtime')
|
||
""", (name, href))
|
||
conn.commit()
|
||
|
||
cursor.execute("SELECT id, name, href FROM javdb_tags")
|
||
for row in cursor.fetchall():
|
||
cached_tags[row[2]] = {'id': row[0], 'name':row[2]}
|
||
|
||
if href in cached_tags:
|
||
dist_id = cached_tags[href]['id']
|
||
logging.debug(f"insert/update tags succ. id: {dist_id}, name: {name}")
|
||
return dist_id
|
||
else:
|
||
return None
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error(f"数据库错误: {e}")
|
||
return None
|
||
|
||
# 查询tags
|
||
def query_tags(href, name):
|
||
global cached_tags
|
||
try:
|
||
if href not in cached_tags:
|
||
cursor.execute("SELECT id, name, href FROM javdb_tags")
|
||
for row in cursor.fetchall():
|
||
cached_tags[row[2]] = {'id': row[0], 'name':row[2]}
|
||
|
||
if href in cached_tags:
|
||
return cached_tags[href]['id'], cached_tags[href]['name']
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询失败: {e}")
|
||
return 0, name
|
||
|
||
# 插入影片和tags的关联数据
|
||
def insert_movie_tags( movie_id, tag_id, tags=''):
|
||
try:
|
||
cursor.execute("""
|
||
INSERT INTO javdb_movies_tags (movie_id, tag_id, tags, updated_at)
|
||
VALUES (?, ?, ?, datetime('now', 'localtime'))
|
||
ON CONFLICT(tag_id, movie_id) DO UPDATE SET tags=excluded.tags, updated_at=datetime('now', 'localtime')
|
||
""",
|
||
(movie_id, tag_id, tags)
|
||
)
|
||
conn.commit()
|
||
|
||
#logging.debug(f'insert one performer_movie, performer_id: {performer_id}, movie_id: {movie_id}')
|
||
|
||
return movie_id
|
||
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logging.error("Error inserting movie: %s", e)
|
||
return None
|
||
|
||
# """插入或更新电影数据"""
|
||
def insert_or_update_movie(movie):
|
||
try:
|
||
# 获取相关 ID
|
||
makers_id = get_id_by_href('javdb_makers', movie['maker_link']) if movie['maker_link'] else None
|
||
series_id = get_id_by_href('javdb_series', movie['series_link']) if movie['series_link'] else None
|
||
pub_id = get_id_by_href('javdb_publishers', movie['pub_link']) if movie['pub_link'] else None
|
||
|
||
# 如果不存在，插入
|
||
if makers_id is None and movie['maker_link']:
|
||
makers_id = insert_or_update_makers({'name' : movie.get('maker_name', ''), 'href' : movie.get('maker_link', '')}, caller='movie')
|
||
if series_id is None and movie['series_link']:
|
||
series_id = insert_or_update_series({'name' : movie.get('series_name', ''), 'href' : movie.get('series_link', '')}, caller='movie')
|
||
if pub_id is None and movie['pub_link']:
|
||
pub_id = insert_or_update_publishers({'name' : movie.get('pub_name', ''), 'href' : movie.get('pub_link', '')}, caller='movie')
|
||
|
||
cursor.execute("""
|
||
INSERT INTO javdb_movies (href, title, cover_url, serial_number, release_date, duration,
|
||
maker_id, series_id, pub_id, is_full_data, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
title=excluded.title,
|
||
cover_url=excluded.cover_url,
|
||
serial_number=excluded.serial_number,
|
||
release_date=excluded.release_date,
|
||
duration=excluded.duration,
|
||
maker_id=excluded.maker_id,
|
||
series_id=excluded.series_id,
|
||
pub_id=excluded.pub_id,
|
||
is_full_data=1,
|
||
updated_at=datetime('now', 'localtime')
|
||
""", (movie['href'], movie['title'], movie['cover_url'], movie['serial_number'],
|
||
movie['release_date'], movie['duration'], makers_id, series_id, pub_id))
|
||
|
||
conn.commit()
|
||
|
||
# 获取插入的 movie_id
|
||
movie_id = get_id_by_href('javdb_movies', movie['href'])
|
||
if movie_id is None:
|
||
return None
|
||
|
||
logging.debug(f"insert one move, id: {movie_id}, title: {movie['title']}, href: {movie['href']}")
|
||
|
||
# 插入 performers_movies 关系表
|
||
for performer in movie.get('actors', []):
|
||
performer_id = get_id_by_href('javdb_actors', performer['href'])
|
||
# 如果演员不存在，先插入
|
||
if performer_id is None:
|
||
performer_id = insert_actor_index(performer['name'], performer['href'], from_movie_list=1)
|
||
logging.debug(f"insert new perfomer. perfomer_id: {performer_id}, name:{performer['name']}")
|
||
if performer_id:
|
||
tmp_id = insert_actor_movie(performer_id, movie_id)
|
||
if tmp_id:
|
||
logging.debug(f"insert one perfomer_movie. perfomer_id: {performer_id}, movie_id:{movie_id}")
|
||
else:
|
||
logging.debug(f"insert perfomer_movie failed. perfomer_id: {performer_id}, movie_id:{movie_id}")
|
||
else:
|
||
logging.warning(f"insert perfomer failed. name: {performer['name']}, href: {performer['href']}")
|
||
|
||
# 插入 tags 表
|
||
for tag in movie.get('tags', []):
|
||
tag_name = tag.get('name', '')
|
||
tag_href = tag.get('href', '')
|
||
tag_id = insert_or_update_tags(tag_name, tag_href)
|
||
if tag_id:
|
||
logging.debug(f"insert one tags. tag_id: {tag_id}, name: {tag_name}")
|
||
tmp_id = insert_movie_tags(movie_id=movie_id, tag_id=tag_id, tags=tag_name)
|
||
if tmp_id:
|
||
logging.debug(f"insert one movie_tag. movie_id: {movie_id}, tag_id: {tag_id}, name: {tag_name}")
|
||
else:
|
||
logging.warning(f"insert one movie_tag error. movie_id: {movie_id}, tag_id: {tag_id}, name: {tag_name}")
|
||
else:
|
||
logging.warning(f"insert tags error. name:{tag_name}, href: {tag_href}")
|
||
|
||
return movie_id
|
||
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logging.error("Error inserting movie: %s", e)
|
||
return None
|
||
|
||
# """插入或更新电影数据(异常url的处理，比如404链接)"""
|
||
def insert_or_update_movie_404(title, href, is_full_data=1):
|
||
try:
|
||
# 插入或更新电影信息
|
||
cursor.execute(
|
||
"""
|
||
INSERT INTO javdb_movies (title, href, is_full_data, updated_at)
|
||
VALUES (?, ?, ?, datetime('now', 'localtime'))
|
||
ON CONFLICT(href) DO UPDATE SET
|
||
title=excluded.title, is_full_data=excluded.is_full_data, updated_at = datetime('now', 'localtime')
|
||
""",
|
||
(title, href, is_full_data)
|
||
)
|
||
conn.commit()
|
||
|
||
# 获取插入的 movie_id
|
||
movie_id = get_id_by_href('javdb_movies', href)
|
||
if movie_id is None:
|
||
return None
|
||
|
||
return movie_id
|
||
|
||
except Exception as e:
|
||
conn.rollback()
|
||
logging.error("Error inserting movie: %s", e)
|
||
return None
|
||
|
||
|
||
# 删除电影数据"""
|
||
def delete_movie(identifier):
|
||
try:
|
||
if isinstance(identifier, int):
|
||
cursor.execute("DELETE FROM javdb_movies WHERE id = ?", (identifier,))
|
||
elif isinstance(identifier, str):
|
||
cursor.execute("DELETE FROM javdb_movies WHERE href = ?", (identifier,))
|
||
else:
|
||
logging.warning("无效的删除参数")
|
||
return
|
||
conn.commit()
|
||
logging.info(f"Deleted movie with {identifier}")
|
||
|
||
except sqlite3.Error as e:
|
||
conn.rollback()
|
||
logging.error("Error deleting movie: %s", e)
|
||
|
||
# 查找电影数据"""
|
||
def query_movies(identifier):
|
||
try:
|
||
if isinstance(identifier, int):
|
||
cursor.execute("SELECT * FROM javdb_movies WHERE id = ?", (identifier,))
|
||
elif "http" in identifier:
|
||
cursor.execute("SELECT * FROM javdb_movies WHERE href = ?", (identifier,))
|
||
else:
|
||
cursor.execute("SELECT * FROM javdb_movies WHERE title LIKE ?", (f"%{identifier}%",))
|
||
|
||
movie = cursor.fetchone()
|
||
if movie:
|
||
cursor.execute("SELECT * FROM javdb_actors_movies WHERE performer_id = ?", (movie[0],))
|
||
performers = [row[0] for row in cursor.fetchall()]
|
||
result = dict(zip([desc[0] for desc in cursor.description], performers))
|
||
result["performers"] = performers
|
||
return result
|
||
else:
|
||
logging.warning(f"find no data: {identifier}")
|
||
return None
|
||
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询失败: {e}")
|
||
return None
|
||
'''
|
||
# 按条件查询 href 列表
|
||
def query_movie_hrefs_old(**filters):
|
||
try:
|
||
sql = "SELECT href, title, id FROM javdb_movies WHERE 1=1"
|
||
params = []
|
||
|
||
if "id" in filters:
|
||
sql += " AND id = ?"
|
||
params.append(filters["id"])
|
||
if "href" in filters:
|
||
sql += " AND href = ?"
|
||
params.append(filters["href"])
|
||
if "title" in filters:
|
||
sql += " AND title LIKE ?"
|
||
params.append(f"%{filters['title']}%")
|
||
if "is_full_data" in filters:
|
||
sql += " AND is_full_data = ?"
|
||
params.append(filters["is_full_data"])
|
||
if "from_actor_list" in filters:
|
||
sql += " AND from_actor_list = ?"
|
||
params.append(filters["from_actor_list"])
|
||
if "is_full_data_in" in filters:
|
||
values = filters["is_full_data_in"]
|
||
if values:
|
||
placeholders = ", ".join(["?"] * len(values))
|
||
sql += f" AND is_full_data IN ({placeholders})"
|
||
params.extend(values)
|
||
if "is_full_data_not_in" in filters:
|
||
values = filters["is_full_data_not_in"]
|
||
if values:
|
||
placeholders = ", ".join(["?"] * len(values))
|
||
sql += f" AND is_full_data NOT IN ({placeholders})"
|
||
params.extend(values)
|
||
if "before_updated_at" in filters:
|
||
sql += " AND updated_at <= ?"
|
||
params.append(filters["before_updated_at"])
|
||
if "after_updated_at" in filters:
|
||
sql += " AND updated_at >= ?"
|
||
params.append(filters["after_updated_at"])
|
||
if "start_id" in filters:
|
||
sql += " AND id > ?"
|
||
params.append(filters["start_id"])
|
||
if "order_by" in filters:
|
||
sql += " order by ?"
|
||
params.append(filters["order_by"])
|
||
if 'limit' in filters:
|
||
sql += " limit ?"
|
||
params.append(filters["limit"])
|
||
|
||
cursor.execute(sql, params)
|
||
#return [row[0].lower() for row in cursor.fetchall()] # 链接使用小写
|
||
return [{'href': row[0], 'title': row[1], 'id':row[2]} for row in cursor.fetchall()]
|
||
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询 href 失败: {e}")
|
||
return []
|
||
'''
|
||
# 查询
|
||
def query_movie_hrefs(**filters):
|
||
try:
|
||
sql = "SELECT href, title, id FROM javdb_movies WHERE 1=1"
|
||
params = []
|
||
|
||
conditions = {
|
||
"id": " AND id = ?",
|
||
"href": " AND href = ?",
|
||
"title": " AND title LIKE ?",
|
||
"is_full_data": " AND is_full_data = ?",
|
||
"uncensored": " AND uncensored = ?",
|
||
"from_actor_list": " AND from_actor_list = ?",
|
||
"before_updated_at": " AND updated_at <= ?",
|
||
"after_updated_at": " AND updated_at >= ?",
|
||
"start_id": " AND id > ?",
|
||
}
|
||
|
||
for key, condition in conditions.items():
|
||
if key in filters:
|
||
sql += condition
|
||
if key == "title":
|
||
params.append(f"%{filters[key]}%")
|
||
else:
|
||
params.append(filters[key])
|
||
|
||
for key in ["is_full_data_in", "is_full_data_not_in"]:
|
||
if key in filters:
|
||
values = filters[key]
|
||
if values:
|
||
placeholders = ", ".join(["?"] * len(values))
|
||
operator = "IN" if key == "is_full_data_in" else "NOT IN"
|
||
sql += f" AND is_full_data {operator} ({placeholders})"
|
||
params.extend(values)
|
||
|
||
if "order_by" in filters:
|
||
# 注意：这里 order by 后面直接跟字段名，不能用占位符，否则会被当作字符串处理
|
||
sql += f" ORDER BY {filters['order_by']} "
|
||
|
||
if 'limit' in filters:
|
||
sql += " LIMIT ?"
|
||
params.append(filters["limit"])
|
||
|
||
cursor.execute(sql, params)
|
||
#return [row[0].lower() for row in cursor.fetchall()] # 返回小写
|
||
#return [{'href': row[0], 'name': row[1]} for row in cursor.fetchall()]
|
||
return [{'href': row[0], 'title': row[1], 'id':row[2]} for row in cursor.fetchall()]
|
||
|
||
except sqlite3.Error as e:
|
||
logging.error(f"查询 href 失败: {e}")
|
||
return None
|
||
|
||
# 插入一条任务日志
|
||
def insert_task_log():
|
||
try:
|
||
cursor.execute("""
|
||
INSERT INTO javdb_task_log (task_status) VALUES ('Start')
|
||
""")
|
||
conn.commit()
|
||
|
||
task_id = cursor.lastrowid
|
||
if task_id is None:
|
||
return None
|
||
update_task_log(task_id=task_id, task_status='Start')
|
||
|
||
return task_id # 获取插入的 task_id
|
||
except sqlite3.Error as e:
|
||
logging.error(f"插入任务失败: {e}")
|
||
return None
|
||
|
||
# 更新任务日志的字段
|
||
def update_task_log_inner(task_id, **kwargs):
|
||
try:
|
||
fields = ", ".join(f"{key} = ?" for key in kwargs.keys())
|
||
params = list(kwargs.values()) + [task_id]
|
||
|
||
sql = f"UPDATE javdb_task_log SET {fields}, updated_at = datetime('now', 'localtime') WHERE task_id = ?"
|
||
cursor.execute(sql, params)
|
||
conn.commit()
|
||
except sqlite3.Error as e:
|
||
logging.error(f"更新任务 {task_id} 失败: {e}")
|
||
|
||
# 更新任务日志的字段
|
||
def update_task_log(task_id, task_status):
|
||
try:
|
||
# 获取 performers、studios 等表的最终行数
|
||
cursor.execute("SELECT COUNT(*) FROM javdb_actors where is_full_data=1")
|
||
full_data_actors = cursor.fetchone()[0]
|
||
cursor.execute("SELECT COUNT(*) FROM javdb_actors")
|
||
total_actors = cursor.fetchone()[0]
|
||
|
||
cursor.execute("SELECT COUNT(*) FROM javdb_movies where is_full_data=1")
|
||
full_data_movies = cursor.fetchone()[0]
|
||
cursor.execute("SELECT COUNT(*) FROM javdb_movies")
|
||
total_movies = cursor.fetchone()[0]
|
||
|
||
cursor.execute("SELECT COUNT(*) FROM javdb_makers")
|
||
total_makers = cursor.fetchone()[0]
|
||
|
||
cursor.execute("SELECT COUNT(*) FROM javdb_series")
|
||
total_series = cursor.fetchone()[0]
|
||
|
||
# 更新 task_log
|
||
update_task_log_inner(task_id,
|
||
full_data_actors=full_data_actors,
|
||
total_actors=total_actors,
|
||
full_data_movies=full_data_movies,
|
||
total_movies=total_movies,
|
||
total_makers=total_makers,
|
||
total_series=total_series,
|
||
task_status=task_status)
|
||
|
||
except sqlite3.Error as e:
|
||
logging.error(f"更新任务 {task_id} 失败: {e}")
|
||
|
||
|
||
# 任务结束，更新字段
|
||
def finalize_task_log(task_id):
|
||
try:
|
||
# 更新 task_log
|
||
update_task_log(task_id, task_status="Success")
|
||
except sqlite3.Error as e:
|
||
logging.error(f"任务 {task_id} 结束失败: {e}")
|
||
|
||
|
||
# 测试代码
|
||
if __name__ == "__main__":
|
||
|
||
sample_data = [
|
||
{
|
||
'name': '上原亜衣',
|
||
'href': 'https://www.javdb.com/actors/MkAX',
|
||
'pic': 'https://c0.jdbstatic.com/avatars/mk/MkAX.jpg',
|
||
'alias': ['上原亜衣', '下原舞', '早瀬クリスタル', '阿蘇山百式屏風奉行']
|
||
},
|
||
{
|
||
'name': '大橋未久',
|
||
'href': 'https://www.javdb.com/actors/21Jp',
|
||
'pic': 'https://c0.jdbstatic.com/avatars/21/21Jp.jpg',
|
||
'alias': ['大橋未久']
|
||
},
|
||
]
|
||
|
||
for actor in sample_data:
|
||
insert_or_update_actor(actor)
|
||
|
||
print(query_actors("name LIKE '%未久%'"))
|
||
#delete_actor_by_href('https://www.javdb.com/actors/MkAX')
|
||
print(query_actors())
|