import sqlite3 import json import config import logging from datetime import datetime # 连接 SQLite 数据库 DB_PATH = f"{config.global_share_data_dir}/sqlite/shared.db" # 替换为你的数据库文件 conn = sqlite3.connect(DB_PATH, check_same_thread=False) cursor = conn.cursor() cached_tags = {} # """从指定表中通过 href 查找 id""" def get_id_by_href(table: str, href: str) -> int: if href is None: return None cursor.execute(f"SELECT id FROM {table} WHERE href = ?", (href,)) row = cursor.fetchone() return row[0] if row else None def insert_actor_index(name, href, from_actor_list=None, from_movie_list=None): try: # **查询是否已存在该演员** cursor.execute("SELECT id, name, from_actor_list, from_movie_list FROM javdb_actors WHERE href = ?", (href,)) existing_actor = cursor.fetchone() if existing_actor: # **如果演员已存在** actor_id, existing_name, existing_actor_list, existing_movie_list = existing_actor # **如果没有传入值,则保持原有值** from_actor_list = from_actor_list if from_actor_list is not None else existing_actor_list from_movie_list = from_movie_list if from_movie_list is not None else existing_movie_list cursor.execute(""" UPDATE javdb_actors SET name = ?, from_actor_list = ?, from_movie_list = ?, updated_at = datetime('now', 'localtime') WHERE href = ? """, (name, from_actor_list, from_movie_list, href)) else: # **如果演员不存在,插入** cursor.execute(""" INSERT INTO javdb_actors (href, name, from_actor_list, from_movie_list) VALUES (?, ?, COALESCE(?, 0), COALESCE(?, 0)) """, (href, name, from_actor_list, from_movie_list)) conn.commit() performer_id = get_id_by_href('javdb_actors', href) if performer_id: logging.debug(f"Inserted/Updated actor index, id: {performer_id}, name: {name}, href: {href}") return performer_id except sqlite3.Error as e: conn.rollback() logging.error(f"数据库错误: {e}") return None except Exception as e: conn.rollback() logging.error(f"未知错误: {e}") return None ''' def insert_movie_index_old(title, href, from_actor_list=None, from_movie_makers=None, from_movie_series=None, maker_id=None, series_id=None, from_movie_publishers=None, pub_id=None): try: # **先检查数据库中是否已有该电影** cursor.execute("SELECT id, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, from_movie_publishers, pub_id FROM javdb_movies WHERE href = ?", (href,)) existing_movie = cursor.fetchone() if existing_movie: # **如果电影已存在** movie_id, existing_actor, existing_maker, existing_series, existing_maker_id, existing_series_id, existing_pub, existing_pub_id = existing_movie # **如果没有传入值,就用原来的值** from_actor_list = from_actor_list if from_actor_list is not None else existing_actor from_movie_makers = from_movie_makers if from_movie_makers is not None else existing_maker from_movie_series = from_movie_series if from_movie_series is not None else existing_series maker_id = maker_id if maker_id is not None else existing_maker_id series_id = series_id if series_id is not None else existing_series_id from_movie_publishers = from_movie_publishers if from_movie_publishers is not None else existing_pub pub_id = pub_id if pub_id is not None else existing_pub_id cursor.execute(""" UPDATE javdb_movies SET title = ?, from_actor_list = ?, from_movie_makers = ?, from_movie_series = ?, maker_id = ?, series_id = ?, from_movie_publishers = ?, pub_id = ?, updated_at = datetime('now', 'localtime') WHERE href = ? """, (title, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, from_movie_publishers, pub_id, href)) else: # **如果电影不存在,插入** cursor.execute(""" INSERT INTO javdb_movies (title, href, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, from_movie_publishers, pub_id) VALUES (?, ?, COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0)) """, (title, href, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, from_movie_publishers, pub_id)) conn.commit() movie_id = get_id_by_href('javdb_movies', href) if movie_id: logging.debug(f'Inserted/Updated movie index, id: {movie_id}, title: {title}, href: {href}') return movie_id except Exception as e: conn.rollback() logging.error(f"Error inserting/updating movie: {e}") return None ''' def insert_movie_index(title, href, **kwargs): try: # 先检查数据库中是否已有该电影 cursor.execute("SELECT * FROM javdb_movies WHERE href = ?", (href,)) existing_movie = cursor.fetchone() # 获取列名 column_names = [description[0] for description in cursor.description] fields = [ 'from_actor_list', 'from_movie_makers', 'from_movie_series', 'from_movie_publishers', 'maker_id', 'series_id', 'pub_id', 'uncensored' ] if existing_movie: # 如果电影已存在 existing_values = dict(zip(column_names, existing_movie)) movie_id = existing_values['id'] logging.debug(f"values in db: {existing_values}") # 如果没有传入值,就用原来的值 for field in fields: kwargs[field] = kwargs.get(field) if kwargs.get(field) is not None else existing_values[field] set_clauses = ", ".join([f"{field} = ?" for field in fields]) sql = f""" UPDATE javdb_movies SET title = ?, {set_clauses}, updated_at = datetime('now', 'localtime') WHERE href = ? """ values = [title] + [kwargs[field] for field in fields] + [href] logging.debug(f"sql: {sql}, values: {values}") cursor.execute(sql, values) else: # 如果电影不存在,插入 columns = ', '.join(['title', 'href'] + fields) placeholders = ', '.join(['?'] * (len(fields) + 2)) sql = f"INSERT INTO javdb_movies ({columns}) VALUES ({placeholders})" values = [title, href] + [kwargs.get(field, 0) for field in fields] logging.debug(f"sql: {sql}, values: {values}") cursor.execute(sql, values) conn.commit() movie_id = get_id_by_href('javdb_movies', href) if movie_id: logging.debug(f'Inserted/Updated movie index, id: {movie_id}, title: {title}, href: {href}') return movie_id except Exception as e: conn.rollback() logging.error(f"Error inserting/updating movie: {e}") return None # 插入演员和电影的关联数据 def insert_actor_movie(performer_id, movie_id, tags=''): try: cursor.execute(""" INSERT INTO javdb_actors_movies (actor_id, movie_id, tags, updated_at) VALUES (?, ?, ?, datetime('now', 'localtime')) ON CONFLICT(actor_id, movie_id) DO UPDATE SET tags=excluded.tags, updated_at=datetime('now', 'localtime') """, (performer_id, movie_id, tags) ) conn.commit() #logging.debug(f'insert one performer_movie, performer_id: {performer_id}, movie_id: {movie_id}') return performer_id except Exception as e: conn.rollback() logging.error("Error inserting movie: %s", e) return None # 插入演员数据 def insert_or_update_actor(actor): try: cursor.execute(''' INSERT INTO javdb_actors (name, href, pic, is_full_data, updated_at) VALUES (?, ?, ?, 1, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name=excluded.name, pic=excluded.pic, is_full_data=1, updated_at=datetime('now', 'localtime') ''', (actor['name'], actor['href'], actor['pic'])) conn.commit() # 查询刚插入的数据 cursor.execute('SELECT id, from_actor_list FROM javdb_actors WHERE href = ?', (actor['href'],)) actor_id, uncensored = cursor.fetchone() if actor_id is None: logging.warning(f'insert data error. name: {actor['name']}, href: {actor['href']}') return None logging.debug(f'insert one actor, id: {actor_id}, name: {actor['name']}, href: {actor['href']}') # 插入别名 for alias in actor.get("alias") or []: cursor.execute(''' INSERT OR IGNORE INTO javdb_actors_alias (actor_id, alias, updated_at) VALUES (?, ?, datetime('now', 'localtime')) ''', (actor_id, alias)) conn.commit() # 插入影片列表 for movie in actor.get("credits") or []: # from_actor_list = 1 表示无码影星的,其他不处理 if uncensored and uncensored > 0: movie_id = insert_movie_index(movie['title'], movie['href'], from_actor_list=1, uncensored=uncensored) else: movie_id = insert_movie_index(movie['title'], movie['href'], from_actor_list=1) if movie_id: tmp_id = insert_actor_movie(actor_id, movie_id) if tmp_id : logging.debug(f'insert one performer_movie, performer_id: {actor_id}, movie_id: {movie_id}') else: logging.warning(f'insert performer_movie failed. performer_id: {actor_id}, moive href: {movie['href']}') return actor_id except Exception as e: logging.error(f"插入/更新演员 {actor['name']} 失败: {e}") conn.rollback() # """插入或更新电影数据(异常url的处理,比如404链接)""" def insert_or_update_actor_404(name, href, is_full_data=1): try: # 插入或更新电影信息 cursor.execute( """ INSERT INTO javdb_actors (name, href, is_full_data, updated_at) VALUES (?, ?, ?, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name=excluded.name, is_full_data=excluded.is_full_data, updated_at = datetime('now', 'localtime') """, (name, href, is_full_data) ) conn.commit() # 获取插入的 movie_id actor_id = get_id_by_href('javdb_actors', href) if actor_id is None: return None return actor_id except Exception as e: conn.rollback() logging.error("Error inserting movie: %s", e) return None # 删除演员 def delete_actor_by_href(href): try: cursor.execute('DELETE FROM javdb_actors WHERE href = ?', (href,)) conn.commit() logging.info(f"成功删除演员: {href}") except Exception as e: logging.error(f"删除演员 {href} 失败: {e}") conn.rollback() # 查询 def query_actors(**filters): try: sql = "SELECT href, name FROM javdb_actors WHERE 1=1" params = [] conditions = { "id": " AND id = ?", "href": " AND href = ?", "name": " AND name LIKE ?", "is_full_data": " AND is_full_data = ?", "from_actor_list": " AND from_actor_list = ?", "before_updated_at": " AND updated_at <= ?", "after_updated_at": " AND updated_at >= ?", "start_id": " AND id > ?", } for key, condition in conditions.items(): if key in filters: sql += condition if key == "name": params.append(f"%{filters[key]}%") else: params.append(filters[key]) for key in ["is_full_data_in", "is_full_data_not_in"]: if key in filters: values = filters[key] if values: placeholders = ", ".join(["?"] * len(values)) operator = "IN" if key == "is_full_data_in" else "NOT IN" sql += f" AND is_full_data {operator} ({placeholders})" params.extend(values) if "order_by" in filters: # 注意:这里 order by 后面直接跟字段名,不能用占位符,否则会被当作字符串处理 sql += f" ORDER BY {filters['order_by']} " if 'limit' in filters: sql += " LIMIT ?" params.append(filters["limit"]) cursor.execute(sql, params) #return [row[0].lower() for row in cursor.fetchall()] # 返回小写 return [{'href': row[0], 'name': row[1]} for row in cursor.fetchall()] except sqlite3.Error as e: logging.error(f"查询 href 失败: {e}") return None # 插入或更新发行商 """ def insert_or_update_makers(data, caller='list'): try: if caller == 'list': cursor.execute(""" INSERT INTO javdb_makers (name, href, from_list, updated_at) VALUES (?, ? , 1, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name = excluded.name, from_list = 1, updated_at = datetime('now', 'localtime') """, (data["name"], data["href"])) conn.commit() elif caller == 'movie': cursor.execute(""" INSERT INTO javdb_makers (name, href, from_movie_list, updated_at) VALUES (?, ? , 1, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name = excluded.name, from_movie_list = 1, updated_at = datetime('now', 'localtime') """, (data["name"], data["href"])) conn.commit() else: logging.warning(f"unexpected caller: {caller}") return None # 获取 performer_id cursor.execute("SELECT id FROM javdb_makers WHERE href = ?", (data["href"],)) dist_id = cursor.fetchone()[0] if dist_id: logging.debug(f"成功插入/更新发行商: {data['name']}") return dist_id else: return None except sqlite3.Error as e: conn.rollback() logging.error(f"数据库错误: {e}") return None # 删除发行商(按 id 或 name) """ def delete_maker(identifier): try: if isinstance(identifier, int): cursor.execute("DELETE FROM javdb_makers WHERE id = ?", (identifier,)) elif isinstance(identifier, str): cursor.execute("DELETE FROM javdb_makers WHERE name = ?", (identifier,)) conn.commit() logging.info(f"成功删除发行商: {identifier}") except sqlite3.Error as e: conn.rollback() logging.error(f"删除失败: {e}") # 查询发行商(按 id 或 name) """ def query_maker(identifier): try: if isinstance(identifier, int): cursor.execute("SELECT * FROM javdb_makers WHERE id = ?", (identifier,)) else: cursor.execute("SELECT * FROM javdb_makers WHERE name LIKE ?", (f"%{identifier}%",)) distributor = cursor.fetchone() if distributor: return dict(zip([desc[0] for desc in cursor.description], distributor)) else: logging.warning(f"未找到发行商: {identifier}") return None except sqlite3.Error as e: logging.error(f"查询失败: {e}") return None # 按条件查询 href 列表 def query_maker_hrefs(**filters): try: sql = "SELECT href, id, from_list FROM javdb_makers WHERE 1=1" params = [] if "id" in filters: sql += " AND id = ?" params.append(filters["id"]) if "from_list" in filters: sql += " AND from_list = ?" params.append(filters["from_list"]) if "url" in filters: sql += " AND href = ?" params.append(filters["href"]) if "name" in filters: sql += " AND name LIKE ?" params.append(f"%{filters['name']}%") if 'limit' in filters: sql += " limit ?" params.append(filters["limit"]) cursor.execute(sql, params) #return [row[0] for row in cursor.fetchall()] # 链接使用小写 return [{'href': row[0], 'id': row[1], 'from_list':row[2]} for row in cursor.fetchall()] except sqlite3.Error as e: logging.error(f"查询 href 失败: {e}") return None # """ 插入或更新制作公司 """ def insert_or_update_series(data, caller='list'): try: if caller == 'list': cursor.execute(""" INSERT INTO javdb_series (name, href, from_list, updated_at) VALUES (?, ? , 1, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name = excluded.name, from_list = 1, updated_at = datetime('now', 'localtime') """, (data["name"], data["href"])) conn.commit() elif caller == 'movie': cursor.execute(""" INSERT INTO javdb_series (name, href, from_movie_list, updated_at) VALUES (?, ? , 1, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name = excluded.name, from_movie_list = 1, updated_at = datetime('now', 'localtime') """, (data["name"], data["href"])) conn.commit() else: logging.warning(f"unexpected caller: {caller}") return None # 获取 performer_id cursor.execute("SELECT id FROM javdb_series WHERE href = ?", (data["href"],)) stu_id = cursor.fetchone()[0] if stu_id: logging.debug(f"成功插入/更新发行商: {data['name']}") return stu_id else: return None except sqlite3.Error as e: conn.rollback() logging.error(f"数据库错误: {e}") return None # """ 删除制作公司(按 id 或 name) """ def delete_series(identifier): try: if isinstance(identifier, int): cursor.execute("DELETE FROM javdb_series WHERE id = ?", (identifier,)) elif isinstance(identifier, str): cursor.execute("DELETE FROM javdb_series WHERE name = ?", (identifier,)) conn.commit() logging.info(f"成功删除制作公司: {identifier}") except sqlite3.Error as e: conn.rollback() logging.error(f"删除失败: {e}") # """ 查询制作公司(按 id 或 name) """ def query_series(identifier): try: if isinstance(identifier, int): cursor.execute("SELECT * FROM javdb_series WHERE id = ?", (identifier,)) else: cursor.execute("SELECT * FROM javdb_series WHERE name LIKE ?", (f"%{identifier}%",)) studio = cursor.fetchone() if studio: return dict(zip([desc[0] for desc in cursor.description], studio)) else: logging.warning(f"未找到制作公司: {identifier}") return None except sqlite3.Error as e: logging.error(f"查询失败: {e}") return None # 按条件查询 href 列表 def query_series_hrefs(**filters): try: sql = "SELECT href, id, from_list FROM javdb_series WHERE 1=1" params = [] if "id" in filters: sql += " AND id = ?" params.append(filters["id"]) if "from_list" in filters: sql += " AND from_list = ?" params.append(filters["from_list"]) if "href" in filters: sql += " AND href = ?" params.append(filters["href"]) if "name" in filters: sql += " AND name LIKE ?" params.append(f"%{filters['name']}%") if 'limit' in filters: sql += " limit ?" params.append(filters["limit"]) cursor.execute(sql, params) #return [row[0] for row in cursor.fetchall()] # 链接使用小写 #return [{'href': row[0], 'id': row[1]} for row in cursor.fetchall()] return [{'href': row[0], 'id': row[1], 'from_list':row[2]} for row in cursor.fetchall()] except sqlite3.Error as e: logging.error(f"查询 href 失败: {e}") return None # 插入或更新发行商 """ def insert_or_update_publishers(data, caller='list'): try: if caller == 'list': cursor.execute(""" INSERT INTO javdb_publishers (name, href, from_list, updated_at) VALUES (?, ? , 1, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name = excluded.name, from_list = 1, updated_at = datetime('now', 'localtime') """, (data["name"], data["href"])) conn.commit() elif caller == 'movie': cursor.execute(""" INSERT INTO javdb_publishers (name, href, from_movie_list, updated_at) VALUES (?, ? , 1, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name = excluded.name, from_movie_list = 1, updated_at = datetime('now', 'localtime') """, (data["name"], data["href"])) conn.commit() else: logging.warning(f"unexpected caller: {caller}") return None # 获取 performer_id cursor.execute("SELECT id FROM javdb_publishers WHERE href = ?", (data["href"],)) dist_id = cursor.fetchone()[0] if dist_id: logging.debug(f"成功插入/更新发行商: {data['name']}") return dist_id else: return None except sqlite3.Error as e: conn.rollback() logging.error(f"数据库错误: {e}") return None # 删除发行商(按 id 或 name) """ def delete_publishers(identifier): try: if isinstance(identifier, int): cursor.execute("DELETE FROM javdb_publishers WHERE id = ?", (identifier,)) elif isinstance(identifier, str): cursor.execute("DELETE FROM javdb_publishers WHERE name = ?", (identifier,)) conn.commit() logging.info(f"成功删除发行商: {identifier}") except sqlite3.Error as e: conn.rollback() logging.error(f"删除失败: {e}") # 查询发行商(按 id 或 name) """ def query_publishers(identifier): try: if isinstance(identifier, int): cursor.execute("SELECT * FROM javdb_publishers WHERE id = ?", (identifier,)) else: cursor.execute("SELECT * FROM javdb_publishers WHERE name LIKE ?", (f"%{identifier}%",)) distributor = cursor.fetchone() if distributor: return dict(zip([desc[0] for desc in cursor.description], distributor)) else: logging.warning(f"未找到发行商: {identifier}") return None except sqlite3.Error as e: logging.error(f"查询失败: {e}") return None # 按条件查询 href 列表 def query_publishers_hrefs(**filters): try: sql = "SELECT href, id FROM javdb_publishers WHERE 1=1" params = [] if "id" in filters: sql += " AND id = ?" params.append(filters["id"]) if "from_list" in filters: sql += " AND from_list = ?" params.append(filters["from_list"]) if "url" in filters: sql += " AND href = ?" params.append(filters["href"]) if "name" in filters: sql += " AND name LIKE ?" params.append(f"%{filters['name']}%") if 'limit' in filters: sql += " limit ?" params.append(filters["limit"]) cursor.execute(sql, params) #return [row[0] for row in cursor.fetchall()] # 链接使用小写 return [{'href': row[0], 'id': row[1]} for row in cursor.fetchall()] except sqlite3.Error as e: logging.error(f"查询 href 失败: {e}") return None # 插入或更新类别 """ def insert_or_update_tags(name, href): try: if href in cached_tags: return cached_tags[href]['id'] cursor.execute(""" INSERT INTO javdb_tags (name, href, updated_at) VALUES (?, ? , datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET name = excluded.name, updated_at = datetime('now', 'localtime') """, (name, href)) conn.commit() cursor.execute("SELECT id, name, href FROM javdb_tags") for row in cursor.fetchall(): cached_tags[row[2]] = {'id': row[0], 'name':row[2]} if href in cached_tags: dist_id = cached_tags[href]['id'] logging.debug(f"insert/update tags succ. id: {dist_id}, name: {name}") return dist_id else: return None except sqlite3.Error as e: conn.rollback() logging.error(f"数据库错误: {e}") return None # 查询tags def query_tags(href, name): global cached_tags try: if href not in cached_tags: cursor.execute("SELECT id, name, href FROM javdb_tags") for row in cursor.fetchall(): cached_tags[row[2]] = {'id': row[0], 'name':row[2]} if href in cached_tags: return cached_tags[href]['id'], cached_tags[href]['name'] except sqlite3.Error as e: logging.error(f"查询失败: {e}") return 0, name # 插入影片和tags的关联数据 def insert_movie_tags( movie_id, tag_id, tags=''): try: cursor.execute(""" INSERT INTO javdb_movies_tags (movie_id, tag_id, tags, updated_at) VALUES (?, ?, ?, datetime('now', 'localtime')) ON CONFLICT(tag_id, movie_id) DO UPDATE SET tags=excluded.tags, updated_at=datetime('now', 'localtime') """, (movie_id, tag_id, tags) ) conn.commit() #logging.debug(f'insert one performer_movie, performer_id: {performer_id}, movie_id: {movie_id}') return movie_id except Exception as e: conn.rollback() logging.error("Error inserting movie: %s", e) return None # """插入或更新电影数据""" def insert_or_update_movie(movie): try: # 获取相关 ID makers_id = get_id_by_href('javdb_makers', movie['maker_link']) if movie['maker_link'] else None series_id = get_id_by_href('javdb_series', movie['series_link']) if movie['series_link'] else None pub_id = get_id_by_href('javdb_publishers', movie['pub_link']) if movie['pub_link'] else None # 如果不存在,插入 if makers_id is None and movie['maker_link']: makers_id = insert_or_update_makers({'name' : movie.get('maker_name', ''), 'href' : movie.get('maker_link', '')}, caller='movie') if series_id is None and movie['series_link']: series_id = insert_or_update_series({'name' : movie.get('series_name', ''), 'href' : movie.get('series_link', '')}, caller='movie') if pub_id is None and movie['pub_link']: pub_id = insert_or_update_publishers({'name' : movie.get('pub_name', ''), 'href' : movie.get('pub_link', '')}, caller='movie') cursor.execute(""" INSERT INTO javdb_movies (href, title, cover_url, serial_number, release_date, duration, maker_id, series_id, pub_id, is_full_data, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET title=excluded.title, cover_url=excluded.cover_url, serial_number=excluded.serial_number, release_date=excluded.release_date, duration=excluded.duration, maker_id=excluded.maker_id, series_id=excluded.series_id, pub_id=excluded.pub_id, is_full_data=1, updated_at=datetime('now', 'localtime') """, (movie['href'], movie['title'], movie['cover_url'], movie['serial_number'], movie['release_date'], movie['duration'], makers_id, series_id, pub_id)) conn.commit() # 获取插入的 movie_id movie_id = get_id_by_href('javdb_movies', movie['href']) if movie_id is None: return None logging.debug(f"insert one move, id: {movie_id}, title: {movie['title']}, href: {movie['href']}") # 插入 performers_movies 关系表 for performer in movie.get('actors', []): performer_id = get_id_by_href('javdb_actors', performer['href']) # 如果演员不存在,先插入 if performer_id is None: performer_id = insert_actor_index(performer['name'], performer['href'], from_movie_list=1) logging.debug(f"insert new perfomer. perfomer_id: {performer_id}, name:{performer['name']}") if performer_id: tmp_id = insert_actor_movie(performer_id, movie_id) if tmp_id: logging.debug(f"insert one perfomer_movie. perfomer_id: {performer_id}, movie_id:{movie_id}") else: logging.debug(f"insert perfomer_movie failed. perfomer_id: {performer_id}, movie_id:{movie_id}") else: logging.warning(f"insert perfomer failed. name: {performer['name']}, href: {performer['href']}") # 插入 tags 表 for tag in movie.get('tags', []): tag_name = tag.get('name', '') tag_href = tag.get('href', '') tag_id = insert_or_update_tags(tag_name, tag_href) if tag_id: logging.debug(f"insert one tags. tag_id: {tag_id}, name: {tag_name}") tmp_id = insert_movie_tags(movie_id=movie_id, tag_id=tag_id, tags=tag_name) if tmp_id: logging.debug(f"insert one movie_tag. movie_id: {movie_id}, tag_id: {tag_id}, name: {tag_name}") else: logging.warning(f"insert one movie_tag error. movie_id: {movie_id}, tag_id: {tag_id}, name: {tag_name}") else: logging.warning(f"insert tags error. name:{tag_name}, href: {tag_href}") return movie_id except Exception as e: conn.rollback() logging.error("Error inserting movie: %s", e) return None # """插入或更新电影数据(异常url的处理,比如404链接)""" def insert_or_update_movie_404(title, href, is_full_data=1): try: # 插入或更新电影信息 cursor.execute( """ INSERT INTO javdb_movies (title, href, is_full_data, updated_at) VALUES (?, ?, ?, datetime('now', 'localtime')) ON CONFLICT(href) DO UPDATE SET title=excluded.title, is_full_data=excluded.is_full_data, updated_at = datetime('now', 'localtime') """, (title, href, is_full_data) ) conn.commit() # 获取插入的 movie_id movie_id = get_id_by_href('javdb_movies', href) if movie_id is None: return None return movie_id except Exception as e: conn.rollback() logging.error("Error inserting movie: %s", e) return None # 删除电影数据""" def delete_movie(identifier): try: if isinstance(identifier, int): cursor.execute("DELETE FROM javdb_movies WHERE id = ?", (identifier,)) elif isinstance(identifier, str): cursor.execute("DELETE FROM javdb_movies WHERE href = ?", (identifier,)) else: logging.warning("无效的删除参数") return conn.commit() logging.info(f"Deleted movie with {identifier}") except sqlite3.Error as e: conn.rollback() logging.error("Error deleting movie: %s", e) # 查找电影数据""" def query_movies(identifier): try: if isinstance(identifier, int): cursor.execute("SELECT * FROM javdb_movies WHERE id = ?", (identifier,)) elif "http" in identifier: cursor.execute("SELECT * FROM javdb_movies WHERE href = ?", (identifier,)) else: cursor.execute("SELECT * FROM javdb_movies WHERE title LIKE ?", (f"%{identifier}%",)) movie = cursor.fetchone() if movie: cursor.execute("SELECT * FROM javdb_actors_movies WHERE performer_id = ?", (movie[0],)) performers = [row[0] for row in cursor.fetchall()] result = dict(zip([desc[0] for desc in cursor.description], performers)) result["performers"] = performers return result else: logging.warning(f"find no data: {identifier}") return None except sqlite3.Error as e: logging.error(f"查询失败: {e}") return None ''' # 按条件查询 href 列表 def query_movie_hrefs_old(**filters): try: sql = "SELECT href, title, id FROM javdb_movies WHERE 1=1" params = [] if "id" in filters: sql += " AND id = ?" params.append(filters["id"]) if "href" in filters: sql += " AND href = ?" params.append(filters["href"]) if "title" in filters: sql += " AND title LIKE ?" params.append(f"%{filters['title']}%") if "is_full_data" in filters: sql += " AND is_full_data = ?" params.append(filters["is_full_data"]) if "from_actor_list" in filters: sql += " AND from_actor_list = ?" params.append(filters["from_actor_list"]) if "is_full_data_in" in filters: values = filters["is_full_data_in"] if values: placeholders = ", ".join(["?"] * len(values)) sql += f" AND is_full_data IN ({placeholders})" params.extend(values) if "is_full_data_not_in" in filters: values = filters["is_full_data_not_in"] if values: placeholders = ", ".join(["?"] * len(values)) sql += f" AND is_full_data NOT IN ({placeholders})" params.extend(values) if "before_updated_at" in filters: sql += " AND updated_at <= ?" params.append(filters["before_updated_at"]) if "after_updated_at" in filters: sql += " AND updated_at >= ?" params.append(filters["after_updated_at"]) if "start_id" in filters: sql += " AND id > ?" params.append(filters["start_id"]) if "order_by" in filters: sql += " order by ?" params.append(filters["order_by"]) if 'limit' in filters: sql += " limit ?" params.append(filters["limit"]) cursor.execute(sql, params) #return [row[0].lower() for row in cursor.fetchall()] # 链接使用小写 return [{'href': row[0], 'title': row[1], 'id':row[2]} for row in cursor.fetchall()] except sqlite3.Error as e: logging.error(f"查询 href 失败: {e}") return [] ''' # 查询 def query_movie_hrefs(**filters): try: sql = "SELECT href, title, id FROM javdb_movies WHERE 1=1" params = [] conditions = { "id": " AND id = ?", "href": " AND href = ?", "title": " AND title LIKE ?", "is_full_data": " AND is_full_data = ?", "uncensored": " AND uncensored = ?", "from_actor_list": " AND from_actor_list = ?", "before_updated_at": " AND updated_at <= ?", "after_updated_at": " AND updated_at >= ?", "start_id": " AND id > ?", } for key, condition in conditions.items(): if key in filters: sql += condition if key == "title": params.append(f"%{filters[key]}%") else: params.append(filters[key]) for key in ["is_full_data_in", "is_full_data_not_in"]: if key in filters: values = filters[key] if values: placeholders = ", ".join(["?"] * len(values)) operator = "IN" if key == "is_full_data_in" else "NOT IN" sql += f" AND is_full_data {operator} ({placeholders})" params.extend(values) if "order_by" in filters: # 注意:这里 order by 后面直接跟字段名,不能用占位符,否则会被当作字符串处理 sql += f" ORDER BY {filters['order_by']} " if 'limit' in filters: sql += " LIMIT ?" params.append(filters["limit"]) cursor.execute(sql, params) #return [row[0].lower() for row in cursor.fetchall()] # 返回小写 #return [{'href': row[0], 'name': row[1]} for row in cursor.fetchall()] return [{'href': row[0], 'title': row[1], 'id':row[2]} for row in cursor.fetchall()] except sqlite3.Error as e: logging.error(f"查询 href 失败: {e}") return None # 插入一条任务日志 def insert_task_log(): try: cursor.execute(""" INSERT INTO javdb_task_log (task_status) VALUES ('Start') """) conn.commit() task_id = cursor.lastrowid if task_id is None: return None update_task_log(task_id=task_id, task_status='Start') return task_id # 获取插入的 task_id except sqlite3.Error as e: logging.error(f"插入任务失败: {e}") return None # 更新任务日志的字段 def update_task_log_inner(task_id, **kwargs): try: fields = ", ".join(f"{key} = ?" for key in kwargs.keys()) params = list(kwargs.values()) + [task_id] sql = f"UPDATE javdb_task_log SET {fields}, updated_at = datetime('now', 'localtime') WHERE task_id = ?" cursor.execute(sql, params) conn.commit() except sqlite3.Error as e: logging.error(f"更新任务 {task_id} 失败: {e}") # 更新任务日志的字段 def update_task_log(task_id, task_status): try: # 获取 performers、studios 等表的最终行数 cursor.execute("SELECT COUNT(*) FROM javdb_actors where is_full_data=1") full_data_actors = cursor.fetchone()[0] cursor.execute("SELECT COUNT(*) FROM javdb_actors") total_actors = cursor.fetchone()[0] cursor.execute("SELECT COUNT(*) FROM javdb_movies where is_full_data=1") full_data_movies = cursor.fetchone()[0] cursor.execute("SELECT COUNT(*) FROM javdb_movies") total_movies = cursor.fetchone()[0] cursor.execute("SELECT COUNT(*) FROM javdb_makers") total_makers = cursor.fetchone()[0] cursor.execute("SELECT COUNT(*) FROM javdb_series") total_series = cursor.fetchone()[0] # 更新 task_log update_task_log_inner(task_id, full_data_actors=full_data_actors, total_actors=total_actors, full_data_movies=full_data_movies, total_movies=total_movies, total_makers=total_makers, total_series=total_series, task_status=task_status) except sqlite3.Error as e: logging.error(f"更新任务 {task_id} 失败: {e}") # 任务结束,更新字段 def finalize_task_log(task_id): try: # 更新 task_log update_task_log(task_id, task_status="Success") except sqlite3.Error as e: logging.error(f"任务 {task_id} 结束失败: {e}") # 测试代码 if __name__ == "__main__": sample_data = [ { 'name': '上原亜衣', 'href': 'https://www.javdb.com/actors/MkAX', 'pic': 'https://c0.jdbstatic.com/avatars/mk/MkAX.jpg', 'alias': ['上原亜衣', '下原舞', '早瀬クリスタル', '阿蘇山百式屏風奉行'] }, { 'name': '大橋未久', 'href': 'https://www.javdb.com/actors/21Jp', 'pic': 'https://c0.jdbstatic.com/avatars/21/21Jp.jpg', 'alias': ['大橋未久'] }, ] for actor in sample_data: insert_or_update_actor(actor) print(query_actors("name LIKE '%未久%'")) #delete_actor_by_href('https://www.javdb.com/actors/MkAX') print(query_actors())