research/stock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify some scripts.

Browse Source
This commit is contained in:
oscar
2025-03-04 16:05:47 +08:00
parent 0741ac94ad
commit 3f0a8acb6b
4 changed files with 485 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
scripts/iafd/src/sqlite_utils.py
View File

@ -18,7 +18,7 @@ def get_current_time():
def insert_or_update_performer(data):
try:
cursor.execute("""
INSERT INTO performers (href, name, gender, birthday, astrology, birthplace, years_active, ethnicity, nationality, hair_colors,
INSERT INTO iafd_performers (href, name, gender, birthday, astrology, birthplace, years_active, ethnicity, nationality, hair_colors,
eye_color, height_str, weight_str, measurements, tattoos, piercings, weight, height, movies_cnt, vixen_cnt,
blacked_cnt, tushy_cnt, x_art_cnt, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now', 'localtime'))
@ -54,17 +54,17 @@ def insert_or_update_performer(data):
))
# 获取 performer_id
cursor.execute("SELECT id FROM performers WHERE href = ?", (data["href"],))
cursor.execute("SELECT id FROM iafd_performers WHERE href = ?", (data["href"],))
performer_id = cursor.fetchone()[0]
# 删除旧的 alias
cursor.execute("DELETE FROM performer_aliases WHERE performer_id = ?", (performer_id,))
cursor.execute("DELETE FROM iafd_performer_aliases WHERE performer_id = ?", (performer_id,))
# 插入新的 alias
#for alias in data.get("performer_aka", []):
for alias in data.get("performer_aka") or []:
if alias.lower() != "no known aliases":
cursor.execute("INSERT INTO performer_aliases (performer_id, alias) VALUES (?, ?) ON CONFLICT(performer_id, alias) DO NOTHING ", (performer_id, alias))
cursor.execute("INSERT INTO iafd_performer_aliases (performer_id, alias) VALUES (?, ?) ON CONFLICT(performer_id, alias) DO NOTHING ", (performer_id, alias))
conn.commit()
logging.debug(f"成功插入/更新演员: {data['person']}")
@ -83,9 +83,9 @@ def insert_or_update_performer(data):
def delete_performer(identifier):
try:
if isinstance(identifier, int):
cursor.execute("DELETE FROM performers WHERE id = ?", (identifier,))
cursor.execute("DELETE FROM iafd_performers WHERE id = ?", (identifier,))
elif isinstance(identifier, str):
cursor.execute("DELETE FROM performers WHERE href = ?", (identifier,))
cursor.execute("DELETE FROM iafd_performers WHERE href = ?", (identifier,))
else:
logging.warning("无效的删除参数")
return
@ -100,15 +100,15 @@ def delete_performer(identifier):
def query_performer(identifier):
try:
if isinstance(identifier, int):
cursor.execute("SELECT * FROM performers WHERE id = ?", (identifier,))
cursor.execute("SELECT * FROM iafd_performers WHERE id = ?", (identifier,))
elif "http" in identifier:
cursor.execute("SELECT * FROM performers WHERE href = ?", (identifier,))
cursor.execute("SELECT * FROM iafd_performers WHERE href = ?", (identifier,))
else:
cursor.execute("SELECT * FROM performers WHERE name LIKE ?", (f"%{identifier}%",))
cursor.execute("SELECT * FROM iafd_performers WHERE name LIKE ?", (f"%{identifier}%",))
performer = cursor.fetchone()
if performer:
cursor.execute("SELECT alias FROM performer_aliases WHERE performer_id = ?", (performer[0],))
cursor.execute("SELECT alias FROM iafd_performer_aliases WHERE performer_id = ?", (performer[0],))
aliases = [row[0] for row in cursor.fetchall()]
result = dict(zip([desc[0] for desc in cursor.description], performer))
result["performer_aka"] = aliases
@ -124,7 +124,7 @@ def query_performer(identifier):
# 按条件查询 href 列表
def query_performer_hrefs(**filters):
try:
sql = "SELECT href FROM performers WHERE 1=1"
sql = "SELECT href FROM iafd_performers WHERE 1=1"
params = []
if "id" in filters:
@ -149,7 +149,7 @@ def query_performer_hrefs(**filters):
def insert_or_update_distributor(data):
try:
cursor.execute("""
INSERT INTO distributors (name, href, updated_at)
INSERT INTO iafd_distributors (name, href, updated_at)
VALUES (?, ? , datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
name = excluded.name,
@ -158,7 +158,7 @@ def insert_or_update_distributor(data):
conn.commit()
# 获取 performer_id
cursor.execute("SELECT id FROM distributors WHERE href = ?", (data["href"],))
cursor.execute("SELECT id FROM iafd_distributors WHERE href = ?", (data["href"],))
dist_id = cursor.fetchone()[0]
if dist_id:
logging.debug(f"成功插入/更新发行商: {data['name']}")
@ -174,9 +174,9 @@ def insert_or_update_distributor(data):
def delete_distributor(identifier):
try:
if isinstance(identifier, int):
cursor.execute("DELETE FROM distributors WHERE id = ?", (identifier,))
cursor.execute("DELETE FROM iafd_distributors WHERE id = ?", (identifier,))
elif isinstance(identifier, str):
cursor.execute("DELETE FROM distributors WHERE name = ?", (identifier,))
cursor.execute("DELETE FROM iafd_distributors WHERE name = ?", (identifier,))
conn.commit()
logging.info(f"成功删除发行商: {identifier}")
except sqlite3.Error as e:
@ -187,9 +187,9 @@ def delete_distributor(identifier):
def query_distributor(identifier):
try:
if isinstance(identifier, int):
cursor.execute("SELECT * FROM distributors WHERE id = ?", (identifier,))
cursor.execute("SELECT * FROM iafd_distributors WHERE id = ?", (identifier,))
else:
cursor.execute("SELECT * FROM distributors WHERE name LIKE ?", (f"%{identifier}%",))
cursor.execute("SELECT * FROM iafd_distributors WHERE name LIKE ?", (f"%{identifier}%",))
distributor = cursor.fetchone()
if distributor:
@ -204,7 +204,7 @@ def query_distributor(identifier):
# 按条件查询 href 列表
def query_distributor_hrefs(**filters):
try:
sql = "SELECT href FROM distributors WHERE 1=1"
sql = "SELECT href FROM iafd_distributors WHERE 1=1"
params = []
if "id" in filters:
@ -228,7 +228,7 @@ def query_distributor_hrefs(**filters):
def insert_or_update_studio(data):
try:
cursor.execute("""
INSERT INTO studios (name, href, updated_at)
INSERT INTO iafd_studios (name, href, updated_at)
VALUES (?, ?, datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
name = excluded.name,
@ -237,7 +237,7 @@ def insert_or_update_studio(data):
conn.commit()
# 获取 performer_id
cursor.execute("SELECT id FROM studios WHERE href = ?", (data["href"],))
cursor.execute("SELECT id FROM iafd_studios WHERE href = ?", (data["href"],))
stu_id = cursor.fetchone()[0]
if stu_id:
logging.debug(f"成功插入/更新发行商: {data['name']}")
@ -253,9 +253,9 @@ def insert_or_update_studio(data):
def delete_studio(identifier):
try:
if isinstance(identifier, int):
cursor.execute("DELETE FROM studios WHERE id = ?", (identifier,))
cursor.execute("DELETE FROM iafd_studios WHERE id = ?", (identifier,))
elif isinstance(identifier, str):
cursor.execute("DELETE FROM studios WHERE name = ?", (identifier,))
cursor.execute("DELETE FROM iafd_studios WHERE name = ?", (identifier,))
conn.commit()
logging.info(f"成功删除制作公司: {identifier}")
except sqlite3.Error as e:
@ -266,9 +266,9 @@ def delete_studio(identifier):
def query_studio(identifier):
try:
if isinstance(identifier, int):
cursor.execute("SELECT * FROM studios WHERE id = ?", (identifier,))
cursor.execute("SELECT * FROM iafd_studios WHERE id = ?", (identifier,))
else:
cursor.execute("SELECT * FROM studios WHERE name LIKE ?", (f"%{identifier}%",))
cursor.execute("SELECT * FROM iafd_studios WHERE name LIKE ?", (f"%{identifier}%",))
studio = cursor.fetchone()
if studio:
@ -283,7 +283,7 @@ def query_studio(identifier):
# 按条件查询 href 列表
def query_studio_hrefs(**filters):
try:
sql = "SELECT href FROM studios WHERE 1=1"
sql = "SELECT href FROM iafd_studios WHERE 1=1"
params = []
if "id" in filters:
@ -313,14 +313,14 @@ def get_id_by_href(table: str, href: str) -> int:
def insert_or_update_movie(movie_data):
try:
# 获取相关 ID
distributor_id = get_id_by_href('distributors', movie_data['DistributorHref'])
studio_id = get_id_by_href('studios', movie_data['StudioHref'])
director_id = get_id_by_href('performers', movie_data['DirectorHref'])
distributor_id = get_id_by_href('iafd_distributors', movie_data['DistributorHref'])
studio_id = get_id_by_href('iafd_studios', movie_data['StudioHref'])
director_id = get_id_by_href('iafd_performers', movie_data['DirectorHref'])
# 插入或更新电影信息
cursor.execute(
"""
INSERT INTO movies (title, minutes, distributor_id, studio_id, release_date, added_to_IAFD_date,
INSERT INTO iafd_movies (title, minutes, distributor_id, studio_id, release_date, added_to_IAFD_date,
all_girl, all_male, compilation, webscene, director_id, href, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
@ -338,17 +338,17 @@ def insert_or_update_movie(movie_data):
logging.debug("Movie inserted/updated: %s", movie_data['title'])
# 获取插入的 movie_id
cursor.execute("SELECT id FROM movies WHERE href = ?", (movie_data['href'],))
cursor.execute("SELECT id FROM iafd_movies WHERE href = ?", (movie_data['href'],))
movie_id = cursor.fetchone()[0]
# 插入 performers_movies 关系表
for performer in movie_data.get('Performers', []):
performer_id = get_id_by_href('performers', performer['href'])
performer_id = get_id_by_href('iafd_performers', performer['href'])
if performer_id:
notes = '|'.join(performer['tags'])
cursor.execute(
"""
INSERT INTO performers_movies (performer_id, movie_id, role, notes)
INSERT INTO iafd_performers_movies (performer_id, movie_id, role, notes)
VALUES (?, ?, ?, ?)
ON CONFLICT(movie_id, performer_id) DO UPDATE SET notes=excluded.notes
""",
@ -360,11 +360,11 @@ def insert_or_update_movie(movie_data):
# 插入 movies_appers_in 表
for appears in movie_data.get("AppearsIn", []):
appears_in_id = get_id_by_href('movies', appears['href'])
appears_in_id = get_id_by_href('iafd_movies', appears['href'])
if appears_in_id:
appears_in_id = appears_in_id[0]
cursor.execute("""
INSERT INTO movies_appers_in (movie_id, appears_in_id, gradation, notes)
INSERT INTO iafd_movies_appers_in (movie_id, appears_in_id, gradation, notes)
VALUES (?, ?, ?, ?)
ON CONFLICT(movie_id, appears_in_id) DO NOTHING
""", (movie_id, appears_in_id, 1, appears["title"]))
@ -383,9 +383,9 @@ def insert_or_update_movie(movie_data):
def delete_movie(identifier):
try:
if isinstance(identifier, int):
cursor.execute("DELETE FROM movies WHERE id = ?", (identifier,))
cursor.execute("DELETE FROM iafd_movies WHERE id = ?", (identifier,))
elif isinstance(identifier, str):
cursor.execute("DELETE FROM movies WHERE href = ?", (identifier,))
cursor.execute("DELETE FROM iafd_movies WHERE href = ?", (identifier,))
else:
logging.warning("无效的删除参数")
return
@ -400,15 +400,15 @@ def delete_movie(identifier):
def query_movies(identifier):
try:
if isinstance(identifier, int):
cursor.execute("SELECT * FROM movies WHERE id = ?", (identifier,))
cursor.execute("SELECT * FROM iafd_movies WHERE id = ?", (identifier,))
elif "http" in identifier:
cursor.execute("SELECT * FROM movies WHERE href = ?", (identifier,))
cursor.execute("SELECT * FROM iafd_movies WHERE href = ?", (identifier,))
else:
cursor.execute("SELECT * FROM movies WHERE title LIKE ?", (f"%{identifier}%",))
cursor.execute("SELECT * FROM iafd_movies WHERE title LIKE ?", (f"%{identifier}%",))
movie = cursor.fetchone()
if movie:
cursor.execute("SELECT * FROM performer_movie WHERE performer_id = ?", (movie[0],))
cursor.execute("SELECT * FROM iafd_performers_movies WHERE performer_id = ?", (movie[0],))
performers = [row[0] for row in cursor.fetchall()]
result = dict(zip([desc[0] for desc in cursor.description], performers))
result["performers"] = performers
@ -424,7 +424,7 @@ def query_movies(identifier):
# 按条件查询 href 列表
def query_movie_hrefs(**filters):
try:
sql = "SELECT href FROM movies WHERE 1=1"
sql = "SELECT href FROM iafd_movies WHERE 1=1"
params = []
if "id" in filters:
@ -448,7 +448,7 @@ def query_movie_hrefs(**filters):
def insert_task_log():
try:
cursor.execute("""
INSERT INTO task_log (task_status) VALUES ('Start')
INSERT INTO iafd_task_log (task_status) VALUES ('Start')
""")
conn.commit()
return cursor.lastrowid # 获取插入的 task_id
@ -462,7 +462,7 @@ def update_task_log(task_id, **kwargs):
fields = ", ".join(f"{key} = ?" for key in kwargs.keys())
params = list(kwargs.values()) + [task_id]
sql = f"UPDATE task_log SET {fields}, updated_at = datetime('now', 'localtime') WHERE task_id = ?"
sql = f"UPDATE iafd_task_log SET {fields}, updated_at = datetime('now', 'localtime') WHERE task_id = ?"
cursor.execute(sql, params)
conn.commit()
except sqlite3.Error as e:
@ -472,16 +472,16 @@ def update_task_log(task_id, **kwargs):
def finalize_task_log(task_id):
try:
# 获取 performers、studios 等表的最终行数
cursor.execute("SELECT COUNT(*) FROM performers")
cursor.execute("SELECT COUNT(*) FROM iafd_performers")
after_performers = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM movies")
cursor.execute("SELECT COUNT(*) FROM iafd_movies")
after_movies = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM distributors")
cursor.execute("SELECT COUNT(*) FROM iafd_distributors")
after_distributors = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM studios")
cursor.execute("SELECT COUNT(*) FROM iafd_studios")
after_studios = cursor.fetchone()[0]
# 更新 task_log