research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-04-25 10:30:25 +08:00
parent 806e2d98c2
commit 4c80e72a98
2 changed files with 28 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
javdb/src/fetch.py
View File

@ -97,7 +97,9 @@ def fetch_movies_by_maker():
if debug: if debug:
url_list = db_tools.query_maker_hrefs(name='muramura') url_list = db_tools.query_maker_hrefs(name='muramura')
for url in url_list: for row in url_list:
url = row['href']
row_id = row['id']
# 去掉可下载的标志（如果有） # 去掉可下载的标志（如果有）
next_url = utils.remove_url_query(url) next_url = utils.remove_url_query(url)
while next_url: while next_url:
@ -107,7 +109,7 @@ def fetch_movies_by_maker():
list_data, next_url = scraper.parse_maker_detail(soup, next_url) list_data, next_url = scraper.parse_maker_detail(soup, next_url)
if list_data: if list_data:
for movie in list_data: for movie in list_data:
tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], from_movie_makers=1) tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], from_movie_makers=1, maker_id=row_id)
if tmp_id: if tmp_id:
logging.debug(f'insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}') logging.debug(f'insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}')
else: else:
@ -132,7 +134,9 @@ def fetch_movies_by_series():
if debug: if debug:
url_list = db_tools.query_series_hrefs(name='10musume') url_list = db_tools.query_series_hrefs(name='10musume')
for url in url_list: for row in url_list:
url = row['href']
row_id = row['id']
# 去掉可下载的标志（如果有） # 去掉可下载的标志（如果有）
next_url = utils.remove_url_query(url) next_url = utils.remove_url_query(url)
while next_url: while next_url:
@ -142,7 +146,7 @@ def fetch_movies_by_series():
list_data, next_url = scraper.parse_series_detail(soup, next_url) list_data, next_url = scraper.parse_series_detail(soup, next_url)
if list_data: if list_data:
for movie in list_data: for movie in list_data:
tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], from_movie_series=1) tmp_id = db_tools.insert_movie_index(title=movie['title'], href=movie['href'], from_movie_series=1, series_id=row_id)
if tmp_id: if tmp_id:
logging.debug(f'insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}') logging.debug(f'insert one movie index to db. movie_id: {tmp_id}, title: {movie['title']}, href: {movie['href']}')
else: else:
@ -317,14 +321,14 @@ function_map = {
} }
# 主函数 # 主函数
def main(cmd): def main(cmd, args):
# 开启任务 # 开启任务
task_id = db_tools.insert_task_log() task_id = db_tools.insert_task_log()
if task_id is None: if task_id is None:
logging.warning(f'insert task log error.') logging.warning(f'insert task log error.')
return None return None
logging.info(f'running task. id: {task_id}, debug: {debug}, force: {force}, skip_local: {skip_local}, cmd: {cmd}') logging.info(f"running task. id: {task_id}, args: {args}")
# 执行指定的函数 # 执行指定的函数
if cmd: if cmd:
@ -388,4 +392,4 @@ if __name__ == "__main__":
args = parser.parse_args() args = parser.parse_args()
set_env(args) set_env(args)
main(args.cmd) main(args.cmd, args)

28
javdb/src/sqlite_utils.py
View File

@ -64,19 +64,21 @@ def insert_actor_index(name, href, from_actor_list=None, from_movie_list=None):
logging.error(f"未知错误: {e}") logging.error(f"未知错误: {e}")
return None return None
def insert_movie_index(title, href, from_actor_list=None, from_movie_makers=None, from_movie_series=None): def insert_movie_index(title, href, from_actor_list=None, from_movie_makers=None, from_movie_series=None, maker_id=None, series_id=None):
try: try:
# **先检查数据库中是否已有该电影** # **先检查数据库中是否已有该电影**
cursor.execute("SELECT id, from_actor_list, from_movie_makers, from_movie_series FROM javdb_movies WHERE href = ?", (href,)) cursor.execute("SELECT id, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id FROM javdb_movies WHERE href = ?", (href,))
existing_movie = cursor.fetchone() existing_movie = cursor.fetchone()
if existing_movie: # **如果电影已存在** if existing_movie: # **如果电影已存在**
movie_id, existing_actor, existing_maker, existing_series = existing_movie movie_id, existing_actor, existing_maker, existing_series, existing_maker_id, existing_series_id = existing_movie
# **如果没有传入值，就用原来的值** # **如果没有传入值，就用原来的值**
from_actor_list = from_actor_list if from_actor_list is not None else existing_actor from_actor_list = from_actor_list if from_actor_list is not None else existing_actor
from_movie_makers = from_movie_makers if from_movie_makers is not None else existing_maker from_movie_makers = from_movie_makers if from_movie_makers is not None else existing_maker
from_movie_series = from_movie_series if from_movie_series is not None else existing_series from_movie_series = from_movie_series if from_movie_series is not None else existing_series
maker_id = maker_id if maker_id is not None else existing_maker_id
series_id = series_id if series_id is not None else existing_series_id
cursor.execute(""" cursor.execute("""
UPDATE javdb_movies UPDATE javdb_movies
@ -84,14 +86,16 @@ def insert_movie_index(title, href, from_actor_list=None, from_movie_makers=None
from_actor_list = ?, from_actor_list = ?,
from_movie_makers = ?, from_movie_makers = ?,
from_movie_series = ?, from_movie_series = ?,
maker_id = ?,
series_id = ?,
updated_at = datetime('now', 'localtime') updated_at = datetime('now', 'localtime')
WHERE href = ? WHERE href = ?
""", (title, from_actor_list, from_movie_makers, from_movie_series, href)) """, (title, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id, href))
else: # **如果电影不存在，插入** else: # **如果电影不存在，插入**
cursor.execute(""" cursor.execute("""
INSERT INTO javdb_movies (title, href, from_actor_list, from_movie_makers, from_movie_series) INSERT INTO javdb_movies (title, href, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id)
VALUES (?, ?, COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0)) VALUES (?, ?, COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0), COALESCE(?, 0))
""", (title, href, from_actor_list, from_movie_makers, from_movie_series)) """, (title, href, from_actor_list, from_movie_makers, from_movie_series, maker_id, series_id))
conn.commit() conn.commit()
@ -344,7 +348,7 @@ def query_maker(identifier):
# 按条件查询 href 列表 # 按条件查询 href 列表
def query_maker_hrefs(**filters): def query_maker_hrefs(**filters):
try: try:
sql = "SELECT href FROM javdb_makers WHERE 1=1" sql = "SELECT href, id FROM javdb_makers WHERE 1=1"
params = [] params = []
if "id" in filters: if "id" in filters:
@ -361,7 +365,8 @@ def query_maker_hrefs(**filters):
params.append(f"%{filters['name']}%") params.append(f"%{filters['name']}%")
cursor.execute(sql, params) cursor.execute(sql, params)
return [row[0] for row in cursor.fetchall()] # 链接使用小写 #return [row[0] for row in cursor.fetchall()] # 链接使用小写
return [{'href': row[0], 'id': row[1]} for row in cursor.fetchall()]
except sqlite3.Error as e: except sqlite3.Error as e:
logging.error(f"查询 href 失败: {e}") logging.error(f"查询 href 失败: {e}")
@ -441,7 +446,7 @@ def query_series(identifier):
# 按条件查询 href 列表 # 按条件查询 href 列表
def query_series_hrefs(**filters): def query_series_hrefs(**filters):
try: try:
sql = "SELECT href FROM javdb_series WHERE 1=1" sql = "SELECT href, id FROM javdb_series WHERE 1=1"
params = [] params = []
if "id" in filters: if "id" in filters:
@ -458,7 +463,8 @@ def query_series_hrefs(**filters):
params.append(f"%{filters['name']}%") params.append(f"%{filters['name']}%")
cursor.execute(sql, params) cursor.execute(sql, params)
return [row[0] for row in cursor.fetchall()] # 链接使用小写 #return [row[0] for row in cursor.fetchall()] # 链接使用小写
return [{'href': row[0], 'id': row[1]} for row in cursor.fetchall()]
except sqlite3.Error as e: except sqlite3.Error as e:
logging.error(f"查询 href 失败: {e}") logging.error(f"查询 href 失败: {e}")