research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-04-25 10:08:24 +08:00
parent c43b1c4ee0
commit 806e2d98c2
2 changed files with 151 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
javdb/src/fetch.py
View File

@ -17,6 +17,7 @@ force = False
skip_local = False
from_actor = False
abnormal_only = False
fast_mode = False
# 获取演员列表
def fetch_actor_list():
@ -51,7 +52,7 @@ def fetch_makers_list():
if list_data :
# 写入数据库
for row in list_data:
maker_id = db_tools.insert_or_update_makers(row)
maker_id = db_tools.insert_or_update_makers(row, caller='list')
if maker_id:
logging.debug(f'insert maker to db. maker_id:{maker_id}, name: {row['name']}, href:{row['href']}')
else:
@ -74,7 +75,7 @@ def fetch_series_list():
if list_data :
# 写入数据库
for row in list_data:
maker_id = db_tools.insert_or_update_series(row)
maker_id = db_tools.insert_or_update_series(row, caller='list')
if maker_id:
logging.debug(f'insert series to db. maker_id:{maker_id}, name: {row['name']}, href:{row['href']}')
else:
@ -89,7 +90,11 @@ def fetch_series_list():
# 更新makers列表中的影片信息
def fetch_movies_by_maker():
url_list = db_tools.query_maker_hrefs()
if fast_mode:
url_list = db_tools.query_maker_hrefs(from_list=1)
else:
url_list = db_tools.query_maker_hrefs()
if debug:
url_list = db_tools.query_maker_hrefs(name='muramura')
for url in url_list:
@ -120,7 +125,11 @@ def fetch_movies_by_maker():
# 更新series列表中的影片信息
def fetch_movies_by_series():
url_list = db_tools.query_series_hrefs()
if fast_mode:
url_list = db_tools.query_series_hrefs(from_list=1)
else:
url_list = db_tools.query_series_hrefs()
if debug:
url_list = db_tools.query_series_hrefs(name='10musume')
for url in url_list:
@ -308,25 +317,7 @@ function_map = {
}
# 主函数
def main(cmd, args_debug, args_force, args_skip_local, args_from_actor, args_abnormal_only):
global debug
debug = args_debug
if debug:
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
global force
force = args_force
global skip_local
skip_local = args_skip_local
global from_actor
from_actor = args_from_actor
global abnormal_only
abnormal_only = args_abnormal_only
def main(cmd):
# 开启任务
task_id = db_tools.insert_task_log()
if task_id is None:
@ -359,6 +350,29 @@ def main(cmd, args_debug, args_force, args_skip_local, args_from_actor, args_abn
# TODO:
# 1,
# 设置环境变量
def set_env(args):
global debug
debug = args.debug
if debug:
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
global force
force = args.force
global skip_local
skip_local = args.skip_local
global from_actor
from_actor = args.from_actor
global abnormal_only
abnormal_only = args.abnormal_only
global fast_mode
fast_mode = args.fast_mode
if __name__ == "__main__":
# 命令行参数处理
keys_str = ",".join(function_map.keys())
@ -370,6 +384,8 @@ if __name__ == "__main__":
parser.add_argument('--skip_local', action='store_true', help='skip if cached html (true for skip)')
parser.add_argument('--from_actor', action='store_true', help='只遍历来自 actor_list 的 演员或者影片 (在force模式下有效)')
parser.add_argument('--abnormal_only', action='store_true', help='只遍历异常URL(404或者需要登陆查看等) 的 演员或影片 (在force模式下有效)')
parser.add_argument('--fast_mode', action='store_true', help='只遍历所有 uncensored 的 makers 和 series ')
args = parser.parse_args()
main(args.cmd, args.debug, args.force, args.skip_local, args.from_actor, args.abnormal_only)
set_env(args)
main(args.cmd)

136
javdb/src/sqlite_utils.py
View File

@ -9,6 +9,8 @@ DB_PATH = f"{config.global_share_data_dir}/sqlite/shared.db" # 替换为你的
conn = sqlite3.connect(DB_PATH, check_same_thread=False)
cursor = conn.cursor()
cached_tags = {}
# """从指定表中通过 href 查找 id"""
def get_id_by_href(table: str, href: str) -> int:
if href is None:
@ -269,16 +271,31 @@ def query_actors(**filters):
# 插入或更新发行商 """
def insert_or_update_makers(data):
def insert_or_update_makers(data, caller='list'):
try:
cursor.execute("""
INSERT INTO javdb_makers (name, href, updated_at)
VALUES (?, ? , datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
name = excluded.name,
updated_at = datetime('now', 'localtime')
""", (data["name"], data["href"]))
conn.commit()
if caller == 'list':
cursor.execute("""
INSERT INTO javdb_makers (name, href, from_list, updated_at)
VALUES (?, ? , 1, datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
name = excluded.name,
from_list = 1,
updated_at = datetime('now', 'localtime')
""", (data["name"], data["href"]))
conn.commit()
elif caller == 'movie':
cursor.execute("""
INSERT INTO javdb_makers (name, href, from_movie_list, updated_at)
VALUES (?, ? , 1, datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
name = excluded.name,
from_movie_list = 1,
updated_at = datetime('now', 'localtime')
""", (data["name"], data["href"]))
conn.commit()
else:
logging.warning(f"unexpected caller: {caller}")
return None
# 获取 performer_id
cursor.execute("SELECT id FROM javdb_makers WHERE href = ?", (data["href"],))
@ -333,6 +350,9 @@ def query_maker_hrefs(**filters):
if "id" in filters:
sql += " AND id = ?"
params.append(filters["id"])
if "from_list" in filters:
sql += " AND from_list = ?"
params.append(filters["from_list"])
if "url" in filters:
sql += " AND href = ?"
params.append(filters["href"])
@ -348,16 +368,31 @@ def query_maker_hrefs(**filters):
return None
# """ 插入或更新制作公司 """
def insert_or_update_series(data):
def insert_or_update_series(data, caller='list'):
try:
cursor.execute("""
INSERT INTO javdb_series (name, href, updated_at)
VALUES (?, ?, datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
name = excluded.name,
updated_at = datetime('now', 'localtime')
""", (data["name"], data["href"]))
conn.commit()
if caller == 'list':
cursor.execute("""
INSERT INTO javdb_series (name, href, from_list, updated_at)
VALUES (?, ? , 1, datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
name = excluded.name,
from_list = 1,
updated_at = datetime('now', 'localtime')
""", (data["name"], data["href"]))
conn.commit()
elif caller == 'movie':
cursor.execute("""
INSERT INTO javdb_series (name, href, from_movie_list, updated_at)
VALUES (?, ? , 1, datetime('now', 'localtime'))
ON CONFLICT(href) DO UPDATE SET
name = excluded.name,
from_movie_list = 1,
updated_at = datetime('now', 'localtime')
""", (data["name"], data["href"]))
conn.commit()
else:
logging.warning(f"unexpected caller: {caller}")
return None
# 获取 performer_id
cursor.execute("SELECT id FROM javdb_series WHERE href = ?", (data["href"],))
@ -412,6 +447,9 @@ def query_series_hrefs(**filters):
if "id" in filters:
sql += " AND id = ?"
params.append(filters["id"])
if "from_list" in filters:
sql += " AND from_list = ?"
params.append(filters["from_list"])
if "href" in filters:
sql += " AND href = ?"
params.append(filters["href"])
@ -430,6 +468,9 @@ def query_series_hrefs(**filters):
# 插入或更新类别 """
def insert_or_update_tags(name, href):
try:
if href in cached_tags:
return cached_tags[href]['id']
cursor.execute("""
INSERT INTO javdb_tags (name, href, updated_at)
VALUES (?, ? , datetime('now', 'localtime'))
@ -439,10 +480,12 @@ def insert_or_update_tags(name, href):
""", (name, href))
conn.commit()
# 获取 performer_id
cursor.execute("SELECT id FROM javdb_tags WHERE href = ?", (href,))
dist_id = cursor.fetchone()[0]
if dist_id:
cursor.execute("SELECT id, name, href FROM javdb_tags")
for row in cursor.fetchall():
cached_tags[row[2]] = {'id': row[0], 'name':row[2]}
if href in cached_tags:
dist_id = cached_tags[href]['id']
logging.debug(f"insert/update tags succ. id: {dist_id}, name: {name}")
return dist_id
else:
@ -452,6 +495,42 @@ def insert_or_update_tags(name, href):
logging.error(f"数据库错误: {e}")
return None
# 查询tags
def query_tags(href, name):
global cached_tags
try:
if href not in cached_tags:
cursor.execute("SELECT id, name, href FROM javdb_tags")
for row in cursor.fetchall():
cached_tags[row[2]] = {'id': row[0], 'name':row[2]}
if href in cached_tags:
return cached_tags[href]['id'], cached_tags[href]['name']
except sqlite3.Error as e:
logging.error(f"查询失败: {e}")
return 0, name
# 插入影片和tags的关联数据
def insert_movie_tags( movie_id, tag_id, tags=''):
try:
cursor.execute("""
INSERT INTO javdb_movies_tags (movie_id, tag_id, tags, updated_at)
VALUES (?, ?, ?, datetime('now', 'localtime'))
ON CONFLICT(tag_id, movie_id) DO UPDATE SET tags=excluded.tags, updated_at=datetime('now', 'localtime')
""",
(movie_id, tag_id, tags)
)
conn.commit()
#logging.debug(f'insert one performer_movie, performer_id: {performer_id}, movie_id: {movie_id}')
return movie_id
except Exception as e:
conn.rollback()
logging.error("Error inserting movie: %s", e)
return None
# """插入或更新电影数据"""
def insert_or_update_movie(movie):
try:
@ -461,9 +540,9 @@ def insert_or_update_movie(movie):
# 如果不存在，插入
if makers_id is None and movie['maker_link']:
makers_id = insert_or_update_makers({'name' : movie.get('maker_name', ''), 'href' : movie.get('maker_link', '')})
makers_id = insert_or_update_makers({'name' : movie.get('maker_name', ''), 'href' : movie.get('maker_link', '')}, caller='movie')
if series_id is None and movie['series_link']:
series_id = insert_or_update_series({'name' : movie.get('series_name', ''), 'href' : movie.get('series_link', '')})
series_id = insert_or_update_series({'name' : movie.get('series_name', ''), 'href' : movie.get('series_link', '')}, caller='movie')
cursor.execute("""
INSERT INTO javdb_movies (href, title, cover_url, serial_number, release_date, duration,
@ -513,7 +592,14 @@ def insert_or_update_movie(movie):
tag_href = tag.get('href', '')
tag_id = insert_or_update_tags(tag_name, tag_href)
if tag_id:
logging.debug(f"insert one tags. tag_id: {tag_id}, name:{tag_name}")
logging.debug(f"insert one tags. tag_id: {tag_id}, name: {tag_name}")
tmp_id = insert_movie_tags(movie_id=movie_id, tag_id=tag_id, tags=tag_name)
if tmp_id:
logging.debug(f"insert one movie_tag. movie_id: {movie_id}, tag_id: {tag_id}, name: {tag_name}")
else:
logging.warning(f"insert one movie_tag error. movie_id: {movie_id}, tag_id: {tag_id}, name: {tag_name}")
else:
logging.warning(f"insert tags error. name:{tag_name}, href: {tag_href}")
return movie_id