research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-04-24 14:48:02 +08:00
parent 16fe4ec54e
commit a6f99a2551
4 changed files with 38 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
javdb/src/fetch.py
View File

@ -14,6 +14,7 @@ config.setup_logging()
debug = False
force = False
skip_local = True
# 获取演员列表
def fetch_actor_list():
@ -236,8 +237,14 @@ def fetch_movies_detail():
for movie in movies_list:
url = movie['href']
title = movie['title']
curr_id = movie['id']
logging.debug(f"Fetching data for movie ({title}), url {url} ...")
soup, status_code = scraper.fetch_page(url, partial(scraper.generic_validator, tag="div", identifier="video-meta-panel", attr_type="class"))
# 从本地读取的文件，忽略
if skip_local and status_code == 99 :
last_movie_id = curr_id
continue
# 解析页面，写入数据库
if soup:
movie_data = scraper.parse_movie_detail(soup, url, title)
if movie_data :
@ -278,7 +285,7 @@ function_map = {
}
# 主函数
def main(cmd, args_debug, args_force):
def main(cmd, args_debug, args_force, args_skip_local):
global debug
debug = args_debug
if debug:
@ -288,13 +295,16 @@ def main(cmd, args_debug, args_force):
global force
force = args_force
global skip_local
skip_local = args_skip_local
# 开启任务
task_id = db_tools.insert_task_log()
if task_id is None:
logging.warning(f'insert task log error.')
return None
logging.info(f'running task. id: {task_id}, debug: {debug}, force: {force}, cmd: {cmd}')
logging.info(f'running task. id: {task_id}, debug: {debug}, force: {force}, skip_local: {skip_local}, cmd: {cmd}')
# 执行指定的函数
if cmd:
@ -328,6 +338,7 @@ if __name__ == "__main__":
parser.add_argument("--cmd", type=str, help=f"Comma-separated list of function shortcuts: {keys_str}")
parser.add_argument('--debug', action='store_true', help='Enable debug mode (limit records)')
parser.add_argument('--force', action='store_true', help='force update (true for rewrite all)')
parser.add_argument('--skip_local', action='store_true', help='skip if cached html (true for skip)')
args = parser.parse_args()
main(args.cmd, args.debug, args.force)
main(args.cmd, args.debug, args.force, args.skip_local)

4
javdb/src/sqlite_utils.py
View File

@ -595,7 +595,7 @@ def query_movies(identifier):
# 按条件查询 href 列表
def query_movie_hrefs(**filters):
try:
sql = "SELECT href, title FROM javdb_movies WHERE 1=1"
sql = "SELECT href, title, id FROM javdb_movies WHERE 1=1"
params = []
if "id" in filters:
@ -643,7 +643,7 @@ def query_movie_hrefs(**filters):
cursor.execute(sql, params)
#return [row[0].lower() for row in cursor.fetchall()] # 链接使用小写
return [{'href': row[0], 'title': row[1]} for row in cursor.fetchall()]
return [{'href': row[0], 'title': row[1], 'id':row[2]} for row in cursor.fetchall()]
except sqlite3.Error as e:
logging.error(f"查询 href 失败: {e}")