diff --git a/javdb/src/fetch.py b/javdb/src/fetch.py index a3f0ab4..ca42e7b 100644 --- a/javdb/src/fetch.py +++ b/javdb/src/fetch.py @@ -220,8 +220,8 @@ def fetch_movies_detail(): logging.warning(f'parse_page_movie error. url: {url}') elif status_code and status_code == 404: - logging.warning(f'fetch page error. httpcode: {status_code}, url: {next_url}') - break + movie_id = db_tools.insert_or_update_movie_404(title=title, href=url) + logging.warning(f'404 page. id: {movie_id}, title: ({title}), url: {url}, Skiping...') else: logging.warning(f'fetch_page error. url: {url}') time.sleep(1) diff --git a/javdb/src/sqlite_utils.py b/javdb/src/sqlite_utils.py index f60823b..8ff8045 100644 --- a/javdb/src/sqlite_utils.py +++ b/javdb/src/sqlite_utils.py @@ -428,6 +428,34 @@ def insert_or_update_movie(movie): logging.error("Error inserting movie: %s", e) return None +# """插入或更新电影数据(异常url的处理,比如404链接)""" +def insert_or_update_movie_404(title, href): + try: + # 插入或更新电影信息 + cursor.execute( + """ + INSERT INTO javdb_movies (title, href, is_full_data, updated_at) + VALUES (?, ?, 1, datetime('now', 'localtime')) + ON CONFLICT(href) DO UPDATE SET + title=excluded.title, is_full_data=1, updated_at = datetime('now', 'localtime') + """, + (title, href) + ) + conn.commit() + + # 获取插入的 movie_id + movie_id = get_id_by_href('javdb_movies', href) + if movie_id is None: + return None + + return movie_id + + except Exception as e: + conn.rollback() + logging.error("Error inserting movie: %s", e) + return None + + # 删除电影数据""" def delete_movie(identifier): try: