research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-06-27 09:20:47 +08:00
parent d91ba1cd17
commit ac7cff9454
4 changed files with 39 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/crawling/craw.py
View File

@ -2,11 +2,13 @@ import logging
import sys
import requests
import re
import time
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import src.utils.utils as utils
http_code_404 = 404
http_code_403 = 403
http_code_redirect = 401
http_code_url = 601
http_code_local = 99
@ -59,10 +61,10 @@ class GenericCrawler:
response = self.scraper.get(url, headers=self.headers, cookies=self.cookies)
# 处理 HTTP 状态码
if response.status_code == http_code_404:
logging.debug(f"Page not found (404): {url}")
return None, http_code_404 # 直接返回 404，调用方可以跳过
if response.status_code in [http_code_404, http_code_403]:
logging.debug(f"get http code: {response.status_code}, url: {url}")
return None, response.status_code # 直接返回，调用方可以跳过
response.raise_for_status() # 处理 HTTP 错误
# 检查是否发生跳转，比如到登录页面
@ -86,6 +88,7 @@ class GenericCrawler:
logging.warning(f"Validation failed on attempt {attempt + 1} for {url}")
except Exception as e:
logging.error(f"Unexpected error on {url}: {e}, Retrying...")
time.sleep(0.3)
logging.error(f'Fetching failed after max retries. {url}')
return None, None # 达到最大重试次数仍然失败

34
src/db_utils/sqlite_db.py
View File

@ -634,7 +634,7 @@ class JavbusDBHandler(DatabaseHandler):
return result
# 处理影片的 无码 字段
def reset_movies_uncensored(self):
def reset_movies_uncensored(self, check_and_do = 0):
try:
logging.info("创建临时表以便于保存待更新记录")
self.cursor.execute("""
@ -669,24 +669,26 @@ class JavbusDBHandler(DatabaseHandler):
total_movies = self.cursor.execute("SELECT COUNT(*) FROM javbus_movies").fetchone()[0]
logging.info(f"共收集到 {total_count} 部需要更新的影片, 共有 {total_movies} 部影片")
# 1. 将所有记录的uncensored默认设为0
logging.info("开始将所有影片的uncensored设为默认值0...")
self.cursor.execute("UPDATE javbus_movies SET uncensored = 0")
logging.info(f"已将 {self.cursor.rowcount} 条记录的uncensored设为0")
if check_and_do:
# 1. 将所有记录的uncensored默认设为0
logging.info("开始将所有影片的uncensored设为默认值0...")
self.cursor.execute("UPDATE javbus_movies SET uncensored = 0")
logging.info(f"已将 {self.cursor.rowcount} 条记录的uncensored设为0")
# 2. 将临时表中匹配的记录设为1
logging.info("开始将匹配的影片的uncensored设为1...")
self.cursor.execute("""
UPDATE javbus_movies
SET uncensored = 1
WHERE id IN (SELECT movie_id FROM temp_movies_to_update)
""")
logging.info(f"已将 {self.cursor.rowcount} 条记录的uncensored设为1")
# 2. 将临时表中匹配的记录设为1
logging.info("开始将匹配的影片的uncensored设为1...")
self.cursor.execute("""
UPDATE javbus_movies
SET uncensored = 1
WHERE id IN (SELECT movie_id FROM temp_movies_to_update)
""")
logging.info(f"已将 {self.cursor.rowcount} 条记录的uncensored设为1")
# 3. 清理临时表，也可以不清理，以便于抽检
self.conn.commit()
else:
logging.info("check完毕，本次忽略更新。。。")
self.conn.commit()
logging.info("所有更新已提交")
logging.info("任务执行完成！")
except sqlite3.Error as e:
self.conn.rollback()

30
src/javbus/fetch.py
View File

@ -315,7 +315,8 @@ def fetch_performers_detail():
uncensored = int(performer['uncensored'])
avatar = None
if not utils.is_valid_url(url):
logging.info(f'invalid url ({url}), name: {person}. skipping...')
actor_id = db_tools.update_actor_detail_404({'href': url, 'is_full_data': craw.http_code_404})
logging.info(f'invalid url ({url}), id: {actor_id}, name: {person}. skipping...')
continue
next_url = url
@ -331,14 +332,9 @@ def fetch_performers_detail():
avatar = data.get('avatar')
all_movies.extend(data.get('movies', []))
elif status_code and status_code == craw.http_code_404:
actor_id = db_tools.update_actor_detail_404({'href': url, 'is_full_data': craw.http_code_404})
logging.warning(f'404 page. id: {actor_id}, name: ({person}), url: {url}, Skiping...')
need_insert = False
break
elif status_code and status_code == craw.http_code_redirect:
actor_id = db_tools.update_actor_detail_404({'href': url, 'is_full_data': craw.http_code_redirect})
logging.warning(f'401 page(need login). id: {actor_id}, name: ({person}), url: {url}, Skiping...')
elif status_code and status_code in [craw.http_code_404, craw.http_code_403, craw.http_code_redirect]:
actor_id = db_tools.update_actor_detail_404({'href': url, 'is_full_data': status_code})
logging.warning(f'get page http code {status_code}. id: {actor_id}, name: ({person}), url: {url}, Skiping...')
need_insert = False
break
else:
@ -414,6 +410,7 @@ def fetch_movies_detail():
curr_id = movie['id']
uncensored = int(movie['uncensored'])
if not utils.is_valid_url(url):
movie_id = db_tools.insert_or_update_movie_404({'href': url, 'is_full_data': craw.http_code_404})
logging.info(f'invalid url ({url}), row id: {curr_id}. skipping...')
continue
@ -442,12 +439,9 @@ def fetch_movies_detail():
logging.warning(f'parse_page_movie error. url: {url}')
time.sleep(2)
elif status_code and status_code == craw.http_code_404:
movie_id = db_tools.insert_or_update_movie_404({'href': url, 'is_full_data': craw.http_code_404})
logging.warning(f'404 page. id: {movie_id}, title: ({title}), url: {url}, Skiping...')
elif status_code and status_code == craw.http_code_redirect:
movie_id = db_tools.insert_or_update_movie_404({'href': url, 'is_full_data': craw.http_code_redirect})
logging.warning(f'401 page(need login). id: {movie_id}, title: ({title}), url: {url}, Skiping...')
elif status_code and status_code in [craw.http_code_404, craw.http_code_403, craw.http_code_redirect]:
movie_id = db_tools.insert_or_update_movie_404({'href': url, 'is_full_data': status_code})
logging.warning(f'get page http code {status_code}. id: {movie_id}, title: ({title}), url: {url}, Skiping...')
else:
logging.warning(f'fetch_page error. url: {url}')
time.sleep(0.5)
@ -456,7 +450,10 @@ def fetch_movies_detail():
if debug:
return True
# 重置 movies 表的 uncensored 标志位
def reset_movies_uncensored():
db_tools.reset_movies_uncensored(check_and_do=0 if debug else 1)
# 建立缩写到函数的映射
function_map = {
"actor_list": fetch_actor_list,
@ -467,6 +464,7 @@ function_map = {
"movies" : fetch_movies_detail,
"langs" : update_multi_langs,
"tags" : update_multilang_tags,
"reset_un" : reset_movies_uncensored
}
# 主函数

3
src/monitor/check_javbus.py
View File

@ -2,9 +2,7 @@ import json
import time
import src.db_utils.sqlite_db as sqlite_db
import src.utils.utils as utils
import src.logger.logger as logger
logger.setup_logging()
db_tools = sqlite_db.JavbusDBHandler()
if __name__ == "__main__":
@ -12,5 +10,4 @@ if __name__ == "__main__":
result = db_tools.get_statics()
utils.pretty_print_json(result)
db_tools.reset_movies_uncensored()