modify scripts

javdb/src/fetch.py

@@ -14,6 +14,7 @@ config.setup_logging()
 
 debug = False
 force = False
+skip_local = True
 
 # 获取演员列表
 def fetch_actor_list():    
@@ -236,8 +237,14 @@ def fetch_movies_detail():
         for movie in movies_list:
             url = movie['href']
             title = movie['title']
+            curr_id = movie['id']
             logging.debug(f"Fetching data for movie ({title}), url {url} ...")
             soup, status_code = scraper.fetch_page(url, partial(scraper.generic_validator, tag="div", identifier="video-meta-panel", attr_type="class"))
+            # 从本地读取的文件，忽略
+            if skip_local and status_code == 99 :
+                last_movie_id = curr_id
+                continue
+            # 解析页面，写入数据库
             if soup:
                 movie_data = scraper.parse_movie_detail(soup, url, title)
                 if movie_data :
@@ -278,7 +285,7 @@ function_map = {
 }   
 
 # 主函数
-def main(cmd, args_debug, args_force):
+def main(cmd, args_debug, args_force, args_skip_local):
     global debug
     debug = args_debug
     if debug:
@@ -288,13 +295,16 @@ def main(cmd, args_debug, args_force):
     global force
     force = args_force
 
+    global skip_local
+    skip_local = args_skip_local
+
     # 开启任务
     task_id = db_tools.insert_task_log()
     if task_id is None:
         logging.warning(f'insert task log error.')
         return None
     
-    logging.info(f'running task. id: {task_id}, debug: {debug}, force: {force}, cmd: {cmd}')
+    logging.info(f'running task. id: {task_id}, debug: {debug}, force: {force}, skip_local: {skip_local}, cmd: {cmd}')
 
     # 执行指定的函数
     if cmd:
@@ -328,6 +338,7 @@ if __name__ == "__main__":
     parser.add_argument("--cmd", type=str, help=f"Comma-separated list of function shortcuts: {keys_str}")
     parser.add_argument('--debug', action='store_true', help='Enable debug mode (limit records)')
     parser.add_argument('--force', action='store_true', help='force update (true for rewrite all)')
+    parser.add_argument('--skip_local', action='store_true', help='skip if cached html (true for skip)')
     args = parser.parse_args()
     
-    main(args.cmd, args.debug, args.force)
+    main(args.cmd, args.debug, args.force, args.skip_local)