From 096957705d589b85222b819739a633e35e23604c Mon Sep 17 00:00:00 2001 From: oscarz Date: Sun, 6 Jul 2025 19:47:42 +0800 Subject: [PATCH] modify scripts --- .../db_wapper/spider_db_handler.py | 9 ++-- sqlite_tools/shared_db_schema.sql | 45 ++++++++++--------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/scrapy_proj/scrapy_proj/db_wapper/spider_db_handler.py b/scrapy_proj/scrapy_proj/db_wapper/spider_db_handler.py index 1cbf625..b619cf2 100644 --- a/scrapy_proj/scrapy_proj/db_wapper/spider_db_handler.py +++ b/scrapy_proj/scrapy_proj/db_wapper/spider_db_handler.py @@ -353,7 +353,7 @@ class PboxDBHandler(SQLiteDBHandler): if not actor_id: actor_id = self.insert_or_update_common(actor, self.tbl_actor, uniq_key='href', exists_do_nothing=True) if actor_id: - lnk_id = self.insert_actor_movie(performer_id=actor_id, movie_id=mov_id) + lnk_id = self.insert_or_update_common({'movie_id':mov_id, 'actor_id':actor_id, 'actor_mov': f'{actor_id}_f{mov_id}'}, tbl_name=self.tbl_actor_mov, uniq_key='actor_mov', exists_do_nothing=True) if not lnk_id: logging.error(f"insert actor_movie error. actor id: {actor_id}, mov id: {mov_id}") @@ -363,13 +363,16 @@ class PboxDBHandler(SQLiteDBHandler): if not tag_id: tag_id = self.insert_or_update_common(tag, self.tbl_tags, uniq_key='href') if tag_id: - lnk_id = self.insert_movie_tags(movie_id=mov_id, tag_id=tag_id, tags='') + lnk_id = self.insert_or_update_common({'movie_id':mov_id, 'tag_id':tag_id, 'movid_tagid': f'{mov_id}_f{tag_id}', 'tags':''}, tbl_name=self.tbl_mov_tags, uniq_key='movid_tagid', exists_do_nothing=True) if not lnk_id: logging.error(f"insert movie_tag error. tag id: {tag_id}, mov id: {mov_id}") # 插入别名 for alt in item.get('mov_alt_list', []): - self.insert_mov_alt(alt.get('min_mov_id', 0), alt.get('max_mov_id', 0)) + min_max = f"{alt['min_mov_id']}_{alt['max_mov_id']}" + lnk_id = self.insert_or_update_common({'min_mov_id':alt['min_mov_id'], 'max_mov_id':alt['max_mov_id'], 'min_max': min_max}, tbl_name=self.tbl_mov_alts, uniq_key='min_max', exists_do_nothing=True) + if not lnk_id: + logging.error(f"insert movie_alt error. item: {alt}") def get_studios(self, **filters): diff --git a/sqlite_tools/shared_db_schema.sql b/sqlite_tools/shared_db_schema.sql index f7b10e9..1cf94a7 100644 --- a/sqlite_tools/shared_db_schema.sql +++ b/sqlite_tools/shared_db_schema.sql @@ -585,25 +585,6 @@ CREATE TABLE IF NOT EXISTS "pbox_tags" ( `created_at` TEXT DEFAULT (datetime('now', 'localtime')), `updated_at` TEXT DEFAULT (datetime('now', 'localtime')) ); -CREATE TABLE IF NOT EXISTS "pbox_movies_tags" ( - movie_row_id INTEGER, - tag_row_id INTEGER, - tags TEXT, - created_at TEXT DEFAULT (datetime('now', 'localtime')), - updated_at TEXT DEFAULT (datetime('now', 'localtime')), - PRIMARY KEY (movie_row_id, tag_row_id), - FOREIGN KEY (tag_row_id) REFERENCES pbox_tags(id), - FOREIGN KEY (movie_row_id) REFERENCES "pbox_movies"(id) -); -CREATE TABLE IF NOT EXISTS "pbox_movies_alts" ( - min_mov_id INTEGER, - max_mov_id INTEGER, - created_at TEXT DEFAULT (datetime('now', 'localtime')), - updated_at TEXT DEFAULT (datetime('now', 'localtime')), - PRIMARY KEY (min_mov_id, max_mov_id), - FOREIGN KEY (min_mov_id) REFERENCES "pbox_movies"(id), - FOREIGN KEY (max_mov_id) REFERENCES "pbox_movies"(id) -); CREATE TABLE IF NOT EXISTS "pbox_actors" ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, @@ -623,12 +604,34 @@ CREATE TABLE IF NOT EXISTS "pbox_actor_aliases" ( foreign key(`actor_id`) references "pbox_actors"(`id`) on delete CASCADE, PRIMARY KEY(`actor_id`, `alias`) ); +CREATE TABLE IF NOT EXISTS "pbox_movies_tags" ( + `id` integer not null primary key autoincrement, + movie_id INTEGER, + tag_id INTEGER, + movid_tagid varchar(255) UNIQUE, + tags TEXT, + created_at TEXT DEFAULT (datetime('now', 'localtime')), + updated_at TEXT DEFAULT (datetime('now', 'localtime')), + FOREIGN KEY (tag_id) REFERENCES pbox_tags(id), + FOREIGN KEY (movie_id) REFERENCES "pbox_movies"(id) +); +CREATE TABLE IF NOT EXISTS "pbox_movies_alts" ( + `id` integer not null primary key autoincrement, + min_mov_id INTEGER, + max_mov_id INTEGER, + min_max varchar(255) UNIQUE, + created_at TEXT DEFAULT (datetime('now', 'localtime')), + updated_at TEXT DEFAULT (datetime('now', 'localtime')), + FOREIGN KEY (min_mov_id) REFERENCES "pbox_movies"(id), + FOREIGN KEY (max_mov_id) REFERENCES "pbox_movies"(id) +); CREATE TABLE IF NOT EXISTS "pbox_actors_movies" ( + `id` integer not null primary key autoincrement, `actor_id` integer, `movie_id` integer, + actor_mov varchar(255) UNIQUE, created_at TEXT DEFAULT (datetime('now', 'localtime')), updated_at TEXT DEFAULT (datetime('now', 'localtime')), tags TEXT, foreign key(`actor_id`) references "pbox_actor"(`id`) on delete CASCADE, - foreign key(`movie_id`) references "pbox_movies"(`id`) on delete CASCADE, - PRIMARY KEY (`actor_id`, `movie_id`) + foreign key(`movie_id`) references "pbox_movies"(`id`) on delete CASCADE );