From 56b88bbcc9d7091a57ea127cd0f3b9ae554c29ab Mon Sep 17 00:00:00 2001 From: sophon Date: Thu, 20 Nov 2025 16:06:43 +0800 Subject: [PATCH] modify scripts --- scrapy_proj/scrapy_proj/spiders/theporndb.py | 10 + src/tools/paco.py | 678 +++++++++++++++++++ src/tools/sync_db.py | 225 ++++++ 3 files changed, 913 insertions(+) create mode 100644 scrapy_proj/scrapy_proj/spiders/theporndb.py create mode 100644 src/tools/paco.py create mode 100644 src/tools/sync_db.py diff --git a/scrapy_proj/scrapy_proj/spiders/theporndb.py b/scrapy_proj/scrapy_proj/spiders/theporndb.py new file mode 100644 index 0000000..6dcb30f --- /dev/null +++ b/scrapy_proj/scrapy_proj/spiders/theporndb.py @@ -0,0 +1,10 @@ +import scrapy + + +class TheporndbSpider(scrapy.Spider): + name = "theporndb" + allowed_domains = ["theporndb.net"] + start_urls = ["https://theporndb.net"] + + def parse(self, response): + pass diff --git a/src/tools/paco.py b/src/tools/paco.py new file mode 100644 index 0000000..3135d8b --- /dev/null +++ b/src/tools/paco.py @@ -0,0 +1,678 @@ +import os +import sys + +rim_all = { + "data": { + "searchScene": [ + { + "id": "bcda549f-54ac-40a4-a0b6-68544f05a369", + "title": "Mature Lady Piledriver Position Bj: Mizuki", + "release_date": "2022-12-30", + "duration": 3273, + "code": "123022_766", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "299402d2-87c3-4dc4-8933-bc1e7449441a", + "title": "Mature Lady Piledriver Position BJ: Luna", + "release_date": "2024-08-17", + "duration": 3590, + "code": "081724_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "89322a4b-f412-4cba-97db-30a4d112f022", + "title": "Mature Lady Piledriver Position BJ: Mina Sakura", + "release_date": "2025-10-07", + "duration": 0, + "code": "100725_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "ebebdb28-e070-439b-9078-81c31d870d90", + "title": "Mature Lady Piledriver Position Bj: Sanae Yamaguchi", + "release_date": "2022-04-02", + "duration": 3599, + "code": "040222_626", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "60fb6b29-665f-4e8b-92f7-af8ded439a0f", + "title": "Mature Lady Piledriver Position Bj: Aisa Tamano", + "release_date": "2022-04-16", + "duration": 0, + "code": "041622_632", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "f0b61b58-c6d3-4089-82d8-b99bc6e570e8", + "title": "Mature Lady Piledriver Position Bj: Hiroko Otsuka", + "release_date": "2022-05-12", + "duration": 3673, + "code": "051222_646", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "7a78dc09-a6b8-4e38-80de-836287d440bd", + "title": "Mature Lady Piledriver Position Bj: Hitomi Yoneda", + "release_date": "2022-06-21", + "duration": 3661, + "code": "062122_663", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "40af4af5-79d7-4f8e-8ba3-da6e005baa33", + "title": "Mature Lady Piledriver Position Bj: Minami Yamazaki", + "release_date": "2022-05-26", + "duration": 3601, + "code": "052622_652", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "647d26d4-0caf-4c59-b24f-c53954175f10", + "title": "Mature Lady Piledriver Position Bj: Moena Nishiuchi", + "release_date": "2022-06-11", + "duration": 3655, + "code": "061122_659", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "1f1ea0eb-1253-4111-a95b-cd75b91862f5", + "title": "Mature Lady Piledriver Position BJ : Sumire Maeda", + "release_date": "2021-01-05", + "duration": 3677, + "code": "010521_413", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "f73e0fc0-8cf2-456d-9752-fddf7e5fcee8", + "title": "Mature Lady Piledriver Position BJ : Nagisa Shinohara", + "release_date": "2021-09-25", + "duration": 3457, + "code": "092521_537", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "72bf88b5-9726-4683-a064-4cf39c607b72", + "title": "Mature Lady Piledriver Position BJ : Misato Wakaba", + "release_date": "2021-09-18", + "duration": 3738, + "code": "091821_533", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "57e19438-a461-42d4-8bdb-d88baad2c90b", + "title": "Mature Lady Piledriver Position BJ : Moe Narumiya", + "release_date": "2021-09-09", + "duration": 0, + "code": "090921_529", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "9d9f3024-e7c3-4883-be2c-b1881febb40f", + "title": "Mature Lady Piledriver Position BJ : Maki Sekiguchi", + "release_date": "2021-08-26", + "duration": 3791, + "code": "082621_523", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "82d322e9-8c80-444a-a5d3-123023f75f55", + "title": "Mature Lady Piledriver Position BJ : Chika Yoda", + "release_date": "2021-08-14", + "duration": 0, + "code": "081421_517", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "a7ab8f1a-b0f1-4bd2-b2d5-677070223e1b", + "title": "Mature Lady Piledriver Position BJ : Chisato Nishiyama", + "release_date": "2021-08-10", + "duration": 0, + "code": "081021_513", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "7265a9a2-9bba-4cc7-a613-b4b9998765c4", + "title": "Mature Lady Piledriver Position BJ : Chihiro Hinata", + "release_date": "2021-07-03", + "duration": 3597, + "code": "070321_498", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "2d1aff8a-b0b1-4e56-aafa-0429342e6069", + "title": "Mature Lady Piledriver Position BJ : Sakura Kazuki", + "release_date": "2021-05-15", + "duration": 3037, + "code": "051521_477", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "82dd6280-ea1c-4afe-af73-46c0a56416b1", + "title": "Mature Lady Piledriver Position Bj: Chris Aoki", + "release_date": "2022-02-05", + "duration": 3265, + "code": "020522_603", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "e08f0706-3147-475a-8618-b43ac7bdb26e", + "title": "Mature Lady Piledriver Position Bj: AI Yamamoto", + "release_date": "2021-12-11", + "duration": 3659, + "code": "121121_570", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "db33e0e3-19c8-4ba0-ae5e-092ca6e8ac37", + "title": "Mature Lady Piledriver Position Bj: Noriko Masaki", + "release_date": "2021-12-04", + "duration": 3755, + "code": "120421_567", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "5acb2675-3d3f-410f-9021-016708e27519", + "title": "Mature Lady Piledriver Position BJ: Mizuki Reina", + "release_date": "2021-11-23", + "duration": 3300, + "code": "112321_562", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "e0423c3f-5c86-46af-be70-5a1bc48c4bdd", + "title": "Mature Lady Piledriver Position BJ : Hitomi Okano", + "release_date": "2021-10-09", + "duration": 3587, + "code": "100921_543", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "525ee6a2-793d-4fc1-befa-718b073b05d6", + "title": "Mature Lady Piledriver Position BJ : Runa Akasaka", + "release_date": "2021-04-03", + "duration": 3413, + "code": "040321_455", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "1de1815b-3dc0-40f7-9f4e-31baaf8881cf", + "title": "Mature Lady Piledriver Position BJ : Mari Onodera", + "release_date": "2021-04-02", + "duration": 3636, + "code": "060521_486", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "f516f556-00c7-43a6-b4cb-b2c15288131a", + "title": "Mature Lady Piledriver Position Bj: Nono Otsuki", + "release_date": "2022-03-19", + "duration": 3601, + "code": "031922_621", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "c2e6437a-53bd-422b-8fb8-d99e218ffbfc", + "title": "Mature Lady Piledriver Position Bj: Yukari Sato", + "release_date": "2022-03-24", + "duration": 3638, + "code": "032422_622", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "c98b41e1-6986-4a00-a2bd-5417a296858b", + "title": "Mature Lady Piledriver Position Bj: Junko Nakajima", + "release_date": "2022-04-12", + "duration": 0, + "code": "041222_630", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "ebff3bf9-869f-459c-b067-f73ec1ec060c", + "title": "Mature Lady Piledriver Position Bj: Azusa Yamamoto", + "release_date": "2022-12-08", + "duration": 3890, + "code": "120822_752", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "664c87bb-d260-45a1-9e28-46d32de97a66", + "title": "Mature Lady Piledriver Position Bj: Wakaba Sakai", + "release_date": "2023-06-01", + "duration": 3601, + "code": "060123_858", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "55a546b5-aeeb-4727-848e-4a7cdd1b03be", + "title": "Mature Lady Piledriver Position BJ: Nobuko Nakagawa", + "release_date": "2023-01-21", + "duration": 3676, + "code": "012123_783", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "9e113e21-2d2b-4c47-93a0-2c0dd8df4d99", + "title": "Mature Lady Piledriver Position Bj: Hiroko Sunada", + "release_date": "2023-01-09", + "duration": 3617, + "code": "010923_776", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "4e721a23-4555-4a79-a2a9-6d3a32fe3657", + "title": "Mature Lady Piledriver Position BJ: Iori Otoha", + "release_date": "2023-01-26", + "duration": 3037, + "code": "012623_785", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "40deb000-ccc6-4c02-b91d-3c931b68146f", + "title": "Mature Lady Piledriver Position Bj: Keiko Fujiyama", + "release_date": "2023-08-22", + "duration": 3506, + "code": "082223_899", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "b38103a2-b988-4630-a9d0-bd383d7b3c2a", + "title": "Mature Lady Piledriver Position Bj: Rio Ayanami", + "release_date": "2022-09-22", + "duration": 3469, + "code": "092222_708", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "c95087ad-c13b-4c0a-8fe8-c5457098e2fd", + "title": "Mature Lady Piledriver Position Bj: Mizue Miura", + "release_date": "2022-07-29", + "duration": 3714, + "code": "072922_680", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "11e4876b-a1e4-415c-bd5e-928d33bc02f2", + "title": "Mature Lady Piledriver Position Bj: Nana Natsume", + "release_date": "2022-08-14", + "duration": 3608, + "code": "081422_690", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "a68bdcaa-ad41-4a99-a636-547c7575c9c6", + "title": "Mature Lady Piledriver Position Bj: Momoko Tanaka", + "release_date": "2022-09-06", + "duration": 3528, + "code": "090622_700", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "e628bddd-d3d8-488d-bdc2-ee67da88b479", + "title": "Mature Lady Piledriver Position Bj: Eri Saeki", + "release_date": "2023-11-23", + "duration": 1410, + "code": "112323_942", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "13f2443c-c682-4244-ba30-7c8768ba11c2", + "title": "Mature Lady Piledriver Position Bj: Miyo Inoue", + "release_date": "2023-10-28", + "duration": 3762, + "code": "102823_930", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "d2479320-31c9-4ab6-9787-0077e094272a", + "title": "Mature Lady Piledriver Position Bj: Yoshie Fujimoto", + "release_date": "2024-02-03", + "duration": 3603, + "code": "020324_979", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "d5d48718-9fbc-4199-b1ca-4be2175b7da1", + "title": "Mature Lady Piledriver Position Bj: Keiko Yamada", + "release_date": "2023-11-28", + "duration": 3847, + "code": "112823_945", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "a0775fb3-020d-4576-a3d9-8e2caff41e9f", + "title": "Mature Lady Piledriver Position Bj: Yurie Goto", + "release_date": "2023-12-14", + "duration": 3681, + "code": "121423_952", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "31c536c4-9b0b-4715-a708-13c22a2c1273", + "title": "Mature Lady Piledriver Position Bj: Hina Mori", + "release_date": "2023-02-28", + "duration": 3636, + "code": "022823_804", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "489d219b-f330-43b4-94e8-68331b7e93d6", + "title": "Mature Lady Piledriver Position Bj: Maki Koizumi", + "release_date": "2023-03-02", + "duration": 1355, + "code": "030223_805", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "3c8c1889-1d6c-4f3e-b3a6-75f7e27a27bd", + "title": "Mature Lady Piledriver Position Bj: Miki Hoshino", + "release_date": "2023-05-04", + "duration": 1208, + "code": "050423_843", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "8b8df30f-94f8-426a-bf44-b7341927eca9", + "title": "Mature Lady Piledriver Position BJ: Misato Hara", + "release_date": "2024-05-07", + "duration": 3753, + "code": "050724_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "77425f73-47e4-4fb4-956e-5bf18111983e", + "title": "Mature Lady Piledriver Position BJ: Sawa Minami", + "release_date": "2025-01-25", + "duration": 4177, + "code": "012525_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "f539d76c-f675-4c5d-be7e-80e1cac7b456", + "title": "Mature Lady Piledriver Position BJ: Arisa Murase", + "release_date": "2024-05-16", + "duration": 3636, + "code": "051624_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "12ff01c3-7d26-410e-b142-7239893209ce", + "title": "Mature Lady Piledriver Position BJ: Yuko Nakahara", + "release_date": "2024-06-04", + "duration": 3745, + "code": "060424_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "d1868a36-53f5-4c33-84b4-1fa1edc6931a", + "title": "Mature Lady Piledriver Position BJ: Yoko Iida", + "release_date": "2024-06-11", + "duration": 3473, + "code": "061124_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "f25f2f88-9021-4a44-ac8b-82fd05271185", + "title": "Mature Lady Piledriver Position BJ: Sana Rukawa", + "release_date": "2024-06-27", + "duration": 3037, + "code": "062724_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "5c77d8f7-2aca-4356-a62b-3c619ce47475", + "title": "Mature Lady Piledriver Position BJ: Yumi Yamada", + "release_date": "2024-07-20", + "duration": 3848, + "code": "072024_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "3f3b2976-7808-4edf-8523-d696132354ec", + "title": "Mature Lady Piledriver Position BJ: Yumi Saeki", + "release_date": "2024-09-05", + "duration": 4805, + "code": "090524_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "d48255ad-7591-4d18-80c4-fd0165e4fd2a", + "title": "Mature Lady Piledriver Position BJ: Kyoko Ishizaki", + "release_date": "2024-07-30", + "duration": 3205, + "code": "073024_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "7f1357ca-d9ff-43a2-b33a-f418e1773d80", + "title": "Mature Lady Piledriver Position BJ: Aya Kawai", + "release_date": "2025-05-29", + "duration": 3563, + "code": "052925_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "693f64d0-ada7-4738-b6ec-e1e116b26586", + "title": "Mature Lady Piledriver Position BJ: Rino Sakuragi", + "release_date": "2025-03-01", + "duration": 1589, + "code": "030125_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "d3990afc-0ccf-4bcf-b19f-bccac3271c8d", + "title": "Mature Lady Piledriver Position BJ: Maria Sakamoto", + "release_date": "2025-02-15", + "duration": 3467, + "code": "021525_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "8b5faa4b-eb55-401b-8037-35c54a2cc9af", + "title": "Mature Lady Piledriver Position BJ : Hitomi Honda", + "release_date": "2021-06-17", + "duration": 3627, + "code": "061721_491", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "4e2a257a-b719-4af0-966e-438fc7145366", + "title": "Mature Lady Piledriver Position BJ: Suzune Miyashita", + "release_date": "2022-05-04", + "duration": 3419, + "code": "050422_641", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "d0ddfbe2-6dc4-4dc8-a1f0-8acc9f378bfa", + "title": "Mature Lady Piledriver Position Bj: Misao Himeno", + "release_date": "2021-11-13", + "duration": 0, + "code": "111321_558", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "e5a4aae0-861e-437c-8a77-53537cdbcbf1", + "title": "Mature Lady Piledriver Position Bj: Akemi Kihara", + "release_date": "2021-10-30", + "duration": 3337, + "code": "103021_552", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "1fe58d93-be28-4ca1-b8be-36969a25f8b9", + "title": "Mature Lady Piledriver Position BJ: Seri Mizutani", + "release_date": "2024-07-06", + "duration": 3637, + "code": "070624_100", + "studio": { + "name": "Pacopacomama" + } + }, + { + "id": "e820f61c-16cd-4ce3-9f7e-0b285e2a8691", + "title": "Mature Lady Piledriver Position Bj: Yui Shinjyo", + "release_date": "2022-07-15", + "duration": 4179, + "code": "071522_674", + "studio": { + "name": "Pacopacomama" + } + } + ] + } +} + +rim_done = ["121121_570","040222_626","020324_979","032422_622","061124_100","010521_413","041622_632","051222_646","052622_652","123022_766","081724_100","061122_659","062122_663","040222_626"] +rim_new ={} +print("---------done:-----------") +for scene in rim_all['data']['searchScene']: + code = scene['code'] + title = scene['title'] + date = scene['release_date'] + duration = scene['duration'] + studio = scene['studio']['name'] + if code in rim_done: + print(f"{code} | {date} | {title} | {studio}") + continue + + rim_new[code] = { + 'title': title, + 'date': date, + 'duration': duration, + 'studio': studio + } + +print("\n\n--------- to be downloaded:-----------") +# 按照发布日期排序并打印结果 +for code, info in sorted(rim_new.items(), key=lambda x: x[1]['date']): + title = info['title'] + date = info['date'] + studio = info['studio'] + print(f"{code} | {date} | {title} | {studio}") + diff --git a/src/tools/sync_db.py b/src/tools/sync_db.py new file mode 100644 index 0000000..df73122 --- /dev/null +++ b/src/tools/sync_db.py @@ -0,0 +1,225 @@ +import sqlite3 +import pymysql +from pymysql.cursors import DictCursor +import os +import argparse + + +def sync_sqlite_table_to_mysql(sqlite_path, sqlite_table, mysql_conn, mysql_table): + """ + 将SQLite单个表同步到MySQL + 功能： + 1. 检查目标表是否存在 + 2. 如果不存在则创建 + 3. 如果存在则 TRUNCATE + 4. 执行数据批量复制 + """ + + # ------------------------------------------------------------ + # 1. 连接 SQLite 读取数据结构 + # ------------------------------------------------------------ + sqlite_conn = sqlite3.connect(sqlite_path) + sqlite_conn.row_factory = sqlite3.Row + cursor_sqlite = sqlite_conn.cursor() + + # 获取表结构 + cursor_sqlite.execute(f"PRAGMA table_info({sqlite_table})") + columns_info = cursor_sqlite.fetchall() + + if not columns_info: + raise Exception(f"SQLite 表 {sqlite_table} 不存在") + + columns = [col["name"] for col in columns_info] + column_definitions = [] + + # SQLite → MySQL 类型映射 + type_mapping = { + "INTEGER": "INT", + "TEXT": "TEXT", + "REAL": "DOUBLE", + "BLOB": "BLOB", + "NUMERIC": "DECIMAL(18,4)" + } + + pk_column = None + + for col in columns_info: + colname = col["name"] + coltype = col["type"].upper() + mysql_type = next((mysql_t for t, mysql_t in type_mapping.items() if t in coltype), "TEXT") + + coldef = f"`{colname}` {mysql_type}" + + if col["pk"] == 1: # 主键 + pk_column = colname + coldef += " PRIMARY KEY" + + column_definitions.append(coldef) + + create_table_sql = f"CREATE TABLE `{mysql_table}` ({','.join(column_definitions)}) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;" + + # ------------------------------------------------------------ + # 2. 检查 MySQL 目标表是否存在 + # ------------------------------------------------------------ + cursor_mysql = mysql_conn.cursor() + + cursor_mysql.execute("SHOW TABLES LIKE %s", (mysql_table,)) + exists = cursor_mysql.fetchone() + + if not exists: + print(f"目标表 {mysql_table} 不存在，创建中...") + cursor_mysql.execute(create_table_sql) + mysql_conn.commit() + else: + print(f"目标表 {mysql_table} 已存在，执行 TRUNCATE...") + cursor_mysql.execute(f"TRUNCATE TABLE `{mysql_table}`") + mysql_conn.commit() + + # ------------------------------------------------------------ + # 3. 从 SQLite 读取全部记录 + # ------------------------------------------------------------ + cursor_sqlite.execute(f"SELECT * FROM {sqlite_table}") + rows = cursor_sqlite.fetchall() + + if not rows: + print(f"SQLite 表 {sqlite_table} 没有数据") + return + + rows_count = len(rows) + # ------------------------------------------------------------ + # 4. 批量插入到 MySQL + # ------------------------------------------------------------ + placeholders = ",".join(["%s"] * len(columns)) + insert_sql = f"INSERT INTO `{mysql_table}` ({','.join(columns)}) VALUES ({placeholders})" + + # 分批插入数据，避免一次过大量数据导致内存问题 + batch_size = 10000 + while rows: + batch = rows[:batch_size] + rows = rows[batch_size:] + + data_list = [] + for row in batch: + data_list.append(tuple(row[col] for col in columns)) + + cursor_mysql.executemany(insert_sql, data_list) + mysql_conn.commit() + + print(f"同步完成：{sqlite_table} → {mysql_table}, 共 {rows_count} 条记录") + +# 示例 MySQL 配置 +mysql_config = { + 'dev': { + "host": "192.168.2.22", + "port": 13306, + "user": "root", + "password": "root", + "database": "resources", + "charset": "utf8mb4" + }, + 'nas': { + "host": "mariadb", + "port": 3306, + "user": "root", + "password": "root", + "database": "resources", + "charset": "utf8mb4" + } +} + +sqlite_config = { + 'dev': { + 'shared_db_path': "/root/sharedata/sqlite/shared.db", + 'whisparr_db_path': "/root/sharedata/sqlite/whisparr2.db", + 'stash_db_path': "/root/sharedata/sqlite/stash-go.sqlite" + }, + 'nas': { + 'shared_db_path': "/root/sharedata/sqlite/shared.db", + 'whisparr_db_path': "/root/sharedata/sqlite/stash_db/stash-go.sqlite", + 'stash_db_path': "/root/sharedata/sqlite/whisper_db/whisparr2.db" + } +} + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='sync data from sqlite3 to mariadb.\n\n', + formatter_class=argparse.RawDescriptionHelpFormatter + ) + #parser = argparse.ArgumentParser(description='fetch javdb data.') + parser.add_argument("--source", type=str, help=f"source database, stash, whisper, shared", default="stash,whisper,shared") + parser.add_argument("--env", type=str, help=f"execute enviroment", default="dev") + args = parser.parse_args() + + # get env config + current_env = args.env + if current_env not in mysql_config or current_env not in sqlite_config: + current_env = 'dev' # 默认使用开发环境配置 + + mysql_conn = pymysql.connect( + host=mysql_config[current_env]['host'], + port=mysql_config[current_env]['port'], + user=mysql_config[current_env]['user'], + password=mysql_config[current_env]['password'], + database=mysql_config[current_env]['database'], + charset=mysql_config[current_env]['charset'], + cursorclass=DictCursor + ) + + # parse source to list + source_list = [s.strip().lower() for s in args.source.split(",")] + + # sync IAFD tables + if "shared" in source_list: + sync_sqlite_table_to_mysql( + sqlite_path=sqlite_config[current_env]['shared_db_path'], + sqlite_table="iafd_studios", + mysql_conn=mysql_conn, + mysql_table="sync_iafd_studios" + ) + + sync_sqlite_table_to_mysql( + sqlite_path=sqlite_config[current_env]['shared_db_path'], + sqlite_table="iafd_distributors", + mysql_conn=mysql_conn, + mysql_table="sync_iafd_distributors" + ) + + sync_sqlite_table_to_mysql( + sqlite_path=sqlite_config[current_env]['shared_db_path'], + sqlite_table="iafd_movies", + mysql_conn=mysql_conn, + mysql_table="sync_iafd_movies" + ) + + # sync Whisparr tables + if "whisper" in source_list: + sync_sqlite_table_to_mysql( + sqlite_path=sqlite_config[current_env]['whisparr_db_path'], + sqlite_table="Series", + mysql_conn=mysql_conn, + mysql_table="sync_whisper_series" + ) + + + sync_sqlite_table_to_mysql( + sqlite_path=sqlite_config[current_env]['whisparr_db_path'], + sqlite_table="Episodes", + mysql_conn=mysql_conn, + mysql_table="sync_whisper_episodes" + ) + + # sync Stash tables + if "stash" in source_list: + sync_sqlite_table_to_mysql( + sqlite_path=sqlite_config[current_env]['stash_db_path'], + sqlite_table="studios", + mysql_conn=mysql_conn, + mysql_table="sync_stash_studios" + ) + + sync_sqlite_table_to_mysql( + sqlite_path=sqlite_config[current_env]['stash_db_path'], + sqlite_table="scenes", + mysql_conn=mysql_conn, + mysql_table="sync_stash_scenes" + )