modify scripts
This commit is contained in:
@ -10,4 +10,4 @@ scrapy crawl pbox -a debug='1' -s STATS_PUSH_MSG=False -a cmd='movies' -s LOG_LE
|
||||
|
||||
scrapy crawl javbus -a cmd=actors -s HTTPCACHE_DIR=/home/ubuntu/sharedata/scrapy_cached/
|
||||
|
||||
scrapy crawl iafd -a cmd='astro,ethnic,dist,stu' -s HTTPCACHE_DIR=/home/ubuntu/sharedata/scrapy_cached/
|
||||
scrapy crawl iafd -a cmd='astro,ethnic,dist,stu' -s HTTPCACHE_DIR=/home/ubuntu/sharedata/scrapy_cached/ -s JOBDIR=/home/ubuntu/sharedata/scrapy_job
|
||||
@ -1020,6 +1020,321 @@ class IAFDDBHandler(SQLiteDBHandler):
|
||||
return final_result
|
||||
|
||||
|
||||
@register_handler(comm.SPIDER_NAME_JAVDB)
|
||||
class JavDBHandler(SQLiteDBHandler):
|
||||
#def __init__(self, db_path=shared_db_path):
|
||||
def __init__(self, db_path=test_db_path):
|
||||
super().__init__(db_path)
|
||||
self.tbl_javdb_actors = 'javdb_actors'
|
||||
self.tbl_javdb_alias = 'javdb_actors_alias'
|
||||
self.tbl_javdb_actor_movies = 'javdb_actors_movies'
|
||||
self.tbl_javdb_makers = 'javdb_makers'
|
||||
self.tbl_javdb_movies = 'javdb_movies'
|
||||
self.tbl_javdb_movie_tags = 'javdb_movies_tags'
|
||||
self.tbl_javdb_publishers = 'javdb_publishers'
|
||||
self.tbl_javdb_series = 'javdb_series'
|
||||
self.tbl_javdb_tags = 'javdb_tags'
|
||||
|
||||
def insert_item(self, item):
|
||||
# 获取Item中所有定义的字段（包括父类继承的）
|
||||
all_fields = item.fields.keys()
|
||||
# 获取已被赋值的字段（存储在Item的内部属性_values中）
|
||||
assigned_fields = set(item._values.keys())
|
||||
# 过滤被赋值过的字段，其他预定义的字段不处理，这样在插入/更新时才不影响无关字段的值
|
||||
processed_item = {}
|
||||
for field in assigned_fields:
|
||||
processed_item[field] = item[field]
|
||||
|
||||
if isinstance(item, items_def.JavdbActorsItem):
|
||||
self.insert_or_update_actor(processed_item)
|
||||
|
||||
elif isinstance(item, items_def.JavdbMoviesItem):
|
||||
self.insert_or_update_movie(processed_item)
|
||||
|
||||
elif isinstance(item, items_def.JavdbMakersItem):
|
||||
self.insert_or_update_makers(data=processed_item)
|
||||
|
||||
elif isinstance(item, items_def.JavdbSeriesItem):
|
||||
self.insert_or_update_series(data=processed_item)
|
||||
|
||||
elif isinstance(item, items_def.JavdbPublishersItem):
|
||||
self.insert_or_update_publishers(data=processed_item)
|
||||
|
||||
else:
|
||||
logging.error(f"unkown item. {processed_item}")
|
||||
|
||||
return item
|
||||
|
||||
# 统计函数
|
||||
def get_stat(self):
|
||||
stats_config = [
|
||||
# 演员相关统计
|
||||
{'table': self.tbl_javdb_actors, 'alias': 'actors'},
|
||||
|
||||
# 电影相关统计
|
||||
{'table': self.tbl_javdb_movies, 'alias': 'movies'},
|
||||
{'table': self.tbl_javdb_movies, 'alias': 'mov_un', 'where': 'uncensored=1'},
|
||||
{'table': self.tbl_javdb_movies, 'alias': 'mov_full', 'where': 'is_full_data=1'},
|
||||
{'table': self.tbl_javdb_movies, 'alias': 'mov_un_full', 'where': 'uncensored=1 AND is_full_data=1'},
|
||||
|
||||
# 其他表统计
|
||||
{'table': self.tbl_javdb_makers, 'alias': 'makers'},
|
||||
{'table': self.tbl_javdb_publishers, 'alias': 'pubs'},
|
||||
{'table': self.tbl_javdb_series, 'alias': 'series'}
|
||||
]
|
||||
return self.generic_stats_query(stats_config)
|
||||
|
||||
|
||||
def insert_actor_index(self, name, href, from_actor_list=None, from_movie_list=None):
|
||||
fields = [
|
||||
'from_actor_list', 'from_movie_list'
|
||||
]
|
||||
data = {'name': name, 'href': href}
|
||||
# 如果没有传入值，就用原来的值
|
||||
for field in fields:
|
||||
if kwargs.get(field) is not None:
|
||||
data[field] = kwargs.get(field)
|
||||
|
||||
return self.insert_or_update_common(data=data, tbl_name=self.tbl_javdb_actors, uniq_key='href', exists_do_nothing=False)
|
||||
|
||||
def insert_movie_index(self, title, href, **kwargs):
|
||||
fields = [
|
||||
'from_actor_list', 'from_movie_makers', 'from_movie_series', 'from_movie_publishers',
|
||||
'maker_id', 'series_id', 'pub_id', 'uncensored'
|
||||
]
|
||||
data = {'title': title, 'href': href}
|
||||
# 如果没有传入值，就用原来的值
|
||||
for field in fields:
|
||||
if kwargs.get(field) is not None:
|
||||
data[field] = kwargs.get(field)
|
||||
|
||||
return self.insert_or_update_common(data=data, tbl_name=self.tbl_javdb_movies, uniq_key='href', exists_do_nothing=False)
|
||||
|
||||
# 插入演员和电影的关联数据
|
||||
def insert_actor_movie(self, performer_id, movie_id, tags=''):
|
||||
return self.insert_or_update_with_composite_pk(
|
||||
data = {'actor_id': performer_id, 'movie_id': movie_id, 'tags': tags},
|
||||
tbl_name = self.tbl_javdb_actor_movies,
|
||||
composite_pk = ['actor_id', 'movie_id'],
|
||||
exists_do_nothing = True
|
||||
)
|
||||
|
||||
# 插入演员数据
|
||||
def insert_or_update_actor(self, actor):
|
||||
try:
|
||||
actor_id = self.insert_or_update_common(data=actor, tbl_name=self.tbl_javdb_actors, uniq_key='href', exists_do_nothing=False)
|
||||
if not actor_id:
|
||||
logging.error(f"insert/update acotr error. data:{actor}")
|
||||
return None
|
||||
|
||||
# 查询刚插入的数据
|
||||
self.cursor.execute('SELECT id, from_actor_list FROM javdb_actors WHERE href = ?', (actor['href'],))
|
||||
actor_id, uncensored = self.cursor.fetchone()
|
||||
if actor_id is None:
|
||||
logging.warning(f'insert data error. name: {actor['name']}, href: {actor['href']}')
|
||||
return None
|
||||
|
||||
logging.debug(f'insert one actor, id: {actor_id}, name: {actor['name']}, href: {actor['href']}')
|
||||
|
||||
# 插入别名
|
||||
for alias in actor.get("alias") or []:
|
||||
self.insert_or_update_with_composite_pk(
|
||||
data = {'actor_id': performer_id, 'alias': alias},
|
||||
tbl_name = self.tbl_javdb_alias,
|
||||
composite_pk = ['actor_id', 'alias'],
|
||||
exists_do_nothing = True
|
||||
)
|
||||
|
||||
# 插入影片列表
|
||||
for movie in actor.get("credits") or []:
|
||||
# from_actor_list = 1 表示无码影星的，其他不处理
|
||||
if uncensored and uncensored > 0:
|
||||
movie_id = self.insert_movie_index(movie['title'], movie['href'], from_actor_list=1, uncensored=uncensored)
|
||||
else:
|
||||
movie_id = self.insert_movie_index(movie['title'], movie['href'], from_actor_list=1)
|
||||
if movie_id:
|
||||
tmp_id = self.insert_actor_movie(actor_id, movie_id)
|
||||
if tmp_id :
|
||||
logging.debug(f'insert one performer_movie, performer_id: {actor_id}, movie_id: {movie_id}')
|
||||
else:
|
||||
logging.warning(f'insert performer_movie failed. performer_id: {actor_id}, moive href: {movie['href']}')
|
||||
|
||||
return actor_id
|
||||
except Exception as e:
|
||||
logging.error(f"插入/更新演员 {actor['name']} 失败: {e}")
|
||||
self.conn.rollback()
|
||||
|
||||
# """插入或更新电影数据(异常url的处理，比如404链接)"""
|
||||
def insert_or_update_actor_404(self, name, href, is_full_data=1):
|
||||
return self.insert_or_update_common(
|
||||
data={'name': name, 'href':href, 'is_full_data':is_full_data},
|
||||
tbl_name=self.tbl_javdb_actors,
|
||||
uniq_key='href',
|
||||
exists_do_nothing=False)
|
||||
|
||||
# 查询 # TODO: 表结构需要增加个 movies_cnt 字段
|
||||
def query_actors(self, **filters):
|
||||
return self.generic_query(
|
||||
table_name = self.tbl_javdb_actors,
|
||||
fields = ['href', 'name', 'is_full_data', 'movies_cnt'],
|
||||
filters = filters
|
||||
)
|
||||
|
||||
# 插入或更新发行商 """
|
||||
def insert_or_update_makers(self, data, caller='list'):
|
||||
if caller == 'list':
|
||||
data['from_list'] = 1
|
||||
elif caller == 'movie':
|
||||
data['from_movie_list'] = 1
|
||||
|
||||
return self.insert_or_update_common(data=data, tbl_name=self.tbl_javdb_makers, uniq_key='href', exists_do_nothing=False)
|
||||
|
||||
# 按条件查询 href 列表
|
||||
def query_maker_hrefs(self, **filters):
|
||||
return self.generic_query(
|
||||
table_name = self.tbl_javdb_makers,
|
||||
fields = ['href', 'id', 'from_list'],
|
||||
filters = filters
|
||||
)
|
||||
|
||||
# """ 插入或更新制作公司 """
|
||||
def insert_or_update_series(self, data, caller='list'):
|
||||
if caller == 'list':
|
||||
data['from_list'] = 1
|
||||
elif caller == 'movie':
|
||||
data['from_movie_list'] = 1
|
||||
|
||||
return self.insert_or_update_common(data=data, tbl_name=self.tbl_javdb_series, uniq_key='href', exists_do_nothing=False)
|
||||
|
||||
# 按条件查询 href 列表
|
||||
def query_series_hrefs(self, **filters):
|
||||
return self.generic_query(
|
||||
table_name = self.tbl_javdb_series,
|
||||
fields = ['href', 'id', 'from_list'],
|
||||
filters = filters
|
||||
)
|
||||
|
||||
# 插入或更新发行商 """
|
||||
def insert_or_update_publishers(self, data, caller='list'):
|
||||
if caller == 'list':
|
||||
data['from_list'] = 1
|
||||
elif caller == 'movie':
|
||||
data['from_movie_list'] = 1
|
||||
|
||||
return self.insert_or_update_common(data=data, tbl_name=self.tbl_javdb_publishers, uniq_key='href', exists_do_nothing=False)
|
||||
|
||||
# 按条件查询 href 列表
|
||||
def query_publishers_hrefs(self, **filters):
|
||||
return self.generic_query(
|
||||
table_name = self.tbl_javdb_publishers,
|
||||
fields = ['href', 'id'],
|
||||
filters = filters
|
||||
)
|
||||
|
||||
# 插入或更新类别 """
|
||||
def insert_or_update_tags(self, name, href):
|
||||
return self.insert_or_update_common(
|
||||
data={'name': name, 'href': href},
|
||||
tbl_name=self.tbl_javdb_tags, uniq_key='href', exists_do_nothing=True
|
||||
)
|
||||
|
||||
# 查询tags
|
||||
def query_tags(self, href, name):
|
||||
return self.generic_query(
|
||||
table_name = self.tbl_javdb_tags,
|
||||
fields = ['id', 'name', 'href'],
|
||||
filters = {}
|
||||
)
|
||||
|
||||
# 插入影片和tags的关联数据
|
||||
def insert_movie_tags(self, movie_id, tag_id, tags=''):
|
||||
return self.insert_or_update_with_composite_pk(
|
||||
data = {'tag_id': tag_id, 'movie_id': movie_id, 'tags': tags},
|
||||
tbl_name = self.tbl_javdb_movie_tags,
|
||||
composite_pk = ['tag_id', 'movie_id'],
|
||||
exists_do_nothing = True
|
||||
)
|
||||
|
||||
# """插入或更新电影数据"""
|
||||
def insert_or_update_movie(self, movie):
|
||||
try:
|
||||
# 获取相关 ID
|
||||
makers_id = self.get_id_by_key(tlb=self.tbl_javdb_makers, uniq_key='href', val=movie['maker_link']) if movie['maker_link'] else None
|
||||
series_id = self.get_id_by_key(tlb=self.tbl_javdb_series, uniq_key='href', val=movie['series_link']) if movie['series_link'] else None
|
||||
pub_id = self.get_id_by_key(tlb=self.tbl_javdb_publishers, uniq_key='href', val=movie['pub_link']) if movie['pub_link'] else None
|
||||
|
||||
# 如果不存在，插入
|
||||
if makers_id is None and movie['maker_link']:
|
||||
makers_id = self.insert_or_update_makers({'name' : movie.get('maker_name', ''), 'href' : movie.get('maker_link', '')}, caller='movie')
|
||||
if series_id is None and movie['series_link']:
|
||||
series_id = self.insert_or_update_series({'name' : movie.get('series_name', ''), 'href' : movie.get('series_link', '')}, caller='movie')
|
||||
if pub_id is None and movie['pub_link']:
|
||||
pub_id = self.insert_or_update_publishers({'name' : movie.get('pub_name', ''), 'href' : movie.get('pub_link', '')}, caller='movie')
|
||||
|
||||
movie['maker_id'] = makers_id if makers_id else 0
|
||||
movie['series_id'] = series_id if series_id else 0
|
||||
movie['pub_id'] = pub_id if pub_id else 0
|
||||
movie_id = self.insert_or_update_common(movie, tbl_name=self.tbl_javdb_tags, uniq_key='href', exists_do_nothing=True)
|
||||
if movie_id is None:
|
||||
return None
|
||||
|
||||
logging.debug(f"insert one move, id: {movie_id}, title: {movie['title']}, href: {movie['href']}")
|
||||
|
||||
# 插入 performers_movies 关系表
|
||||
for performer in movie.get('actors', []):
|
||||
performer_id = self.get_id_by_key(tlb=self.tbl_javdb_actors, uniq_key='href', val=performer['href'])
|
||||
# 如果演员不存在，先插入
|
||||
if performer_id is None:
|
||||
performer_id = self.insert_actor_index(performer['name'], performer['href'], from_movie_list=1)
|
||||
logging.debug(f"insert new perfomer. perfomer_id: {performer_id}, name:{performer['name']}")
|
||||
if performer_id:
|
||||
tmp_id = self.insert_actor_movie(performer_id, movie_id)
|
||||
if tmp_id:
|
||||
logging.debug(f"insert one perfomer_movie. perfomer_id: {performer_id}, movie_id:{movie_id}")
|
||||
else:
|
||||
logging.debug(f"insert perfomer_movie failed. perfomer_id: {performer_id}, movie_id:{movie_id}")
|
||||
else:
|
||||
logging.warning(f"insert perfomer failed. name: {performer['name']}, href: {performer['href']}")
|
||||
|
||||
# 插入 tags 表
|
||||
for tag in movie.get('tags', []):
|
||||
tag_name = tag.get('name', '')
|
||||
tag_href = tag.get('href', '')
|
||||
tag_id = self.insert_or_update_tags(tag_name, tag_href)
|
||||
if tag_id:
|
||||
logging.debug(f"insert one tags. tag_id: {tag_id}, name: {tag_name}")
|
||||
tmp_id = self.insert_movie_tags(movie_id=movie_id, tag_id=tag_id, tags=tag_name)
|
||||
if tmp_id:
|
||||
logging.debug(f"insert one movie_tag. movie_id: {movie_id}, tag_id: {tag_id}, name: {tag_name}")
|
||||
else:
|
||||
logging.warning(f"insert one movie_tag error. movie_id: {movie_id}, tag_id: {tag_id}, name: {tag_name}")
|
||||
else:
|
||||
logging.warning(f"insert tags error. name:{tag_name}, href: {tag_href}")
|
||||
|
||||
return movie_id
|
||||
|
||||
except Exception as e:
|
||||
self.conn.rollback()
|
||||
logging.error("Error inserting movie: %s", e)
|
||||
return None
|
||||
|
||||
# """插入或更新电影数据(异常url的处理，比如404链接)"""
|
||||
def insert_or_update_movie_404(self, title, href, is_full_data=1):
|
||||
return self.insert_or_update_common(
|
||||
data={'title': title, 'href':href, 'is_full_data':is_full_data},
|
||||
tbl_name=self.tbl_javdb_movies,
|
||||
uniq_key='href',
|
||||
exists_do_nothing=False)
|
||||
|
||||
# 查询
|
||||
def query_movie_hrefs(self, **filters):
|
||||
return self.generic_query(
|
||||
table_name = self.tbl_javdb_movies,
|
||||
fields = ['href', 'title', 'id'],
|
||||
filters = filters
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
db = IAFDDBHandler()
|
||||
print(f"case1, 预期： 有结果：")
|
||||
|
||||
@ -398,6 +398,9 @@ class JavdbActorsItem(scrapy.Item):
|
||||
is_full_data = scrapy.Field()
|
||||
from_actor_list = scrapy.Field()
|
||||
from_movie_list = scrapy.Field()
|
||||
# 以下为添加字段
|
||||
alias = scrapy.Field()
|
||||
credits = scrapy.Field()
|
||||
|
||||
class JavdbActorsAliasItem(scrapy.Item):
|
||||
actor_id = scrapy.Field()
|
||||
@ -432,6 +435,15 @@ class JavdbMoviesItem(scrapy.Item):
|
||||
from_movie_publishers = scrapy.Field()
|
||||
pub_id = scrapy.Field()
|
||||
uncensored = scrapy.Field()
|
||||
# 以下为添加字段
|
||||
maker_name = scrapy.Field()
|
||||
maker_link = scrapy.Field()
|
||||
series_name = scrapy.Field()
|
||||
series_link = scrapy.Field()
|
||||
pub_name = scrapy.Field()
|
||||
pub_link = scrapy.Field()
|
||||
actors = scrapy.Field()
|
||||
tags = scrapy.Field()
|
||||
|
||||
class JavdbMoviesTagsItem(scrapy.Item):
|
||||
movie_id = scrapy.Field()
|
||||
|
||||
@ -164,3 +164,23 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
||||
|
||||
# Set settings whose default value is deprecated to a future-proof value
|
||||
FEED_EXPORT_ENCODING = "utf-8"
|
||||
|
||||
# 设置磁盘队列存储目录
|
||||
JOBDIR = 'crawl_state'
|
||||
|
||||
# 调度器配置
|
||||
SCHEDULER = "scrapy.core.scheduler.Scheduler"
|
||||
SCHEDULER_PERSIST = True # 是否持久化调度器状态（避免中断丢失请求）
|
||||
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
|
||||
SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" # 磁盘队列（FIFO）
|
||||
|
||||
# 如果需要 LIFO（栈式调度）
|
||||
# SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue"
|
||||
# SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue"
|
||||
|
||||
# Scrapy 自带的 AutoThrottle 可以根据服务器响应速度动态调整爬取速率：
|
||||
AUTOTHROTTLE_ENABLED = True
|
||||
AUTOTHROTTLE_START_DELAY = 1 # 初始延迟
|
||||
AUTOTHROTTLE_MAX_DELAY = 10 # 最大延迟
|
||||
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0
|
||||
AUTOTHROTTLE_DEBUG = False
|
||||
@ -1,10 +1,338 @@
|
||||
import scrapy
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import urljoin, quote_plus
|
||||
from scrapy_proj.spiders.base_spider import BaseSpider
|
||||
from scrapy_proj.items import JavdbActorsAliasItem, JavdbActorsItem, JavdbActorsMoviesItem, JavdbMakersItem, JavdbMoviesItem, JavdbMoviesTagsItem, JavdbPublishersItem, JavdbSeriesItem, JavdbTagsItem
|
||||
from scrapy_proj.db_wapper.spider_db_handler import JavDBHandler
|
||||
from scrapy_proj.comm.comm_def import SPIDER_NAME_JAVDB
|
||||
from scrapy_proj.spiders.parser.javdb_parser import common_parser
|
||||
from scrapy_proj.utils.utils import pretty_json_simple, normalize_url, generate_multilang_urls, is_valid_url
|
||||
|
||||
db_tools = JavDBHandler()
|
||||
|
||||
class JavdbSpiderSpider(BaseSpider):
|
||||
name = SPIDER_NAME_JAVDB
|
||||
allowed_domains = ["javdb.com", "www.javdb.com"]
|
||||
|
||||
# 配置请求头（复用curl中的头部信息）
|
||||
custom_settings = {
|
||||
"DEFAULT_REQUEST_HEADERS": {
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'priority': 'u=0, i',
|
||||
'referer': 'https://javdb.com/',
|
||||
'sec-ch-ua': '"Not)A;Brand";v="8", "Chromium";v="138", "Microsoft Edge";v="138"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"macOS"',
|
||||
'sec-fetch-dest': 'document',
|
||||
'sec-fetch-mode': 'navigate',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'sec-fetch-user': '?1',
|
||||
'upgrade-insecure-requests': '1',
|
||||
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0'
|
||||
},
|
||||
"COOKIES_ENABLED": True # 启用Cookie支持
|
||||
}
|
||||
|
||||
host_url = "https://www.javdb.com"
|
||||
|
||||
def __init__(self, debug='false', cmd='', mod='all', *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.debug = True if (str(debug).lower() == 'true' or str(debug).lower() == '1') else False
|
||||
self.update_mode = True if mod and mod.lower() == 'update' else False
|
||||
self.logger.info(f"RUN CMD: {' '.join(sys.argv)}")
|
||||
|
||||
self.cmd_actors = 'actors'
|
||||
self.cmd_movies = 'movies'
|
||||
self.cmd_dist = 'dist_list'
|
||||
self.cmd_list = [self.cmd_actors, self.cmd_movies, self.cmd_dist]
|
||||
if cmd and cmd != '':
|
||||
self.cmd_list = cmd.split(',')
|
||||
|
||||
self.existed_actors = {}
|
||||
self.existed_movies = {}
|
||||
self.load_existed_actors()
|
||||
self.load_existed_movies()
|
||||
|
||||
self.requested_url = set()
|
||||
|
||||
# 入口函数，由基类的方法触发
|
||||
def custom_start_requests(self):
|
||||
self.crawler.stats.set_value(f"{self.name}/actor_all", 0)
|
||||
self.crawler.stats.set_value(f"{self.name}/actor_done", 0)
|
||||
self.crawler.stats.set_value(f"{self.name}/movie_all", 0)
|
||||
self.crawler.stats.set_value(f"{self.name}/movie_done", 0)
|
||||
# 根据命令字执行
|
||||
if self.cmd_actors in self.cmd_list:
|
||||
url = urljoin(self.host_url, "/actors/uncensored")
|
||||
yield scrapy.Request(url,
|
||||
callback=self.parser_actor_list,
|
||||
headers=self.settings.get('DEFAULT_REQUEST_HEADERS'), # 使用GET头
|
||||
meta={'uncensored':1, 'from_actor_list':1, 'depth':1})
|
||||
|
||||
'''
|
||||
url = urljoin(self.host_url, "/actors/censored")
|
||||
yield scrapy.Request(url,
|
||||
callback=self.parser_actor_list,
|
||||
headers=self.settings.get('DEFAULT_REQUEST_HEADERS'), # 使用GET头
|
||||
meta={'uncensored':1, 'from_actor_list':1})
|
||||
'''
|
||||
|
||||
# 演员列表页解析
|
||||
def parser_actor_list(self, response):
|
||||
uncensored = response.meta.get('uncensored', 1)
|
||||
depth = response.meta.get('depth', 1)
|
||||
if self.debug and depth>=3:
|
||||
selef.logger.info(f'debug mode. stop next page. url: {response.url}')
|
||||
return
|
||||
data, next_url = common_parser(html=response.text, page='actor_list', href=response.url)
|
||||
if data:
|
||||
self.logger.info(f"fetched data from {response.url}, data count: {len(data)}")
|
||||
for item in data:
|
||||
url = item['href']
|
||||
name = item['name']
|
||||
# 更新对应语言的姓名
|
||||
item = JavdbActorsItem()
|
||||
item['href'] = url
|
||||
item["name"] = name
|
||||
yield item
|
||||
|
||||
# 发起查询详情，
|
||||
if self._can_request(url) :
|
||||
yield from self._create_performer_request(href=url, name=name, actor_url=url, depth=1)
|
||||
|
||||
if next_url:
|
||||
yield scrapy.Request(next_url,
|
||||
callback=self.parser_actor_list,
|
||||
headers=self.settings.get('DEFAULT_REQUEST_HEADERS'), # 使用GET头
|
||||
meta={'uncensored':1, 'from_actor_list':1, 'depth':depth+1}
|
||||
)
|
||||
else:
|
||||
self._handle_invalid_response(response, page='actor_list')
|
||||
|
||||
# 处理详细的解析页面
|
||||
def parse_actor_detail_page(self, response):
|
||||
actor_url = response.meta.get('actor_url', '')
|
||||
actor_name = response.meta.get('actor_name', '')
|
||||
depth = response.meta.get('depth', 1)
|
||||
if self.debug and depth>=3:
|
||||
selef.logger.info(f'debug mode. stop next page. url: {response.url}')
|
||||
return
|
||||
data, next_url = common_parser(html=response.text, page='actor', href=response.url)
|
||||
if data:
|
||||
self.logger.debug(f"fetched data from {response.url}, data: {data}")
|
||||
|
||||
# 判断是否需要更新: 存在完整数据，且影片数量相同
|
||||
movies_cnt = data.get('movies_cnt', 0)
|
||||
if not self.need_update_actor(href=actor_url, movies_cnt=movies_cnt):
|
||||
self.crawler.stats.inc_value(f"{self.name}/actor_done")
|
||||
self.logger.info(f"actor ({actor_name}) up to date. movies cnt: {movies_cnt} skipping... url: {actor_url}")
|
||||
return None
|
||||
|
||||
# 需要更新了，先翻页
|
||||
if next_url:
|
||||
yield from self._create_performer_request(href=next_url, name=actor_name, actor_url=actor_url, depth=depth+1)
|
||||
else:
|
||||
self.logger.info(f"actor ({actor_name}) read all pages. url :{response.url}")
|
||||
self.crawler.stats.inc_value(f"{self.name}/actor_done")
|
||||
self.add_actor_to_existed(href=actor_url, movies_cnt=movies_cnt)
|
||||
|
||||
# 更新详情数据
|
||||
item = JavdbActorsItem()
|
||||
item['href'] = actor_url
|
||||
item['name'] = actor_name
|
||||
item['from_actor_list'] = 1
|
||||
item['movies_cnt'] = movies_cnt
|
||||
item['avatar'] = data.get('avatar', {})
|
||||
item['credits'] = data.get('movies', [])
|
||||
for k, v in data.get('avatar', {}).items():
|
||||
if k in item.fields:
|
||||
item[k] = v
|
||||
yield item
|
||||
|
||||
# 影片链接，判断是否需要发起
|
||||
for item in data.get('movies', []):
|
||||
yield from self._create_movie_request(href=item['href'], title=item['title'])
|
||||
else:
|
||||
self._handle_invalid_response(response, page='actor')
|
||||
|
||||
|
||||
class JavdbSpiderSpider(scrapy.Spider):
|
||||
name = "javdb_spider"
|
||||
allowed_domains = ["www.javdb.com"]
|
||||
start_urls = ["https://www.javdb.com"]
|
||||
# 统一处理发起影片查询的请求
|
||||
def _create_performer_request(self, href, name, actor_url, depth=1):
|
||||
if href == '':
|
||||
return
|
||||
if is_valid_url(href):
|
||||
if self._can_request(href):
|
||||
self.crawler.stats.inc_value(f"{self.name}/actor_all")
|
||||
yield scrapy.Request(href,
|
||||
callback=self.parse_person_detail_page,
|
||||
meta={'actor_name': name, 'actor_url': actor_url, 'item_type':'actor', 'depth':depth }
|
||||
)
|
||||
else:
|
||||
self.logger.warning(f"wrong url. {href}, ignore...")
|
||||
|
||||
def parse(self, response):
|
||||
pass
|
||||
# 统一处理发起影片查询的请求
|
||||
def _create_movie_request(self, href, title):
|
||||
if href == '':
|
||||
return
|
||||
if is_valid_url(href):
|
||||
if self.need_update_movie(href) and self._can_request(href):
|
||||
self.crawler.stats.inc_value(f"{self.name}/movie_all")
|
||||
yield scrapy.Request(href,
|
||||
callback=self.parse_movie_detail_page,
|
||||
meta={'title': title, 'item_type':'movie', 'cache':True}
|
||||
)
|
||||
else:
|
||||
self.logger.warning(f"wrong url. {href}, ignore...")
|
||||
|
||||
# 统一处理发起影片查询的请求
|
||||
def _create_movie_list_request(self, href, name, category, depth=1):
|
||||
if href == '':
|
||||
return
|
||||
if is_valid_url(href):
|
||||
yield scrapy.Request(href,
|
||||
callback=self.parse_movie_list_page,
|
||||
meta={'name': name, 'category':category, 'depth':depth}
|
||||
)
|
||||
else:
|
||||
self.logger.warning(f"wrong url. {href}, ignore...")
|
||||
|
||||
def parse_movie_detail_page(self, response):
|
||||
title = response.meta.get('title', '')
|
||||
data = common_parser(html=response.text, page='movies', href=response.url, title=title)
|
||||
if data:
|
||||
self.crawler.stats.inc_value(f"{self.name}/movie_done")
|
||||
self.logger.debug(f"fetched data from {response.url}, data: {data}")
|
||||
# 把movies信息入库
|
||||
item = JavdbMoviesItem()
|
||||
for k, v in data.items():
|
||||
if k in item.fields:
|
||||
item[k] = v
|
||||
yield item
|
||||
|
||||
# 处理actors列表
|
||||
for actor in data.get('actors', []):
|
||||
yield from self._create_performer_request(href=actor['href'], name=actor['name'], actor_url=actor['href'], depth=1)
|
||||
|
||||
# 处理 maker
|
||||
yield from self._create_movie_list_request(href=data['maker_link'], name=data['maker_name'], category='maker', depth=1)
|
||||
|
||||
# 处理 series
|
||||
yield from self._create_movie_list_request(href=data['series_link'], name=data['series_name'], category='series', depth=1)
|
||||
|
||||
# 处理 pub
|
||||
yield from self._create_movie_list_request(href=data['pub_link'], name=data['pub_name'], category='pub', depth=1)
|
||||
|
||||
else:
|
||||
self._handle_invalid_response(response, page='movie')
|
||||
|
||||
# 处理 tags, studio, label, series 列表的公共函数
|
||||
def parse_movie_list_page(self, response):
|
||||
data, next_url = common_parser(html=response.text, page='movie_list', href=response.url)
|
||||
category = response.meta.get('category', '')
|
||||
name = response.meta.get('name', '')
|
||||
depth = response.meta.get('depth', 1)
|
||||
if self.debug and depth>=3:
|
||||
self.logger.info(f"debug mode, stop next page. url: {response.url}")
|
||||
return
|
||||
|
||||
if data:
|
||||
self.logger.debug(f"fetched data from {response.url}, data: {data}")
|
||||
# 根据 prefix 获取对应的 Item 类
|
||||
ItemClass = ITEM_MAPPING.get(prefix)
|
||||
if not ItemClass:
|
||||
self.logger.warning(f"未找到 {prefix} 对应的 Item 类")
|
||||
return None
|
||||
|
||||
# 影片链接，判断是否需要发起
|
||||
for item in data:
|
||||
yield from self._create_movie_request(href=item['href'], title=item['title'])
|
||||
|
||||
# 处理翻页
|
||||
if next_url:
|
||||
yield from self._create_movie_list_request(href=next_url, name=data['pub_name'], category='pub', depth=1)
|
||||
yield scrapy.Request(next_url,
|
||||
callback=self.parse_movie_list_page,
|
||||
headers=self.settings.get('DEFAULT_REQUEST_HEADERS'), # 使用GET头
|
||||
meta=response.meta
|
||||
)
|
||||
else:
|
||||
self.logger.info(f"movies list ({prefix}) read all pages. url :{response.url}")
|
||||
else:
|
||||
self._handle_invalid_response(response, page='movie_list')
|
||||
|
||||
|
||||
# 统一判断并处理异常
|
||||
def _handle_invalid_response(self, response, page=None):
|
||||
if response.status in [200]:
|
||||
if "404 Page Not Found" in response.text.lower():
|
||||
self.logger.warning(f"404 Page Not Found. url: {response.url}, status_code: {response.status}")
|
||||
else:
|
||||
self.logger.warning(f"unkown page. url:{response.url}, content: {response.text[:500]}")
|
||||
elif response.status in [404, 403]:
|
||||
self.logger.warning(f"get 404 page. url: {response.url}")
|
||||
else:
|
||||
self.logger.warning(f"unkown page. url:{response.url}, status: {response.status}, content: {response.text[:500]}")
|
||||
|
||||
if page:
|
||||
if page == 'actor':
|
||||
item = JavbusActorsItem()
|
||||
item['href'] = response.url
|
||||
item['zh_name'] = response.meta.get('actor_name', '')
|
||||
item['is_full_data'] = 404
|
||||
yield item
|
||||
elif page == 'movie' :
|
||||
item = JavbusMoviesItem()
|
||||
item['href'] = response.url
|
||||
item['title'] = response.meta.get('title', '')
|
||||
item['is_full_data'] = 404
|
||||
yield item
|
||||
|
||||
# TODO: 表结构需要增加个 movies_cnt 字段
|
||||
def load_existed_actors(self):
|
||||
query_args = {}
|
||||
rows = db_tools.query_actors(**query_args)
|
||||
if rows:
|
||||
for item in rows:
|
||||
self.existed_actors[item['href']] = {'is_full_data': item['is_full_data'], 'movies_cnt': item['movies_cnt']}
|
||||
else:
|
||||
self.logger.warning(f"query_actors empty. query args: {query_args}")
|
||||
|
||||
|
||||
def load_existed_movies(self):
|
||||
query_args = {}
|
||||
rows = db_tools.query_movies(**query_args)
|
||||
if rows:
|
||||
for item in rows:
|
||||
self.existed_movies[item['href']] = item['is_full_data']
|
||||
else:
|
||||
self.logger.warning(f"query_movies empty. query args: {query_args}")
|
||||
|
||||
# 内存缓存，也可以改为查询db
|
||||
def need_update_movie(self, href):
|
||||
return not (href in self.existed_movies and self.existed_movies[href] >0)
|
||||
|
||||
# 内存缓存，也可以改为查询db
|
||||
def need_update_actor(self, href, movies_cnt):
|
||||
if href not in self.existed_actors:
|
||||
return True
|
||||
data = self.existed_actors[href]
|
||||
if data['is_full_data'] <=0 :
|
||||
return True
|
||||
if data['movies_cnt'] < movies_cnt:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def add_actor_to_existed(self, href, movies_cnt, is_full_data=1):
|
||||
self.existed_actors[href] = {'is_full_data': is_full_data, 'movies_cnt': movies_cnt}
|
||||
|
||||
def acc_movie_to_existed(self, href, is_full_data=1):
|
||||
self.existed_movies[href] = is_full_data
|
||||
|
||||
def _can_request(self, href):
|
||||
if href in self.requested_url:
|
||||
return False
|
||||
self.requested_url.add(href)
|
||||
return True
|
||||
@ -10,7 +10,7 @@ import re
|
||||
from bs4 import BeautifulSoup
|
||||
from requests.exceptions import RequestException
|
||||
from functools import partial
|
||||
import config
|
||||
#import config
|
||||
#import utils
|
||||
|
||||
# 定义基础 URL 和可变参数
|
||||
@ -42,30 +42,18 @@ def common_parser(html, page, **kwargs):
|
||||
#parse_actors_uncensored(soup, href):
|
||||
#return list_data, next_url
|
||||
return parse_actors_uncensored(soup, **kwargs)
|
||||
elif page == 'series_list':
|
||||
#parse_series_uncensored(soup, href):
|
||||
#return list_data, next_url
|
||||
return parse_series_uncensored(soup, **kwargs)
|
||||
elif page == 'series':
|
||||
#parse_series_detail(soup, href):
|
||||
#return list_data, next_url
|
||||
return parse_series_detail(soup, **kwargs)
|
||||
elif page == 'makers_list':
|
||||
#parse_makers_uncensored(soup, href):
|
||||
#return list_data, next_url
|
||||
return parse_makers_uncensored(soup, **kwargs)
|
||||
elif page == 'makers':
|
||||
#parse_maker_detail(soup, href):
|
||||
#return list_data, next_url
|
||||
return parse_maker_detail(soup, **kwargs)
|
||||
elif page == 'publisher':
|
||||
#parse_publisher_detail(soup, href):
|
||||
#return list_data, next_url
|
||||
return parse_publisher_detail(soup, **kwargs)
|
||||
elif page == 'actor':
|
||||
#parse_actor_detail(soup, href):
|
||||
#return actor, next_url
|
||||
return parse_actor_detail(soup, **kwargs)
|
||||
elif page == 'makers_series_list':
|
||||
#parse_maker_series_list(soup, href, category): series / makers
|
||||
#return list_data, next_url
|
||||
return parse_maker_series_list(soup, **kwargs)
|
||||
elif page == 'movie_list':
|
||||
#parse_movie_list(soup, href):
|
||||
#return list_data, next_url
|
||||
return parse_movie_list(soup, **kwargs)
|
||||
elif page == 'movies':
|
||||
#parse_movie_detail(soup, href, title):
|
||||
#return result
|
||||
@ -226,6 +214,8 @@ def parse_actors_uncensored(soup, href):
|
||||
def parse_actor_detail(soup, href):
|
||||
# 先找一下别名
|
||||
alias_list = []
|
||||
movies_text = ''
|
||||
movies_cnt = 0
|
||||
|
||||
div_meta = soup.find('span', class_='actor-section-name')
|
||||
if not div_meta:
|
||||
@ -237,7 +227,14 @@ def parse_actor_detail(soup, href):
|
||||
meta_list = alias_div.find_all('span', class_='section-meta')
|
||||
if len(meta_list) > 1:
|
||||
alias_list = meta_list[0].text.strip().split(", ")
|
||||
|
||||
movies_text = meta_list[1].text.strip()
|
||||
try:
|
||||
match = re.search(r'(\d+)\s+movie\(s\)', movie_elem.strip(), re.IGNORECASE)
|
||||
if match:
|
||||
movies_cnt = int(match.group(1))
|
||||
except Exception as e:
|
||||
movies_cnt = 0
|
||||
|
||||
# 头像
|
||||
pic = ''
|
||||
avatar = soup.find("div", class_="column actor-avatar")
|
||||
@ -286,63 +283,14 @@ def parse_actor_detail(soup, href):
|
||||
|
||||
actor = {
|
||||
'pic' : pic,
|
||||
'alias' : alias_list,
|
||||
'movies_text' : movies_text,
|
||||
'movies_cnt' : movies_cnt,
|
||||
'movies' : list_data
|
||||
}
|
||||
|
||||
return actor, next_url
|
||||
|
||||
|
||||
# 解析 HTML 内容，提取需要的数据
|
||||
def parse_movie_detail_old(soup, href, title):
|
||||
div_video = soup.find("div", class_='video-meta-panel')
|
||||
if not div_video:
|
||||
logging.warning(f"Warning: No movies div found ")
|
||||
return None, None
|
||||
|
||||
# 获取封面图片
|
||||
cover_img = soup.select_one('.column-video-cover a')
|
||||
cover_url = cover_img['href'] if cover_img else None
|
||||
|
||||
# 获取番号
|
||||
serial = soup.select_one('.panel-block:first-child .value')
|
||||
serial_number = serial.text.strip() if serial else None
|
||||
|
||||
# 获取日期
|
||||
date = soup.select_one('.panel-block:nth-of-type(2) .value')
|
||||
release_date = date.text.strip() if date else None
|
||||
|
||||
# 获取时长
|
||||
duration = soup.select_one('.panel-block:nth-of-type(3) .value')
|
||||
video_duration = duration.text.strip() if duration else None
|
||||
|
||||
# 获取片商
|
||||
maker = soup.select_one('.panel-block:nth-of-type(4) .value a')
|
||||
maker_name = maker.text.strip() if maker else None
|
||||
maker_link = maker['href'] if maker else None
|
||||
|
||||
# 获取系列
|
||||
series = soup.select_one('.panel-block:nth-of-type(5) .value a')
|
||||
series_name = series.text.strip() if series else None
|
||||
series_link = series['href'] if series else None
|
||||
|
||||
# 获取演员（名字 + 链接）
|
||||
actors = [{'name': actor.text.strip(), 'href': host_url + actor['href']} for actor in soup.select('.panel-block:nth-of-type(8) .value a')]
|
||||
|
||||
return {
|
||||
'href' : href,
|
||||
'title' : title,
|
||||
'cover_url': cover_url,
|
||||
'serial_number': serial_number,
|
||||
'release_date': release_date,
|
||||
'duration': video_duration,
|
||||
'maker_name': maker_name,
|
||||
'maker_link': host_url + maker_link if maker_link else '',
|
||||
'series_name': series_name,
|
||||
'series_link': host_url + series_link if series_link else '',
|
||||
'actors': actors
|
||||
}
|
||||
|
||||
# 解析单个元素
|
||||
def parse_movie_one(soup, keys):
|
||||
key_strong = soup.find('strong', string=lambda text: text in keys)
|
||||
@ -413,8 +361,8 @@ def parse_movie_detail(soup, href, title):
|
||||
return result
|
||||
|
||||
# 解析 HTML 内容，提取需要的数据
|
||||
def parse_series_uncensored(soup, href):
|
||||
div_series = soup.find("div", id='series')
|
||||
def parse_maker_series_list(soup, href, category):
|
||||
div_series = soup.find("div", id=category)
|
||||
if not div_series:
|
||||
logging.warning(f"Warning: No div_series div found ")
|
||||
return None, None
|
||||
@ -453,9 +401,8 @@ def parse_series_uncensored(soup, href):
|
||||
|
||||
return list_data, next_url
|
||||
|
||||
|
||||
# 解析 HTML 内容，提取需要的数据
|
||||
def parse_series_detail(soup, href):
|
||||
def parse_movie_list(soup, href):
|
||||
#div_movies = soup.find("div", class_='movie-list h cols-4 vcols-5')
|
||||
div_movies = soup.find("div", class_=re.compile(r'movie-list h cols-4 vcols-(5|8)'))
|
||||
if not div_movies:
|
||||
@ -491,162 +438,3 @@ def parse_series_detail(soup, href):
|
||||
next_url = host_url + next_page_url
|
||||
|
||||
return list_data, next_url
|
||||
|
||||
|
||||
# 解析 HTML 内容，提取需要的数据
|
||||
def parse_makers_uncensored(soup, href):
|
||||
div_series = soup.find("div", id='makers')
|
||||
if not div_series:
|
||||
logging.warning(f"Warning: No makers div found ")
|
||||
return None, None
|
||||
|
||||
# 解析元素
|
||||
rows = div_series.find_all('a', class_='box')
|
||||
|
||||
list_data = []
|
||||
next_url = None
|
||||
for row in rows:
|
||||
name = row.find('strong').text.strip()
|
||||
href = row['href']
|
||||
div_movies = row.find('span')
|
||||
movies = 0
|
||||
if div_movies:
|
||||
match = re.search(r'\((\d+)\)', div_movies.text.strip())
|
||||
if match:
|
||||
movies = int(match.group(1))
|
||||
|
||||
list_data.append({
|
||||
'name' : name,
|
||||
'href' : host_url + href if href else '',
|
||||
'movies' : movies
|
||||
})
|
||||
|
||||
# 查找 "下一页" 按钮
|
||||
next_page_element = soup.find('a', class_='pagination-next')
|
||||
if next_page_element:
|
||||
next_page_url = next_page_element['href']
|
||||
next_page_number = url_page_num(next_page_url)
|
||||
current_page_number = url_page_num(href)
|
||||
if current_page_number is None:
|
||||
current_page_number = 0
|
||||
if next_page_number and next_page_number > current_page_number :
|
||||
next_url = host_url + next_page_url
|
||||
|
||||
return list_data, next_url
|
||||
|
||||
|
||||
# 解析 HTML 内容，提取需要的数据
|
||||
def parse_maker_detail(soup, href):
|
||||
#div_movies = soup.find("div", class_='movie-list h cols-4 vcols-5')
|
||||
div_movies = soup.find("div", class_=re.compile(r'movie-list h cols-4 vcols-(5|8)'))
|
||||
if not div_movies:
|
||||
logging.warning(f"Warning: No movies div found ")
|
||||
return [], None
|
||||
|
||||
# 解析元素
|
||||
rows = div_movies.find_all('div', class_='item')
|
||||
|
||||
list_data = []
|
||||
next_url = None
|
||||
for row in rows:
|
||||
link = row.find('a', class_='box')['href']
|
||||
serial_number = row.find('strong').text.strip()
|
||||
title = row.find('div', class_='video-title').text.strip()
|
||||
release_date = row.find('div', class_='meta').text.strip()
|
||||
list_data.append({
|
||||
'href' : host_url + link if link else '',
|
||||
'serial_number' : serial_number,
|
||||
'title' : title,
|
||||
'release_date': release_date
|
||||
})
|
||||
|
||||
# 查找 "下一页" 按钮
|
||||
next_page_element = soup.find('a', class_='pagination-next')
|
||||
if next_page_element:
|
||||
next_page_url = next_page_element['href']
|
||||
next_page_number = url_page_num(next_page_url)
|
||||
current_page_number = url_page_num(href)
|
||||
if current_page_number is None:
|
||||
current_page_number = 0
|
||||
if next_page_number and next_page_number > current_page_number :
|
||||
next_url = host_url + next_page_url
|
||||
|
||||
return list_data, next_url
|
||||
|
||||
# 解析 HTML 内容，提取需要的数据
|
||||
def parse_publisher_detail(soup, href):
|
||||
#div_movies = soup.find("div", class_='movie-list h cols-4 vcols-5')
|
||||
div_movies = soup.find("div", class_=re.compile(r'movie-list h cols-4 vcols-(5|8)'))
|
||||
if not div_movies:
|
||||
logging.warning(f"Warning: No movies div found ")
|
||||
return [], None
|
||||
|
||||
# 解析元素
|
||||
rows = div_movies.find_all('div', class_='item')
|
||||
|
||||
list_data = []
|
||||
next_url = None
|
||||
for row in rows:
|
||||
link = row.find('a', class_='box')['href']
|
||||
serial_number = row.find('strong').text.strip()
|
||||
title = row.find('div', class_='video-title').text.strip()
|
||||
release_date = row.find('div', class_='meta').text.strip()
|
||||
list_data.append({
|
||||
'href' : host_url + link if link else '',
|
||||
'serial_number' : serial_number,
|
||||
'title' : title,
|
||||
'release_date': release_date
|
||||
})
|
||||
|
||||
# 查找 "下一页" 按钮
|
||||
next_page_element = soup.find('a', class_='pagination-next')
|
||||
if next_page_element:
|
||||
next_page_url = next_page_element['href']
|
||||
next_page_number = url_page_num(next_page_url)
|
||||
current_page_number = url_page_num(href)
|
||||
if current_page_number is None:
|
||||
current_page_number = 0
|
||||
if next_page_number and next_page_number > current_page_number :
|
||||
next_url = host_url + next_page_url
|
||||
|
||||
return list_data, next_url
|
||||
|
||||
|
||||
# 解析 HTML 内容，提取需要的数据
|
||||
def parse_uncensored(soup, href):
|
||||
#div_movies = soup.find("div", class_='movie-list h cols-4 vcols-8')
|
||||
div_movies = soup.find("div", class_=re.compile(r'movie-list h cols-4 vcols-(5|8)'))
|
||||
if not div_movies:
|
||||
logging.warning(f"Warning: No movies div found ")
|
||||
return [], None
|
||||
|
||||
# 解析元素
|
||||
rows = div_movies.find_all('div', class_='item')
|
||||
|
||||
list_data = []
|
||||
next_url = None
|
||||
for row in rows:
|
||||
link = row.find('a', class_='box')['href']
|
||||
serial_number = row.find('strong').text.strip()
|
||||
title = row.find('div', class_='video-title').text.strip()
|
||||
release_date = row.find('div', class_='meta').text.strip()
|
||||
list_data.append({
|
||||
'href' : host_url + link if link else '',
|
||||
'serial_number' : serial_number,
|
||||
'title' : title,
|
||||
'release_date': release_date
|
||||
})
|
||||
|
||||
# 查找 "下一页" 按钮
|
||||
next_page_element = soup.find('a', class_='pagination-next')
|
||||
if next_page_element:
|
||||
next_page_url = next_page_element['href']
|
||||
next_page_number = url_page_num(next_page_url)
|
||||
current_page_number = url_page_num(href)
|
||||
if current_page_number is None:
|
||||
current_page_number = 0
|
||||
if next_page_number and next_page_number > current_page_number :
|
||||
next_url = host_url + next_page_url
|
||||
|
||||
return list_data, next_url
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ import scrapy_proj.comm.comm_def as comm
|
||||
import scrapy_proj.items as items_def
|
||||
from scrapy_proj.utils.utils import pretty_json_simple, is_valid_url
|
||||
|
||||
class IAFDDBHandler(SQLiteDBHandler):
|
||||
class SharedBHandler(SQLiteDBHandler):
|
||||
def __init__(self, db_path=shared_db_path):
|
||||
super().__init__(db_path)
|
||||
self.tbl_name_performers = 'iafd_performers'
|
||||
|
||||
147
scrapy_proj/sqlalchemy/alembic.ini
Normal file
147
scrapy_proj/sqlalchemy/alembic.ini
Normal file
@ -0,0 +1,147 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts.
|
||||
# this is typically a path given in POSIX (e.g. forward slashes)
|
||||
# format, relative to the token %(here)s which refers to the location of this
|
||||
# ini file
|
||||
script_location = %(here)s
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory. for multiple paths, the path separator
|
||||
# is defined by "path_separator" below.
|
||||
prepend_sys_path = .
|
||||
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library.
|
||||
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to <script_location>/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "path_separator"
|
||||
# below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
|
||||
|
||||
# path_separator; This indicates what character is used to split lists of file
|
||||
# paths, including version_locations and prepend_sys_path within configparser
|
||||
# files such as alembic.ini.
|
||||
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
|
||||
# to provide os-dependent path splitting.
|
||||
#
|
||||
# Note that in order to support legacy alembic.ini files, this default does NOT
|
||||
# take place if path_separator is not present in alembic.ini. If this
|
||||
# option is omitted entirely, fallback logic is as follows:
|
||||
#
|
||||
# 1. Parsing of the version_locations option falls back to using the legacy
|
||||
# "version_path_separator" key, which if absent then falls back to the legacy
|
||||
# behavior of splitting on spaces and/or commas.
|
||||
# 2. Parsing of the prepend_sys_path option falls back to the legacy
|
||||
# behavior of splitting on spaces, commas, or colons.
|
||||
#
|
||||
# Valid values for path_separator are:
|
||||
#
|
||||
# path_separator = :
|
||||
# path_separator = ;
|
||||
# path_separator = space
|
||||
# path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
path_separator = os
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
|
||||
# hooks = ruff
|
||||
# ruff.type = module
|
||||
# ruff.module = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Alternatively, use the exec runner to execute a binary found on your PATH
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration. This is also consumed by the user-maintained
|
||||
# env.py script only.
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
1
scrapy_proj/sqlalchemy/migrations/scrapy/README
Normal file
1
scrapy_proj/sqlalchemy/migrations/scrapy/README
Normal file
@ -0,0 +1 @@
|
||||
Generic single-database configuration.
|
||||
147
scrapy_proj/sqlalchemy/migrations/scrapy/alembic.ini
Normal file
147
scrapy_proj/sqlalchemy/migrations/scrapy/alembic.ini
Normal file
@ -0,0 +1,147 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts.
|
||||
# this is typically a path given in POSIX (e.g. forward slashes)
|
||||
# format, relative to the token %(here)s which refers to the location of this
|
||||
# ini file
|
||||
script_location = %(here)s
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory. for multiple paths, the path separator
|
||||
# is defined by "path_separator" below.
|
||||
prepend_sys_path = .
|
||||
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library.
|
||||
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to <script_location>/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "path_separator"
|
||||
# below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
|
||||
|
||||
# path_separator; This indicates what character is used to split lists of file
|
||||
# paths, including version_locations and prepend_sys_path within configparser
|
||||
# files such as alembic.ini.
|
||||
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
|
||||
# to provide os-dependent path splitting.
|
||||
#
|
||||
# Note that in order to support legacy alembic.ini files, this default does NOT
|
||||
# take place if path_separator is not present in alembic.ini. If this
|
||||
# option is omitted entirely, fallback logic is as follows:
|
||||
#
|
||||
# 1. Parsing of the version_locations option falls back to using the legacy
|
||||
# "version_path_separator" key, which if absent then falls back to the legacy
|
||||
# behavior of splitting on spaces and/or commas.
|
||||
# 2. Parsing of the prepend_sys_path option falls back to the legacy
|
||||
# behavior of splitting on spaces, commas, or colons.
|
||||
#
|
||||
# Valid values for path_separator are:
|
||||
#
|
||||
# path_separator = :
|
||||
# path_separator = ;
|
||||
# path_separator = space
|
||||
# path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
path_separator = os
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
|
||||
# hooks = ruff
|
||||
# ruff.type = module
|
||||
# ruff.module = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Alternatively, use the exec runner to execute a binary found on your PATH
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration. This is also consumed by the user-maintained
|
||||
# env.py script only.
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
100
scrapy_proj/sqlalchemy/migrations/scrapy/env.py
Normal file
100
scrapy_proj/sqlalchemy/migrations/scrapy/env.py
Normal file
@ -0,0 +1,100 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = None
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
''' 修改点
|
||||
from models.modelclass_b import Base
|
||||
target_metadata = Base.metadata
|
||||
|
||||
def run_migrations_online():
|
||||
url = "sqlite:///../databases/db_b.db"
|
||||
connectable = create_engine(url)
|
||||
# 保持其他代码不变
|
||||
'''
|
||||
|
||||
import os
|
||||
from alembic import context
|
||||
from sqlalchemy import create_engine
|
||||
from logging.config import fileConfig
|
||||
from models.scrapy import Base
|
||||
target_metadata = Base.metadata
|
||||
|
||||
home_dir = os.path.expanduser("~")
|
||||
global_share_data_dir = f'{home_dir}/sharedata/sqlite'
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
"""
|
||||
|
||||
url = f"sqlite:///{global_share_data_dir}/scrapy.db"
|
||||
connectable = create_engine(url)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection, target_metadata=target_metadata
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
28
scrapy_proj/sqlalchemy/migrations/scrapy/script.py.mako
Normal file
28
scrapy_proj/sqlalchemy/migrations/scrapy/script.py.mako
Normal file
@ -0,0 +1,28 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
1
scrapy_proj/sqlalchemy/migrations/shared_comm/README
Normal file
1
scrapy_proj/sqlalchemy/migrations/shared_comm/README
Normal file
@ -0,0 +1 @@
|
||||
Generic single-database configuration.
|
||||
147
scrapy_proj/sqlalchemy/migrations/shared_comm/alembic.ini
Normal file
147
scrapy_proj/sqlalchemy/migrations/shared_comm/alembic.ini
Normal file
@ -0,0 +1,147 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts.
|
||||
# this is typically a path given in POSIX (e.g. forward slashes)
|
||||
# format, relative to the token %(here)s which refers to the location of this
|
||||
# ini file
|
||||
script_location = %(here)s
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory. for multiple paths, the path separator
|
||||
# is defined by "path_separator" below.
|
||||
prepend_sys_path = .
|
||||
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library.
|
||||
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to <script_location>/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "path_separator"
|
||||
# below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
|
||||
|
||||
# path_separator; This indicates what character is used to split lists of file
|
||||
# paths, including version_locations and prepend_sys_path within configparser
|
||||
# files such as alembic.ini.
|
||||
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
|
||||
# to provide os-dependent path splitting.
|
||||
#
|
||||
# Note that in order to support legacy alembic.ini files, this default does NOT
|
||||
# take place if path_separator is not present in alembic.ini. If this
|
||||
# option is omitted entirely, fallback logic is as follows:
|
||||
#
|
||||
# 1. Parsing of the version_locations option falls back to using the legacy
|
||||
# "version_path_separator" key, which if absent then falls back to the legacy
|
||||
# behavior of splitting on spaces and/or commas.
|
||||
# 2. Parsing of the prepend_sys_path option falls back to the legacy
|
||||
# behavior of splitting on spaces, commas, or colons.
|
||||
#
|
||||
# Valid values for path_separator are:
|
||||
#
|
||||
# path_separator = :
|
||||
# path_separator = ;
|
||||
# path_separator = space
|
||||
# path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
path_separator = os
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
|
||||
# hooks = ruff
|
||||
# ruff.type = module
|
||||
# ruff.module = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Alternatively, use the exec runner to execute a binary found on your PATH
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration. This is also consumed by the user-maintained
|
||||
# env.py script only.
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
101
scrapy_proj/sqlalchemy/migrations/shared_comm/env.py
Normal file
101
scrapy_proj/sqlalchemy/migrations/shared_comm/env.py
Normal file
@ -0,0 +1,101 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = None
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
''' 修改点
|
||||
from models.modelclass_b import Base
|
||||
target_metadata = Base.metadata
|
||||
|
||||
def run_migrations_online():
|
||||
url = "sqlite:///../databases/db_b.db"
|
||||
connectable = create_engine(url)
|
||||
# 保持其他代码不变
|
||||
'''
|
||||
|
||||
import os
|
||||
from alembic import context
|
||||
from sqlalchemy import create_engine
|
||||
from logging.config import fileConfig
|
||||
from models.shared import Base
|
||||
target_metadata = Base.metadata
|
||||
|
||||
home_dir = os.path.expanduser("~")
|
||||
global_share_data_dir = f'{home_dir}/sharedata/sqlite'
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
"""
|
||||
|
||||
url = f"sqlite:///{global_share_data_dir}/shared.db"
|
||||
connectable = create_engine(url)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection, target_metadata=target_metadata
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
28
scrapy_proj/sqlalchemy/migrations/shared_comm/script.py.mako
Normal file
28
scrapy_proj/sqlalchemy/migrations/shared_comm/script.py.mako
Normal file
@ -0,0 +1,28 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
@ -0,0 +1,34 @@
|
||||
"""Auto update from shared_comm
|
||||
|
||||
Revision ID: 098c67f16b5e
|
||||
Revises:
|
||||
Create Date: 2025-07-29 20:01:58.175086
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '098c67f16b5e'
|
||||
down_revision: Union[str, Sequence[str], None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column('javdb_actors', sa.Column('movies_cnt', sa.Integer(), server_default=sa.text('0'), nullable=False))
|
||||
op.add_column('javdb_actors', sa.Column('uncensored', sa.Integer(), server_default=sa.text('0'), nullable=False))
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_column('javdb_actors', 'uncensored')
|
||||
op.drop_column('javdb_actors', 'movies_cnt')
|
||||
# ### end Alembic commands ###
|
||||
1
scrapy_proj/sqlalchemy/migrations/testdb/README
Normal file
1
scrapy_proj/sqlalchemy/migrations/testdb/README
Normal file
@ -0,0 +1 @@
|
||||
Generic single-database configuration.
|
||||
147
scrapy_proj/sqlalchemy/migrations/testdb/alembic.ini
Normal file
147
scrapy_proj/sqlalchemy/migrations/testdb/alembic.ini
Normal file
@ -0,0 +1,147 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts.
|
||||
# this is typically a path given in POSIX (e.g. forward slashes)
|
||||
# format, relative to the token %(here)s which refers to the location of this
|
||||
# ini file
|
||||
script_location = %(here)s
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory. for multiple paths, the path separator
|
||||
# is defined by "path_separator" below.
|
||||
prepend_sys_path = .
|
||||
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library.
|
||||
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to <script_location>/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "path_separator"
|
||||
# below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
|
||||
|
||||
# path_separator; This indicates what character is used to split lists of file
|
||||
# paths, including version_locations and prepend_sys_path within configparser
|
||||
# files such as alembic.ini.
|
||||
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
|
||||
# to provide os-dependent path splitting.
|
||||
#
|
||||
# Note that in order to support legacy alembic.ini files, this default does NOT
|
||||
# take place if path_separator is not present in alembic.ini. If this
|
||||
# option is omitted entirely, fallback logic is as follows:
|
||||
#
|
||||
# 1. Parsing of the version_locations option falls back to using the legacy
|
||||
# "version_path_separator" key, which if absent then falls back to the legacy
|
||||
# behavior of splitting on spaces and/or commas.
|
||||
# 2. Parsing of the prepend_sys_path option falls back to the legacy
|
||||
# behavior of splitting on spaces, commas, or colons.
|
||||
#
|
||||
# Valid values for path_separator are:
|
||||
#
|
||||
# path_separator = :
|
||||
# path_separator = ;
|
||||
# path_separator = space
|
||||
# path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
path_separator = os
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
|
||||
# hooks = ruff
|
||||
# ruff.type = module
|
||||
# ruff.module = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Alternatively, use the exec runner to execute a binary found on your PATH
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration. This is also consumed by the user-maintained
|
||||
# env.py script only.
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
108
scrapy_proj/sqlalchemy/migrations/testdb/env.py
Normal file
108
scrapy_proj/sqlalchemy/migrations/testdb/env.py
Normal file
@ -0,0 +1,108 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = None
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
''' 修改点
|
||||
from models.modelclass_b import Base
|
||||
target_metadata = Base.metadata
|
||||
|
||||
def run_migrations_online():
|
||||
url = "sqlite:///../databases/db_b.db"
|
||||
connectable = create_engine(url)
|
||||
# 保持其他代码不变
|
||||
'''
|
||||
import os
|
||||
from alembic import context
|
||||
from sqlalchemy import create_engine
|
||||
from logging.config import fileConfig
|
||||
from models.shared import Base as BaseShared
|
||||
from models.scrapy import Base as BaseScrapy
|
||||
|
||||
# 合并元数据
|
||||
from sqlalchemy import MetaData
|
||||
target_metadata = MetaData()
|
||||
for t in BaseShared.metadata.tables.values():
|
||||
t.tometadata(target_metadata)
|
||||
for t in BaseScrapy.metadata.tables.values():
|
||||
t.tometadata(target_metadata)
|
||||
|
||||
home_dir = os.path.expanduser("~")
|
||||
global_share_data_dir = f'{home_dir}/sharedata/sqlite'
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
"""
|
||||
|
||||
url = f"sqlite:///{global_share_data_dir}/test.db"
|
||||
connectable = create_engine(url)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection, target_metadata=target_metadata
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
28
scrapy_proj/sqlalchemy/migrations/testdb/script.py.mako
Normal file
28
scrapy_proj/sqlalchemy/migrations/testdb/script.py.mako
Normal file
@ -0,0 +1,28 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
@ -0,0 +1,34 @@
|
||||
"""Initial test schema
|
||||
|
||||
Revision ID: 854378c8e332
|
||||
Revises: bce23e0d0c3a
|
||||
Create Date: 2025-07-29 19:58:01.503647
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '854378c8e332'
|
||||
down_revision: Union[str, Sequence[str], None] = 'bce23e0d0c3a'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column('javdb_actors', sa.Column('movies_cnt', sa.Integer(), server_default=sa.text('0'), nullable=False))
|
||||
op.add_column('javdb_actors', sa.Column('uncensored', sa.Integer(), server_default=sa.text('0'), nullable=False))
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_column('javdb_actors', 'uncensored')
|
||||
op.drop_column('javdb_actors', 'movies_cnt')
|
||||
# ### end Alembic commands ###
|
||||
@ -0,0 +1,674 @@
|
||||
"""Initial test schema
|
||||
|
||||
Revision ID: 86eea10972c0
|
||||
Revises:
|
||||
Create Date: 2025-07-29 19:25:41.776214
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '86eea10972c0'
|
||||
down_revision: Union[str, Sequence[str], None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('clm_index',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('category', sa.Text(), nullable=True),
|
||||
sa.Column('title', sa.Text(), nullable=True),
|
||||
sa.Column('href', sa.Text(), nullable=True),
|
||||
sa.Column('magnet_href', sa.Text(), nullable=True),
|
||||
sa.Column('size_text', sa.Text(), nullable=True),
|
||||
sa.Column('size_gb', sa.REAL(), nullable=True),
|
||||
sa.Column('heat', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('add_date', sa.Text(), nullable=True),
|
||||
sa.Column('last_down_date', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('href')
|
||||
)
|
||||
op.create_table('clm_keywords',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('words', sa.Text(), nullable=True),
|
||||
sa.Column('groups', sa.Text(), nullable=True),
|
||||
sa.Column('tags', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('index_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('words')
|
||||
)
|
||||
op.create_table('iafd_distributors',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('details', sa.Text(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('iafd_meta_ethnic',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('iafd_performers',
|
||||
sa.Column('name', sa.Text(), nullable=False),
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('birth_year', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_astro_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_birth_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_ethnic_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('gender', sa.Text(), nullable=True),
|
||||
sa.Column('birthday', sa.Text(), nullable=True),
|
||||
sa.Column('astrology', sa.Text(), nullable=True),
|
||||
sa.Column('birthplace', sa.Text(), nullable=True),
|
||||
sa.Column('years_active', sa.Text(), nullable=True),
|
||||
sa.Column('ethnicity', sa.Text(), nullable=True),
|
||||
sa.Column('nationality', sa.Text(), nullable=True),
|
||||
sa.Column('hair_colors', sa.Text(), nullable=True),
|
||||
sa.Column('eye_color', sa.Text(), nullable=True),
|
||||
sa.Column('height_str', sa.Text(), nullable=True),
|
||||
sa.Column('weight_str', sa.Text(), nullable=True),
|
||||
sa.Column('measurements', sa.Text(), nullable=True),
|
||||
sa.Column('tattoos', sa.Text(), nullable=True),
|
||||
sa.Column('piercings', sa.Text(), nullable=True),
|
||||
sa.Column('fake_tits', sa.Text(), nullable=True),
|
||||
sa.Column('href', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('weight', sa.Integer(), nullable=True),
|
||||
sa.Column('height', sa.Integer(), nullable=True),
|
||||
sa.Column('rating', sa.Integer(), nullable=True),
|
||||
sa.Column('movies_cnt', sa.Integer(), nullable=True),
|
||||
sa.Column('vixen_cnt', sa.Integer(), nullable=True),
|
||||
sa.Column('blacked_cnt', sa.Integer(), nullable=True),
|
||||
sa.Column('tushy_cnt', sa.Integer(), nullable=True),
|
||||
sa.Column('x_art_cnt', sa.Integer(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('href')
|
||||
)
|
||||
op.create_table('iafd_studios',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('details', sa.Text(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('iafd_task_log',
|
||||
sa.Column('task_id', sa.Integer(), nullable=False),
|
||||
sa.Column('full_data_performers', sa.Integer(), nullable=True),
|
||||
sa.Column('total_performers', sa.Integer(), nullable=True),
|
||||
sa.Column('full_data_movies', sa.Integer(), nullable=True),
|
||||
sa.Column('total_movies', sa.Integer(), nullable=True),
|
||||
sa.Column('total_distributors', sa.Integer(), nullable=True),
|
||||
sa.Column('total_studios', sa.Integer(), nullable=True),
|
||||
sa.Column('task_status', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('task_id')
|
||||
)
|
||||
op.create_table('javbus_actors',
|
||||
sa.Column('uncensored', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_actor_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('movies_cnt', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('ja_name', sa.Text(), nullable=True),
|
||||
sa.Column('zh_name', sa.Text(), nullable=True),
|
||||
sa.Column('en_name', sa.Text(), nullable=True),
|
||||
sa.Column('href', sa.Text(), nullable=True),
|
||||
sa.Column('pic', sa.Text(), nullable=True),
|
||||
sa.Column('birth_date', sa.Text(), nullable=True),
|
||||
sa.Column('height', sa.Text(), nullable=True),
|
||||
sa.Column('breast_size', sa.Text(), nullable=True),
|
||||
sa.Column('measurements', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('href')
|
||||
)
|
||||
op.create_table('javbus_labels',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=True),
|
||||
sa.Column('en_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('ja_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('details', sa.Text(), nullable=True),
|
||||
sa.Column('uncensored', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('movies_cnt', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('magnet_cnt', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('javbus_movies',
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('uncensored', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_actor_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_studios', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_labels', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_series', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('actors_cnt', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('href', sa.Text(), nullable=True),
|
||||
sa.Column('title', sa.Text(), nullable=True),
|
||||
sa.Column('cover_url', sa.Text(), nullable=True),
|
||||
sa.Column('serial_number', sa.Text(), nullable=True),
|
||||
sa.Column('release_date', sa.Text(), nullable=True),
|
||||
sa.Column('duration', sa.Text(), nullable=True),
|
||||
sa.Column('studio_id', sa.Integer(), nullable=True),
|
||||
sa.Column('label_id', sa.Integer(), nullable=True),
|
||||
sa.Column('series_id', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('href')
|
||||
)
|
||||
op.create_table('javbus_series',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=True),
|
||||
sa.Column('en_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('ja_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('details', sa.Text(), nullable=True),
|
||||
sa.Column('uncensored', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('movies_cnt', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('magnet_cnt', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('javbus_studios',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=True),
|
||||
sa.Column('en_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('ja_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('details', sa.Text(), nullable=True),
|
||||
sa.Column('uncensored', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('movies_cnt', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('magnet_cnt', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('javbus_tags',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=True),
|
||||
sa.Column('en_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('ja_name', sa.String(length=255), nullable=True),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('javdb_actors',
|
||||
sa.Column('name', sa.Text(), nullable=False),
|
||||
sa.Column('href', sa.Text(), nullable=False),
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_actor_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('pic', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.DateTime(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('href')
|
||||
)
|
||||
op.create_table('javdb_makers',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('details', sa.Text(), nullable=True),
|
||||
sa.Column('from_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('javdb_movies',
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_actor_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_makers', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_series', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_movie_publishers', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('uncensored', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('href', sa.Text(), nullable=True),
|
||||
sa.Column('title', sa.Text(), nullable=True),
|
||||
sa.Column('cover_url', sa.Text(), nullable=True),
|
||||
sa.Column('serial_number', sa.Text(), nullable=True),
|
||||
sa.Column('release_date', sa.Text(), nullable=True),
|
||||
sa.Column('duration', sa.Text(), nullable=True),
|
||||
sa.Column('maker_id', sa.Text(), nullable=True),
|
||||
sa.Column('series_id', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('pub_id', sa.Integer(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('href')
|
||||
)
|
||||
op.create_table('javdb_publishers',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('details', sa.Text(), nullable=True),
|
||||
sa.Column('from_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('javdb_series',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('details', sa.Text(), nullable=True),
|
||||
sa.Column('from_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.Column('from_movie_list', sa.Integer(), server_default=sa.text('0'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('javdb_tags',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('javdb_task_log',
|
||||
sa.Column('task_id', sa.Integer(), nullable=False),
|
||||
sa.Column('full_data_actors', sa.Integer(), nullable=True),
|
||||
sa.Column('total_actors', sa.Integer(), nullable=True),
|
||||
sa.Column('full_data_movies', sa.Integer(), nullable=True),
|
||||
sa.Column('total_movies', sa.Integer(), nullable=True),
|
||||
sa.Column('total_makers', sa.Integer(), nullable=True),
|
||||
sa.Column('total_series', sa.Integer(), nullable=True),
|
||||
sa.Column('task_status', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('task_id')
|
||||
)
|
||||
op.create_table('javhd_models',
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('rank', sa.Integer(), nullable=True),
|
||||
sa.Column('ja_name', sa.Text(), nullable=True),
|
||||
sa.Column('zh_name', sa.Text(), nullable=True),
|
||||
sa.Column('en_name', sa.Text(), nullable=True),
|
||||
sa.Column('url', sa.Text(), nullable=True),
|
||||
sa.Column('pic', sa.Text(), nullable=True),
|
||||
sa.Column('height', sa.Text(), nullable=True),
|
||||
sa.Column('weight', sa.Text(), nullable=True),
|
||||
sa.Column('breast_size', sa.Text(), nullable=True),
|
||||
sa.Column('breast_factor', sa.Text(), nullable=True),
|
||||
sa.Column('hair_color', sa.Text(), nullable=True),
|
||||
sa.Column('eye_color', sa.Text(), nullable=True),
|
||||
sa.Column('birth_date', sa.Text(), nullable=True),
|
||||
sa.Column('ethnicity', sa.Text(), nullable=True),
|
||||
sa.Column('birth_place', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('url')
|
||||
)
|
||||
op.create_table('pbox_actors',
|
||||
sa.Column('name', sa.Text(), nullable=False),
|
||||
sa.Column('movies_cnt', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('href', sa.Text(), nullable=True),
|
||||
sa.Column('gender', sa.Text(), nullable=True),
|
||||
sa.Column('age', sa.Integer(), nullable=True),
|
||||
sa.Column('nationality', sa.Text(), nullable=True),
|
||||
sa.Column('country', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('href')
|
||||
)
|
||||
op.create_table('pbox_studios',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('label_id', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('scene_count', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('pbox_tags',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('sis',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('plate_name', sa.Text(), nullable=True),
|
||||
sa.Column('title', sa.Text(), nullable=True),
|
||||
sa.Column('url', sa.Text(), nullable=True),
|
||||
sa.Column('size_text', sa.Text(), nullable=True),
|
||||
sa.Column('size_gb', sa.REAL(), nullable=True),
|
||||
sa.Column('update_date', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('url')
|
||||
)
|
||||
op.create_table('thelordofporn_actress',
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('pornstar', sa.Text(), nullable=True),
|
||||
sa.Column('rating', sa.REAL(), nullable=True),
|
||||
sa.Column('rank', sa.Integer(), nullable=True),
|
||||
sa.Column('votes', sa.Integer(), nullable=True),
|
||||
sa.Column('href', sa.Text(), nullable=True),
|
||||
sa.Column('career_start', sa.Text(), nullable=True),
|
||||
sa.Column('measurements', sa.Text(), nullable=True),
|
||||
sa.Column('born', sa.Text(), nullable=True),
|
||||
sa.Column('height', sa.Text(), nullable=True),
|
||||
sa.Column('weight', sa.Text(), nullable=True),
|
||||
sa.Column('date_modified', sa.Text(), nullable=True),
|
||||
sa.Column('global_rank', sa.Integer(), nullable=True),
|
||||
sa.Column('weekly_rank', sa.Integer(), nullable=True),
|
||||
sa.Column('last_month_rating', sa.REAL(), nullable=True),
|
||||
sa.Column('current_rating', sa.REAL(), nullable=True),
|
||||
sa.Column('total_votes', sa.Integer(), nullable=True),
|
||||
sa.Column('birth_date', sa.Text(), nullable=True),
|
||||
sa.Column('birth_year', sa.Text(), nullable=True),
|
||||
sa.Column('birth_place', sa.Text(), nullable=True),
|
||||
sa.Column('height_ft', sa.Text(), nullable=True),
|
||||
sa.Column('height_cm', sa.Text(), nullable=True),
|
||||
sa.Column('weight_lbs', sa.Text(), nullable=True),
|
||||
sa.Column('weight_kg', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('href')
|
||||
)
|
||||
op.create_table('u3c3',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('category', sa.Text(), nullable=True),
|
||||
sa.Column('title', sa.Text(), nullable=True),
|
||||
sa.Column('url', sa.Text(), nullable=True),
|
||||
sa.Column('torrent_url', sa.Text(), nullable=True),
|
||||
sa.Column('magnet_url', sa.Text(), nullable=True),
|
||||
sa.Column('size_text', sa.Text(), nullable=True),
|
||||
sa.Column('size_gb', sa.REAL(), nullable=True),
|
||||
sa.Column('update_date', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('url')
|
||||
)
|
||||
op.create_table('clm_keywords_index',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('words_id', sa.Integer(), nullable=True),
|
||||
sa.Column('index_id', sa.Integer(), nullable=True),
|
||||
sa.Column('wid_iid', sa.String(length=255), nullable=True),
|
||||
sa.Column('tags', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['index_id'], ['clm_index.id'], ),
|
||||
sa.ForeignKeyConstraint(['words_id'], ['clm_keywords.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('iafd_movies',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('release_year', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_performer_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_dist_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('from_stu_list', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('title', sa.String(length=255), nullable=True),
|
||||
sa.Column('minutes', sa.String(length=255), nullable=True),
|
||||
sa.Column('distributor_id', sa.Integer(), nullable=True),
|
||||
sa.Column('studio_id', sa.Integer(), nullable=True),
|
||||
sa.Column('release_date', sa.String(length=255), nullable=True),
|
||||
sa.Column('added_to_IAFD_date', sa.String(length=255), nullable=True),
|
||||
sa.Column('all_girl', sa.String(length=255), nullable=True),
|
||||
sa.Column('all_male', sa.String(length=255), nullable=True),
|
||||
sa.Column('compilation', sa.String(length=255), nullable=True),
|
||||
sa.Column('webscene', sa.String(length=255), nullable=True),
|
||||
sa.Column('director_id', sa.Integer(), nullable=True),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['distributor_id'], ['iafd_distributors.id'], ),
|
||||
sa.ForeignKeyConstraint(['studio_id'], ['iafd_studios.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('iafd_performer_aliases',
|
||||
sa.Column('performer_id', sa.Integer(), nullable=False),
|
||||
sa.Column('alias', sa.String(length=255), nullable=False),
|
||||
sa.ForeignKeyConstraint(['performer_id'], ['iafd_performers.id'], ),
|
||||
sa.PrimaryKeyConstraint('performer_id', 'alias')
|
||||
)
|
||||
op.create_table('iafd_performer_urls',
|
||||
sa.Column('performer_id', sa.Integer(), nullable=False),
|
||||
sa.Column('position', sa.String(length=255), nullable=False),
|
||||
sa.Column('url', sa.String(length=255), nullable=False),
|
||||
sa.ForeignKeyConstraint(['performer_id'], ['iafd_performers.id'], ),
|
||||
sa.PrimaryKeyConstraint('performer_id', 'position', 'url')
|
||||
)
|
||||
op.create_table('javbus_actors_movies',
|
||||
sa.Column('actor_id', sa.Integer(), nullable=False),
|
||||
sa.Column('movie_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tags', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['actor_id'], ['javbus_actors.id'], ),
|
||||
sa.ForeignKeyConstraint(['movie_id'], ['javbus_movies.id'], ),
|
||||
sa.PrimaryKeyConstraint('actor_id', 'movie_id')
|
||||
)
|
||||
op.create_index('idx_actor_movie_actor_id', 'javbus_actors_movies', ['actor_id'], unique=False)
|
||||
op.create_table('javbus_movies_tags',
|
||||
sa.Column('movie_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tags', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['movie_id'], ['javbus_movies.id'], ),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['javbus_tags.id'], ),
|
||||
sa.PrimaryKeyConstraint('movie_id', 'tag_id')
|
||||
)
|
||||
op.create_table('javdb_actors_alias',
|
||||
sa.Column('actor_id', sa.Integer(), nullable=False),
|
||||
sa.Column('alias', sa.Text(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.DateTime(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['actor_id'], ['javdb_actors.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('actor_id', 'alias')
|
||||
)
|
||||
op.create_table('javdb_actors_movies',
|
||||
sa.Column('actor_id', sa.Integer(), nullable=False),
|
||||
sa.Column('movie_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tags', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['actor_id'], ['javdb_actors.id'], ),
|
||||
sa.ForeignKeyConstraint(['movie_id'], ['javdb_movies.id'], ),
|
||||
sa.PrimaryKeyConstraint('actor_id', 'movie_id')
|
||||
)
|
||||
op.create_table('javdb_movies_tags',
|
||||
sa.Column('movie_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tags', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['movie_id'], ['javdb_movies.id'], ),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['javdb_tags.id'], ),
|
||||
sa.PrimaryKeyConstraint('movie_id', 'tag_id')
|
||||
)
|
||||
op.create_table('pbox_actor_aliases',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('actor_id', sa.Integer(), nullable=False),
|
||||
sa.Column('alias', sa.String(length=255), nullable=False),
|
||||
sa.Column('actor_alias', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['actor_id'], ['pbox_actors.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('pbox_movies',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('is_full_data', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('href', sa.String(length=255), nullable=True),
|
||||
sa.Column('title', sa.String(length=255), nullable=True),
|
||||
sa.Column('movie_id', sa.Integer(), nullable=True),
|
||||
sa.Column('content_id', sa.Integer(), nullable=True),
|
||||
sa.Column('duration', sa.String(length=255), nullable=True),
|
||||
sa.Column('publish_date', sa.String(length=255), nullable=True),
|
||||
sa.Column('release_date', sa.String(length=255), nullable=True),
|
||||
sa.Column('studio_id', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['studio_id'], ['pbox_studios.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('thelordofporn_alias',
|
||||
sa.Column('actress_id', sa.Integer(), nullable=False),
|
||||
sa.Column('alias', sa.Text(), nullable=False),
|
||||
sa.Column('updated_at', sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['actress_id'], ['thelordofporn_actress.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('actress_id', 'alias')
|
||||
)
|
||||
op.create_table('iafd_movies_appers_in',
|
||||
sa.Column('movie_id', sa.Integer(), nullable=False),
|
||||
sa.Column('appears_in_id', sa.Integer(), nullable=False),
|
||||
sa.Column('gradation', sa.Integer(), nullable=True),
|
||||
sa.Column('notes', sa.String(length=255), nullable=True),
|
||||
sa.ForeignKeyConstraint(['appears_in_id'], ['iafd_movies.id'], ),
|
||||
sa.ForeignKeyConstraint(['movie_id'], ['iafd_movies.id'], ),
|
||||
sa.PrimaryKeyConstraint('movie_id', 'appears_in_id')
|
||||
)
|
||||
op.create_table('iafd_performers_movies',
|
||||
sa.Column('performer_id', sa.Integer(), nullable=False),
|
||||
sa.Column('movie_id', sa.Integer(), nullable=False),
|
||||
sa.Column('role', sa.String(length=255), nullable=True),
|
||||
sa.Column('notes', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['movie_id'], ['iafd_movies.id'], ),
|
||||
sa.ForeignKeyConstraint(['performer_id'], ['iafd_performers.id'], ),
|
||||
sa.PrimaryKeyConstraint('performer_id', 'movie_id')
|
||||
)
|
||||
op.create_index('idx_iafd_performers_movies_performer_id', 'iafd_performers_movies', ['performer_id'], unique=False)
|
||||
op.create_table('pbox_actors_movies',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('actor_id', sa.Integer(), nullable=True),
|
||||
sa.Column('movie_id', sa.Integer(), nullable=True),
|
||||
sa.Column('actor_mov', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('tags', sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['actor_id'], ['pbox_actors.id'], ),
|
||||
sa.ForeignKeyConstraint(['movie_id'], ['pbox_movies.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('pbox_movies_alts',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('min_mov_id', sa.Integer(), nullable=True),
|
||||
sa.Column('max_mov_id', sa.Integer(), nullable=True),
|
||||
sa.Column('min_max', sa.String(length=255), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['max_mov_id'], ['pbox_movies.id'], ),
|
||||
sa.ForeignKeyConstraint(['min_mov_id'], ['pbox_movies.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('pbox_movies_tags',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('movie_id', sa.Integer(), nullable=True),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=True),
|
||||
sa.Column('movid_tagid', sa.String(length=255), nullable=True),
|
||||
sa.Column('tags', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.Column('updated_at', sa.Text(), server_default=sa.text("(datetime('now', 'localtime'))"), nullable=True),
|
||||
sa.ForeignKeyConstraint(['movie_id'], ['pbox_movies.id'], ),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['pbox_tags.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('pbox_movies_tags')
|
||||
op.drop_table('pbox_movies_alts')
|
||||
op.drop_table('pbox_actors_movies')
|
||||
op.drop_index('idx_iafd_performers_movies_performer_id', table_name='iafd_performers_movies')
|
||||
op.drop_table('iafd_performers_movies')
|
||||
op.drop_table('iafd_movies_appers_in')
|
||||
op.drop_table('thelordofporn_alias')
|
||||
op.drop_table('pbox_movies')
|
||||
op.drop_table('pbox_actor_aliases')
|
||||
op.drop_table('javdb_movies_tags')
|
||||
op.drop_table('javdb_actors_movies')
|
||||
op.drop_table('javdb_actors_alias')
|
||||
op.drop_table('javbus_movies_tags')
|
||||
op.drop_index('idx_actor_movie_actor_id', table_name='javbus_actors_movies')
|
||||
op.drop_table('javbus_actors_movies')
|
||||
op.drop_table('iafd_performer_urls')
|
||||
op.drop_table('iafd_performer_aliases')
|
||||
op.drop_table('iafd_movies')
|
||||
op.drop_table('clm_keywords_index')
|
||||
op.drop_table('u3c3')
|
||||
op.drop_table('thelordofporn_actress')
|
||||
op.drop_table('sis')
|
||||
op.drop_table('pbox_tags')
|
||||
op.drop_table('pbox_studios')
|
||||
op.drop_table('pbox_actors')
|
||||
op.drop_table('javhd_models')
|
||||
op.drop_table('javdb_task_log')
|
||||
op.drop_table('javdb_tags')
|
||||
op.drop_table('javdb_series')
|
||||
op.drop_table('javdb_publishers')
|
||||
op.drop_table('javdb_movies')
|
||||
op.drop_table('javdb_makers')
|
||||
op.drop_table('javdb_actors')
|
||||
op.drop_table('javbus_tags')
|
||||
op.drop_table('javbus_studios')
|
||||
op.drop_table('javbus_series')
|
||||
op.drop_table('javbus_movies')
|
||||
op.drop_table('javbus_labels')
|
||||
op.drop_table('javbus_actors')
|
||||
op.drop_table('iafd_task_log')
|
||||
op.drop_table('iafd_studios')
|
||||
op.drop_table('iafd_performers')
|
||||
op.drop_table('iafd_meta_ethnic')
|
||||
op.drop_table('iafd_distributors')
|
||||
op.drop_table('clm_keywords')
|
||||
op.drop_table('clm_index')
|
||||
# ### end Alembic commands ###
|
||||
@ -0,0 +1,32 @@
|
||||
"""Initial test schema
|
||||
|
||||
Revision ID: bce23e0d0c3a
|
||||
Revises: 86eea10972c0
|
||||
Create Date: 2025-07-29 19:54:54.268814
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'bce23e0d0c3a'
|
||||
down_revision: Union[str, Sequence[str], None] = '86eea10972c0'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
pass
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
pass
|
||||
# ### end Alembic commands ###
|
||||
85
scrapy_proj/sqlalchemy/models/scrapy.py
Normal file
85
scrapy_proj/sqlalchemy/models/scrapy.py
Normal file
@ -0,0 +1,85 @@
|
||||
from sqlalchemy import ForeignKey, Integer, REAL, String, Text, text
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class ClmIndex(Base):
|
||||
__tablename__ = 'clm_index'
|
||||
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
category: Mapped[Optional[str]] = mapped_column(Text)
|
||||
title: Mapped[Optional[str]] = mapped_column(Text)
|
||||
href: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
magnet_href: Mapped[Optional[str]] = mapped_column(Text)
|
||||
size_text: Mapped[Optional[str]] = mapped_column(Text)
|
||||
size_gb: Mapped[Optional[float]] = mapped_column(REAL)
|
||||
heat: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
add_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
last_down_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
clm_keywords_index: Mapped[List['ClmKeywordsIndex']] = relationship('ClmKeywordsIndex', back_populates='index')
|
||||
|
||||
|
||||
class ClmKeywords(Base):
|
||||
__tablename__ = 'clm_keywords'
|
||||
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
words: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
groups: Mapped[Optional[str]] = mapped_column(Text)
|
||||
tags: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
index_count: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
|
||||
clm_keywords_index: Mapped[List['ClmKeywordsIndex']] = relationship('ClmKeywordsIndex', back_populates='words')
|
||||
|
||||
|
||||
class Sis(Base):
|
||||
__tablename__ = 'sis'
|
||||
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
plate_name: Mapped[Optional[str]] = mapped_column(Text)
|
||||
title: Mapped[Optional[str]] = mapped_column(Text)
|
||||
url: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
size_text: Mapped[Optional[str]] = mapped_column(Text)
|
||||
size_gb: Mapped[Optional[float]] = mapped_column(REAL)
|
||||
update_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
|
||||
class U3c3(Base):
|
||||
__tablename__ = 'u3c3'
|
||||
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
category: Mapped[Optional[str]] = mapped_column(Text)
|
||||
title: Mapped[Optional[str]] = mapped_column(Text)
|
||||
url: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
torrent_url: Mapped[Optional[str]] = mapped_column(Text)
|
||||
magnet_url: Mapped[Optional[str]] = mapped_column(Text)
|
||||
size_text: Mapped[Optional[str]] = mapped_column(Text)
|
||||
size_gb: Mapped[Optional[float]] = mapped_column(REAL)
|
||||
update_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
|
||||
class ClmKeywordsIndex(Base):
|
||||
__tablename__ = 'clm_keywords_index'
|
||||
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
words_id: Mapped[Optional[int]] = mapped_column(ForeignKey('clm_keywords.id'))
|
||||
index_id: Mapped[Optional[int]] = mapped_column(ForeignKey('clm_index.id'))
|
||||
wid_iid: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
tags: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
index: Mapped[Optional['ClmIndex']] = relationship('ClmIndex', back_populates='clm_keywords_index')
|
||||
words: Mapped[Optional['ClmKeywords']] = relationship('ClmKeywords', back_populates='clm_keywords_index')
|
||||
670
scrapy_proj/sqlalchemy/models/shared.py
Normal file
670
scrapy_proj/sqlalchemy/models/shared.py
Normal file
@ -0,0 +1,670 @@
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, Integer, REAL, String, Text, text
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
import datetime
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class IafdDistributors(Base):
|
||||
__tablename__ = 'iafd_distributors'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
details: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
iafd_movies: Mapped[List['IafdMovies']] = relationship('IafdMovies', back_populates='distributor')
|
||||
|
||||
|
||||
class IafdMetaEthnic(Base):
|
||||
__tablename__ = 'iafd_meta_ethnic'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
|
||||
class IafdPerformers(Base):
|
||||
__tablename__ = 'iafd_performers'
|
||||
|
||||
name: Mapped[str] = mapped_column(Text)
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
birth_year: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_astro_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_birth_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_ethnic_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
gender: Mapped[Optional[str]] = mapped_column(Text)
|
||||
birthday: Mapped[Optional[str]] = mapped_column(Text)
|
||||
astrology: Mapped[Optional[str]] = mapped_column(Text)
|
||||
birthplace: Mapped[Optional[str]] = mapped_column(Text)
|
||||
years_active: Mapped[Optional[str]] = mapped_column(Text)
|
||||
ethnicity: Mapped[Optional[str]] = mapped_column(Text)
|
||||
nationality: Mapped[Optional[str]] = mapped_column(Text)
|
||||
hair_colors: Mapped[Optional[str]] = mapped_column(Text)
|
||||
eye_color: Mapped[Optional[str]] = mapped_column(Text)
|
||||
height_str: Mapped[Optional[str]] = mapped_column(Text)
|
||||
weight_str: Mapped[Optional[str]] = mapped_column(Text)
|
||||
measurements: Mapped[Optional[str]] = mapped_column(Text)
|
||||
tattoos: Mapped[Optional[str]] = mapped_column(Text)
|
||||
piercings: Mapped[Optional[str]] = mapped_column(Text)
|
||||
fake_tits: Mapped[Optional[str]] = mapped_column(Text)
|
||||
href: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
weight: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
height: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
rating: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
movies_cnt: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
vixen_cnt: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
blacked_cnt: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
tushy_cnt: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
x_art_cnt: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
|
||||
iafd_performer_aliases: Mapped[List['IafdPerformerAliases']] = relationship('IafdPerformerAliases', back_populates='performer')
|
||||
iafd_performer_urls: Mapped[List['IafdPerformerUrls']] = relationship('IafdPerformerUrls', back_populates='performer')
|
||||
iafd_performers_movies: Mapped[List['IafdPerformersMovies']] = relationship('IafdPerformersMovies', back_populates='performer')
|
||||
|
||||
|
||||
class IafdStudios(Base):
|
||||
__tablename__ = 'iafd_studios'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
details: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
iafd_movies: Mapped[List['IafdMovies']] = relationship('IafdMovies', back_populates='studio')
|
||||
|
||||
|
||||
class IafdTaskLog(Base):
|
||||
__tablename__ = 'iafd_task_log'
|
||||
|
||||
task_id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
full_data_performers: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
total_performers: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
full_data_movies: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
total_movies: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
total_distributors: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
total_studios: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
task_status: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
|
||||
class JavbusActors(Base):
|
||||
__tablename__ = 'javbus_actors'
|
||||
|
||||
uncensored: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_actor_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
movies_cnt: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
ja_name: Mapped[Optional[str]] = mapped_column(Text)
|
||||
zh_name: Mapped[Optional[str]] = mapped_column(Text)
|
||||
en_name: Mapped[Optional[str]] = mapped_column(Text)
|
||||
href: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
pic: Mapped[Optional[str]] = mapped_column(Text)
|
||||
birth_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
height: Mapped[Optional[str]] = mapped_column(Text)
|
||||
breast_size: Mapped[Optional[str]] = mapped_column(Text)
|
||||
measurements: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
javbus_actors_movies: Mapped[List['JavbusActorsMovies']] = relationship('JavbusActorsMovies', back_populates='actor')
|
||||
|
||||
|
||||
class JavbusLabels(Base):
|
||||
__tablename__ = 'javbus_labels'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
en_name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
ja_name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
details: Mapped[Optional[str]] = mapped_column(Text)
|
||||
uncensored: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
movies_cnt: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
magnet_cnt: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
|
||||
|
||||
class JavbusMovies(Base):
|
||||
__tablename__ = 'javbus_movies'
|
||||
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
uncensored: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_actor_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_studios: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_labels: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_series: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
actors_cnt: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
href: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
title: Mapped[Optional[str]] = mapped_column(Text)
|
||||
cover_url: Mapped[Optional[str]] = mapped_column(Text)
|
||||
serial_number: Mapped[Optional[str]] = mapped_column(Text)
|
||||
release_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
duration: Mapped[Optional[str]] = mapped_column(Text)
|
||||
studio_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
label_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
series_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
javbus_actors_movies: Mapped[List['JavbusActorsMovies']] = relationship('JavbusActorsMovies', back_populates='movie')
|
||||
javbus_movies_tags: Mapped[List['JavbusMoviesTags']] = relationship('JavbusMoviesTags', back_populates='movie')
|
||||
|
||||
|
||||
class JavbusSeries(Base):
|
||||
__tablename__ = 'javbus_series'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
en_name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
ja_name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
details: Mapped[Optional[str]] = mapped_column(Text)
|
||||
uncensored: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
movies_cnt: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
magnet_cnt: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
|
||||
|
||||
class JavbusStudios(Base):
|
||||
__tablename__ = 'javbus_studios'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
en_name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
ja_name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
details: Mapped[Optional[str]] = mapped_column(Text)
|
||||
uncensored: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
movies_cnt: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
magnet_cnt: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
|
||||
|
||||
class JavbusTags(Base):
|
||||
__tablename__ = 'javbus_tags'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
en_name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
ja_name: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
javbus_movies_tags: Mapped[List['JavbusMoviesTags']] = relationship('JavbusMoviesTags', back_populates='tag')
|
||||
|
||||
|
||||
class JavdbActors(Base):
|
||||
__tablename__ = 'javdb_actors'
|
||||
|
||||
name: Mapped[str] = mapped_column(Text)
|
||||
href: Mapped[str] = mapped_column(Text, unique=True)
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_actor_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
movies_cnt: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
uncensored: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
pic: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[datetime.datetime]] = mapped_column(DateTime, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[datetime.datetime]] = mapped_column(DateTime, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
javdb_actors_alias: Mapped[List['JavdbActorsAlias']] = relationship('JavdbActorsAlias', back_populates='actor')
|
||||
javdb_actors_movies: Mapped[List['JavdbActorsMovies']] = relationship('JavdbActorsMovies', back_populates='actor')
|
||||
|
||||
|
||||
class JavdbMakers(Base):
|
||||
__tablename__ = 'javdb_makers'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
details: Mapped[Optional[str]] = mapped_column(Text)
|
||||
from_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
|
||||
|
||||
class JavdbMovies(Base):
|
||||
__tablename__ = 'javdb_movies'
|
||||
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_actor_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_makers: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_series: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_publishers: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
uncensored: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
href: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
title: Mapped[Optional[str]] = mapped_column(Text)
|
||||
cover_url: Mapped[Optional[str]] = mapped_column(Text)
|
||||
serial_number: Mapped[Optional[str]] = mapped_column(Text)
|
||||
release_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
duration: Mapped[Optional[str]] = mapped_column(Text)
|
||||
maker_id: Mapped[Optional[str]] = mapped_column(Text)
|
||||
series_id: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
pub_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
|
||||
javdb_actors_movies: Mapped[List['JavdbActorsMovies']] = relationship('JavdbActorsMovies', back_populates='movie')
|
||||
javdb_movies_tags: Mapped[List['JavdbMoviesTags']] = relationship('JavdbMoviesTags', back_populates='movie')
|
||||
|
||||
|
||||
class JavdbPublishers(Base):
|
||||
__tablename__ = 'javdb_publishers'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
details: Mapped[Optional[str]] = mapped_column(Text)
|
||||
from_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
|
||||
|
||||
class JavdbSeries(Base):
|
||||
__tablename__ = 'javdb_series'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
details: Mapped[Optional[str]] = mapped_column(Text)
|
||||
from_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
from_movie_list: Mapped[Optional[int]] = mapped_column(Integer, server_default=text('0'))
|
||||
|
||||
|
||||
class JavdbTags(Base):
|
||||
__tablename__ = 'javdb_tags'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
javdb_movies_tags: Mapped[List['JavdbMoviesTags']] = relationship('JavdbMoviesTags', back_populates='tag')
|
||||
|
||||
|
||||
class JavdbTaskLog(Base):
|
||||
__tablename__ = 'javdb_task_log'
|
||||
|
||||
task_id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
full_data_actors: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
total_actors: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
full_data_movies: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
total_movies: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
total_makers: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
total_series: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
task_status: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
|
||||
class JavhdModels(Base):
|
||||
__tablename__ = 'javhd_models'
|
||||
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
rank: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
ja_name: Mapped[Optional[str]] = mapped_column(Text)
|
||||
zh_name: Mapped[Optional[str]] = mapped_column(Text)
|
||||
en_name: Mapped[Optional[str]] = mapped_column(Text)
|
||||
url: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
pic: Mapped[Optional[str]] = mapped_column(Text)
|
||||
height: Mapped[Optional[str]] = mapped_column(Text)
|
||||
weight: Mapped[Optional[str]] = mapped_column(Text)
|
||||
breast_size: Mapped[Optional[str]] = mapped_column(Text)
|
||||
breast_factor: Mapped[Optional[str]] = mapped_column(Text)
|
||||
hair_color: Mapped[Optional[str]] = mapped_column(Text)
|
||||
eye_color: Mapped[Optional[str]] = mapped_column(Text)
|
||||
birth_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
ethnicity: Mapped[Optional[str]] = mapped_column(Text)
|
||||
birth_place: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
|
||||
class PboxActors(Base):
|
||||
__tablename__ = 'pbox_actors'
|
||||
|
||||
name: Mapped[str] = mapped_column(Text)
|
||||
movies_cnt: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
href: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
gender: Mapped[Optional[str]] = mapped_column(Text)
|
||||
age: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
nationality: Mapped[Optional[str]] = mapped_column(Text)
|
||||
country: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
pbox_actor_aliases: Mapped[List['PboxActorAliases']] = relationship('PboxActorAliases', back_populates='actor')
|
||||
pbox_actors_movies: Mapped[List['PboxActorsMovies']] = relationship('PboxActorsMovies', back_populates='actor')
|
||||
|
||||
|
||||
class PboxStudios(Base):
|
||||
__tablename__ = 'pbox_studios'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
label_id: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
scene_count: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
description: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
pbox_movies: Mapped[List['PboxMovies']] = relationship('PboxMovies', back_populates='studio')
|
||||
|
||||
|
||||
class PboxTags(Base):
|
||||
__tablename__ = 'pbox_tags'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
tag_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
pbox_movies_tags: Mapped[List['PboxMoviesTags']] = relationship('PboxMoviesTags', back_populates='tag')
|
||||
|
||||
|
||||
class ThelordofpornActress(Base):
|
||||
__tablename__ = 'thelordofporn_actress'
|
||||
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
id: Mapped[Optional[int]] = mapped_column(Integer, primary_key=True)
|
||||
pornstar: Mapped[Optional[str]] = mapped_column(Text)
|
||||
rating: Mapped[Optional[float]] = mapped_column(REAL)
|
||||
rank: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
votes: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
href: Mapped[Optional[str]] = mapped_column(Text, unique=True)
|
||||
career_start: Mapped[Optional[str]] = mapped_column(Text)
|
||||
measurements: Mapped[Optional[str]] = mapped_column(Text)
|
||||
born: Mapped[Optional[str]] = mapped_column(Text)
|
||||
height: Mapped[Optional[str]] = mapped_column(Text)
|
||||
weight: Mapped[Optional[str]] = mapped_column(Text)
|
||||
date_modified: Mapped[Optional[str]] = mapped_column(Text)
|
||||
global_rank: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
weekly_rank: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
last_month_rating: Mapped[Optional[float]] = mapped_column(REAL)
|
||||
current_rating: Mapped[Optional[float]] = mapped_column(REAL)
|
||||
total_votes: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
birth_date: Mapped[Optional[str]] = mapped_column(Text)
|
||||
birth_year: Mapped[Optional[str]] = mapped_column(Text)
|
||||
birth_place: Mapped[Optional[str]] = mapped_column(Text)
|
||||
height_ft: Mapped[Optional[str]] = mapped_column(Text)
|
||||
height_cm: Mapped[Optional[str]] = mapped_column(Text)
|
||||
weight_lbs: Mapped[Optional[str]] = mapped_column(Text)
|
||||
weight_kg: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
thelordofporn_alias: Mapped[List['ThelordofpornAlias']] = relationship('ThelordofpornAlias', back_populates='actress')
|
||||
|
||||
|
||||
class IafdMovies(Base):
|
||||
__tablename__ = 'iafd_movies'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
release_year: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_performer_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_dist_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
from_stu_list: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
title: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
minutes: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
distributor_id: Mapped[Optional[int]] = mapped_column(ForeignKey('iafd_distributors.id'))
|
||||
studio_id: Mapped[Optional[int]] = mapped_column(ForeignKey('iafd_studios.id'))
|
||||
release_date: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
added_to_IAFD_date: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
all_girl: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
all_male: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
compilation: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
webscene: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
director_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
distributor: Mapped[Optional['IafdDistributors']] = relationship('IafdDistributors', back_populates='iafd_movies')
|
||||
studio: Mapped[Optional['IafdStudios']] = relationship('IafdStudios', back_populates='iafd_movies')
|
||||
iafd_movies_appers_in: Mapped[List['IafdMoviesAppersIn']] = relationship('IafdMoviesAppersIn', foreign_keys='[IafdMoviesAppersIn.appears_in_id]', back_populates='appears_in')
|
||||
iafd_movies_appers_in_: Mapped[List['IafdMoviesAppersIn']] = relationship('IafdMoviesAppersIn', foreign_keys='[IafdMoviesAppersIn.movie_id]', back_populates='movie')
|
||||
iafd_performers_movies: Mapped[List['IafdPerformersMovies']] = relationship('IafdPerformersMovies', back_populates='movie')
|
||||
|
||||
|
||||
class IafdPerformerAliases(Base):
|
||||
__tablename__ = 'iafd_performer_aliases'
|
||||
|
||||
performer_id: Mapped[int] = mapped_column(ForeignKey('iafd_performers.id'), primary_key=True)
|
||||
alias: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
|
||||
performer: Mapped['IafdPerformers'] = relationship('IafdPerformers', back_populates='iafd_performer_aliases')
|
||||
|
||||
|
||||
class IafdPerformerUrls(Base):
|
||||
__tablename__ = 'iafd_performer_urls'
|
||||
|
||||
performer_id: Mapped[int] = mapped_column(ForeignKey('iafd_performers.id'), primary_key=True)
|
||||
position: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
url: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
|
||||
performer: Mapped['IafdPerformers'] = relationship('IafdPerformers', back_populates='iafd_performer_urls')
|
||||
|
||||
|
||||
class JavbusActorsMovies(Base):
|
||||
__tablename__ = 'javbus_actors_movies'
|
||||
__table_args__ = (
|
||||
Index('idx_actor_movie_actor_id', 'actor_id'),
|
||||
)
|
||||
|
||||
actor_id: Mapped[Optional[int]] = mapped_column(ForeignKey('javbus_actors.id'), primary_key=True)
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(ForeignKey('javbus_movies.id'), primary_key=True)
|
||||
tags: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
actor: Mapped[Optional['JavbusActors']] = relationship('JavbusActors', back_populates='javbus_actors_movies')
|
||||
movie: Mapped[Optional['JavbusMovies']] = relationship('JavbusMovies', back_populates='javbus_actors_movies')
|
||||
|
||||
|
||||
class JavbusMoviesTags(Base):
|
||||
__tablename__ = 'javbus_movies_tags'
|
||||
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(ForeignKey('javbus_movies.id'), primary_key=True)
|
||||
tag_id: Mapped[Optional[int]] = mapped_column(ForeignKey('javbus_tags.id'), primary_key=True)
|
||||
tags: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
movie: Mapped[Optional['JavbusMovies']] = relationship('JavbusMovies', back_populates='javbus_movies_tags')
|
||||
tag: Mapped[Optional['JavbusTags']] = relationship('JavbusTags', back_populates='javbus_movies_tags')
|
||||
|
||||
|
||||
class JavdbActorsAlias(Base):
|
||||
__tablename__ = 'javdb_actors_alias'
|
||||
|
||||
actor_id: Mapped[int] = mapped_column(ForeignKey('javdb_actors.id', ondelete='CASCADE'), primary_key=True)
|
||||
alias: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
created_at: Mapped[Optional[datetime.datetime]] = mapped_column(DateTime, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[datetime.datetime]] = mapped_column(DateTime, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
actor: Mapped['JavdbActors'] = relationship('JavdbActors', back_populates='javdb_actors_alias')
|
||||
|
||||
|
||||
class JavdbActorsMovies(Base):
|
||||
__tablename__ = 'javdb_actors_movies'
|
||||
|
||||
actor_id: Mapped[Optional[int]] = mapped_column(ForeignKey('javdb_actors.id'), primary_key=True)
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(ForeignKey('javdb_movies.id'), primary_key=True)
|
||||
tags: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
actor: Mapped[Optional['JavdbActors']] = relationship('JavdbActors', back_populates='javdb_actors_movies')
|
||||
movie: Mapped[Optional['JavdbMovies']] = relationship('JavdbMovies', back_populates='javdb_actors_movies')
|
||||
|
||||
|
||||
class JavdbMoviesTags(Base):
|
||||
__tablename__ = 'javdb_movies_tags'
|
||||
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(ForeignKey('javdb_movies.id'), primary_key=True)
|
||||
tag_id: Mapped[Optional[int]] = mapped_column(ForeignKey('javdb_tags.id'), primary_key=True)
|
||||
tags: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
movie: Mapped[Optional['JavdbMovies']] = relationship('JavdbMovies', back_populates='javdb_movies_tags')
|
||||
tag: Mapped[Optional['JavdbTags']] = relationship('JavdbTags', back_populates='javdb_movies_tags')
|
||||
|
||||
|
||||
class PboxActorAliases(Base):
|
||||
__tablename__ = 'pbox_actor_aliases'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
actor_id: Mapped[int] = mapped_column(ForeignKey('pbox_actors.id'))
|
||||
alias: Mapped[str] = mapped_column(String(255))
|
||||
actor_alias: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
actor: Mapped['PboxActors'] = relationship('PboxActors', back_populates='pbox_actor_aliases')
|
||||
|
||||
|
||||
class PboxMovies(Base):
|
||||
__tablename__ = 'pbox_movies'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
is_full_data: Mapped[int] = mapped_column(Integer, server_default=text('0'))
|
||||
href: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
title: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
content_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
duration: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
publish_date: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
release_date: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
studio_id: Mapped[Optional[int]] = mapped_column(ForeignKey('pbox_studios.id'))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
studio: Mapped[Optional['PboxStudios']] = relationship('PboxStudios', back_populates='pbox_movies')
|
||||
pbox_actors_movies: Mapped[List['PboxActorsMovies']] = relationship('PboxActorsMovies', back_populates='movie')
|
||||
pbox_movies_alts: Mapped[List['PboxMoviesAlts']] = relationship('PboxMoviesAlts', foreign_keys='[PboxMoviesAlts.max_mov_id]', back_populates='max_mov')
|
||||
pbox_movies_alts_: Mapped[List['PboxMoviesAlts']] = relationship('PboxMoviesAlts', foreign_keys='[PboxMoviesAlts.min_mov_id]', back_populates='min_mov')
|
||||
pbox_movies_tags: Mapped[List['PboxMoviesTags']] = relationship('PboxMoviesTags', back_populates='movie')
|
||||
|
||||
|
||||
class ThelordofpornAlias(Base):
|
||||
__tablename__ = 'thelordofporn_alias'
|
||||
|
||||
actress_id: Mapped[int] = mapped_column(ForeignKey('thelordofporn_actress.id', ondelete='CASCADE'), primary_key=True)
|
||||
alias: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
actress: Mapped['ThelordofpornActress'] = relationship('ThelordofpornActress', back_populates='thelordofporn_alias')
|
||||
|
||||
|
||||
class IafdMoviesAppersIn(Base):
|
||||
__tablename__ = 'iafd_movies_appers_in'
|
||||
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(ForeignKey('iafd_movies.id'), primary_key=True)
|
||||
appears_in_id: Mapped[Optional[int]] = mapped_column(ForeignKey('iafd_movies.id'), primary_key=True)
|
||||
gradation: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
notes: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
|
||||
appears_in: Mapped[Optional['IafdMovies']] = relationship('IafdMovies', foreign_keys=[appears_in_id], back_populates='iafd_movies_appers_in')
|
||||
movie: Mapped[Optional['IafdMovies']] = relationship('IafdMovies', foreign_keys=[movie_id], back_populates='iafd_movies_appers_in_')
|
||||
|
||||
|
||||
class IafdPerformersMovies(Base):
|
||||
__tablename__ = 'iafd_performers_movies'
|
||||
__table_args__ = (
|
||||
Index('idx_iafd_performers_movies_performer_id', 'performer_id'),
|
||||
)
|
||||
|
||||
performer_id: Mapped[Optional[int]] = mapped_column(ForeignKey('iafd_performers.id'), primary_key=True)
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(ForeignKey('iafd_movies.id'), primary_key=True)
|
||||
role: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
notes: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
movie: Mapped[Optional['IafdMovies']] = relationship('IafdMovies', back_populates='iafd_performers_movies')
|
||||
performer: Mapped[Optional['IafdPerformers']] = relationship('IafdPerformers', back_populates='iafd_performers_movies')
|
||||
|
||||
|
||||
class PboxActorsMovies(Base):
|
||||
__tablename__ = 'pbox_actors_movies'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
actor_id: Mapped[Optional[int]] = mapped_column(ForeignKey('pbox_actors.id'))
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(ForeignKey('pbox_movies.id'))
|
||||
actor_mov: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
tags: Mapped[Optional[str]] = mapped_column(Text)
|
||||
|
||||
actor: Mapped[Optional['PboxActors']] = relationship('PboxActors', back_populates='pbox_actors_movies')
|
||||
movie: Mapped[Optional['PboxMovies']] = relationship('PboxMovies', back_populates='pbox_actors_movies')
|
||||
|
||||
|
||||
class PboxMoviesAlts(Base):
|
||||
__tablename__ = 'pbox_movies_alts'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
min_mov_id: Mapped[Optional[int]] = mapped_column(ForeignKey('pbox_movies.id'))
|
||||
max_mov_id: Mapped[Optional[int]] = mapped_column(ForeignKey('pbox_movies.id'))
|
||||
min_max: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
max_mov: Mapped[Optional['PboxMovies']] = relationship('PboxMovies', foreign_keys=[max_mov_id], back_populates='pbox_movies_alts')
|
||||
min_mov: Mapped[Optional['PboxMovies']] = relationship('PboxMovies', foreign_keys=[min_mov_id], back_populates='pbox_movies_alts_')
|
||||
|
||||
|
||||
class PboxMoviesTags(Base):
|
||||
__tablename__ = 'pbox_movies_tags'
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
movie_id: Mapped[Optional[int]] = mapped_column(ForeignKey('pbox_movies.id'))
|
||||
tag_id: Mapped[Optional[int]] = mapped_column(ForeignKey('pbox_tags.id'))
|
||||
movid_tagid: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
tags: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
updated_at: Mapped[Optional[str]] = mapped_column(Text, server_default=text("datetime('now', 'localtime')"))
|
||||
|
||||
movie: Mapped[Optional['PboxMovies']] = relationship('PboxMovies', back_populates='pbox_movies_tags')
|
||||
tag: Mapped[Optional['PboxTags']] = relationship('PboxTags', back_populates='pbox_movies_tags')
|
||||
55
scrapy_proj/sqlalchemy/readme.txt
Normal file
55
scrapy_proj/sqlalchemy/readme.txt
Normal file
@ -0,0 +1,55 @@
|
||||
# 从数据库生成模型类
|
||||
sqlacodegen sqlite:////root/sharedata/sqlite/shared.db > models/shared.py
|
||||
sqlacodegen sqlite:////root/sharedata/sqlite/scrapy.db > models/scrapy.py
|
||||
|
||||
# 初始化
|
||||
alembic init migrations/shared_comm
|
||||
alembic init migrations/scrapy
|
||||
alembic init migrations/testdb
|
||||
|
||||
# 同步修改到数据库（读取 models/shared.py ）
|
||||
./scripts/sync_shared_comm.sh
|
||||
|
||||
./scripts/reset_testdb.sh
|
||||
|
||||
### 对视图支持不好，主要是视图的字段没有类型，所以在导入导出时会出错，慎用！
|
||||
|
||||
|
||||
------------------------- 清理掉无法处理的字段 --------------------
|
||||
PRAGMA foreign_keys = OFF; -- 禁用外键检查
|
||||
|
||||
-- 检查字段是否已删除
|
||||
PRAGMA table_info(javdb_series); -- 应看不到parent_id字段
|
||||
|
||||
-- 验证数据完整性
|
||||
SELECT COUNT(*), MIN(id), MAX(id) FROM javdb_series; -- 数量应与操作前一致
|
||||
|
||||
-- 5. 处理 javdb_series 表
|
||||
CREATE TABLE "javdb_series_new" (
|
||||
`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
`name` VARCHAR(255) NOT NULL,
|
||||
`href` VARCHAR(255) UNIQUE,
|
||||
`created_at` TEXT DEFAULT (datetime('now', 'localtime')),
|
||||
`updated_at` TEXT DEFAULT (datetime('now', 'localtime')),
|
||||
`details` TEXT,
|
||||
from_list INTEGER DEFAULT (0),
|
||||
from_movie_list INTEGER DEFAULT (0)
|
||||
);
|
||||
|
||||
INSERT INTO javdb_series_new (id, name, href, created_at, updated_at, details, from_list, from_movie_list)
|
||||
SELECT id, name, href, created_at, updated_at, details, from_list, from_movie_list
|
||||
FROM javdb_series;
|
||||
|
||||
DROP TABLE javdb_series;
|
||||
ALTER TABLE javdb_series_new RENAME TO javdb_series;
|
||||
|
||||
|
||||
|
||||
-- 检查字段是否已删除
|
||||
PRAGMA table_info(javdb_series); -- 应看不到parent_id字段
|
||||
|
||||
-- 验证数据完整性
|
||||
SELECT COUNT(*), MIN(id), MAX(id) FROM javdb_series; -- 数量应与操作前一致
|
||||
|
||||
|
||||
PRAGMA foreign_keys = ON; -- 恢复外键检查
|
||||
6
scrapy_proj/sqlalchemy/requirements.txt
Normal file
6
scrapy_proj/sqlalchemy/requirements.txt
Normal file
@ -0,0 +1,6 @@
|
||||
# requirements.txt
|
||||
sqlalchemy>=2.0.0
|
||||
alembic>=1.12.0
|
||||
sqlacodegen>=3.0.0
|
||||
|
||||
# pip install -r requirements.txt
|
||||
14
scrapy_proj/sqlalchemy/scripts/reset_testdb.sh
Executable file
14
scrapy_proj/sqlalchemy/scripts/reset_testdb.sh
Executable file
@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
cd $(dirname $0)/..
|
||||
|
||||
# 删除测试库文件
|
||||
#rm -f /root/sharedata/sqlite/test.db
|
||||
|
||||
# 重新创建并应用最新迁移
|
||||
alembic -c migrations/testdb/alembic.ini revision --autogenerate -m "Initial test schema"
|
||||
alembic -c migrations/testdb/alembic.ini upgrade head
|
||||
|
||||
# 回滚数据库A到上一版本
|
||||
# alembic -c migrations/testdb/alembic.ini downgrade -1
|
||||
|
||||
echo "测试库已重置并生成最新表结构"
|
||||
7
scrapy_proj/sqlalchemy/scripts/sync_scrapy.sh
Executable file
7
scrapy_proj/sqlalchemy/scripts/sync_scrapy.sh
Executable file
@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
cd $(dirname $0)/..
|
||||
|
||||
alembic -c migrations/scrapy/alembic.ini revision --autogenerate -m "Auto update from scrapy"
|
||||
alembic -c migrations/scrapy/alembic.ini upgrade head
|
||||
|
||||
echo "数据库 scrapy 同步完成"
|
||||
10
scrapy_proj/sqlalchemy/scripts/sync_shared_comm.sh
Executable file
10
scrapy_proj/sqlalchemy/scripts/sync_shared_comm.sh
Executable file
@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
cd $(dirname $0)/.. # 切换到tools目录
|
||||
|
||||
# 生成迁移脚本
|
||||
alembic -c migrations/shared_comm/alembic.ini revision --autogenerate -m "Auto update from shared_comm"
|
||||
|
||||
# 执行迁移
|
||||
alembic -c migrations/shared_comm/alembic.ini upgrade head
|
||||
|
||||
echo "数据库 shared 同步完成"
|
||||
Reference in New Issue
Block a user