research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscarz
2025-07-03 16:24:09 +08:00
parent c34cfb458c
commit 9561f57aa5
2 changed files with 19 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy_proj/scrapy_proj/db_wapper/sqlite_base.py
View File

@ -60,9 +60,9 @@ class SQLiteDBHandler:
return processed_data return processed_data
def insert_or_update_common(self, data, tbl_name, uniq_key='url'): def insert_or_update_common(self, data, tbl_name, uniq_key='url', exists_do_nothing=False):
if self.lower_sqlite_version: if self.lower_sqlite_version:
return self.insert_or_update_common_lower(data, tbl_name, uniq_key) return self.insert_or_update_common_lower(data, tbl_name, uniq_key, exists_do_nothing)
try: try:
processed_data = self.check_and_process_data(data, tbl_name) processed_data = self.check_and_process_data(data, tbl_name)
@ -72,12 +72,17 @@ class SQLiteDBHandler:
columns = ', '.join(processed_data.keys()) columns = ', '.join(processed_data.keys())
values = list(processed_data.values()) values = list(processed_data.values())
placeholders = ', '.join(['?' for _ in values]) placeholders = ', '.join(['?' for _ in values])
if exists_do_nothing:
conflict_clause = f'ON CONFLICT ({uniq_key}) DO NOTHING'
else:
update_clause = ', '.join([f"{col}=EXCLUDED.{col}" for col in processed_data.keys() if col != uniq_key]) update_clause = ', '.join([f"{col}=EXCLUDED.{col}" for col in processed_data.keys() if col != uniq_key])
conflict_clause = f"ON CONFLICT ({uniq_key}) DO UPDATE SET {update_clause}"
sql = f''' sql = f'''
INSERT INTO {tbl_name} ({columns}) INSERT INTO {tbl_name} ({columns})
VALUES ({placeholders}) VALUES ({placeholders})
ON CONFLICT ({uniq_key}) DO UPDATE SET {update_clause} {conflict_clause}
''' '''
self.cursor.execute(sql, values) self.cursor.execute(sql, values)
self.conn.commit() self.conn.commit()
@ -90,7 +95,7 @@ class SQLiteDBHandler:
logging.error(f"Error inserting or updating data: {e}") logging.error(f"Error inserting or updating data: {e}")
return None return None
def insert_or_update_common_lower(self, data, tbl_name, uniq_key='url'): def insert_or_update_common_lower(self, data, tbl_name, uniq_key='url', exists_do_nothing=False):
try: try:
processed_data = self.check_and_process_data(data, tbl_name) processed_data = self.check_and_process_data(data, tbl_name)
if processed_data is None: if processed_data is None:
@ -109,6 +114,7 @@ class SQLiteDBHandler:
self.cursor.execute(sql, values) self.cursor.execute(sql, values)
self.conn.commit() self.conn.commit()
except sqlite3.IntegrityError: # 唯一键冲突，执行更新操作 except sqlite3.IntegrityError: # 唯一键冲突，执行更新操作
if not exists_do_nothing:
update_clause = ', '.join([f"{col}=?" for col in processed_data.keys() if col != uniq_key]) update_clause = ', '.join([f"{col}=?" for col in processed_data.keys() if col != uniq_key])
update_values = [processed_data[col] for col in processed_data.keys() if col != uniq_key] update_values = [processed_data[col] for col in processed_data.keys() if col != uniq_key]
update_values.append(data[uniq_key]) update_values.append(data[uniq_key])

2
scrapy_proj/scrapy_proj/pipelines.py
View File

@ -76,7 +76,7 @@ class SQLitePipeline(SQLiteDBHandler):
def _process_u001_item(self, item, spider): def _process_u001_item(self, item, spider):
logging.debug(f"insert one item. href:{spider.name}") logging.debug(f"insert one item. href:{spider.name}")
return self.insert_or_update_common(item, tbl_name=self.tbl_name_u3c3, uniq_key='url') return self.insert_or_update_common(item, tbl_name=self.tbl_name_u3c3, uniq_key='url', exists_do_nothing=True)
def _process_sis001_item(self, item, spider): def _process_sis001_item(self, item, spider):
self.cursor.execute(''' self.cursor.execute('''