105 lines
3.7 KiB
Python
105 lines
3.7 KiB
Python
import yfinance as yf
|
||
import pymysql
|
||
import logging
|
||
import time
|
||
import sys
|
||
from datetime import datetime
|
||
|
||
|
||
# 配置日志格式
|
||
formatter = logging.Formatter('%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] - %(message)s')
|
||
|
||
log_filename = f'./log/get_sp500_his.log'
|
||
file_handler = logging.FileHandler(log_filename)
|
||
file_handler.setFormatter(formatter)
|
||
|
||
console_handler = logging.StreamHandler(sys.stdout)
|
||
console_handler.setFormatter(formatter)
|
||
|
||
logging.basicConfig(level=logging.INFO, handlers=[file_handler, console_handler])
|
||
logger = logging.getLogger()
|
||
|
||
# MySQL数据库连接
|
||
def connect_to_db():
|
||
return pymysql.connect(
|
||
host="172.18.0.2",
|
||
user="root",
|
||
password="mysqlpw",
|
||
database="stockdb",
|
||
charset="utf8mb4",
|
||
cursorclass=pymysql.cursors.DictCursor
|
||
)
|
||
|
||
# 从MySQL读取sp500表中的股票代码和名称
|
||
def fetch_sp500_codes():
|
||
db = connect_to_db()
|
||
with db.cursor() as cursor:
|
||
# 全量
|
||
cursor.execute("SELECT code_inner, code_name FROM sp500 ")
|
||
|
||
# 下面这些股票不满10年，所以要用 max 参数，而不能用 10y
|
||
#cursor.execute("SELECT code_inner, code_name FROM sp500 where code_inner in ('ABNB', 'CARR', 'CEG', 'GEHC', 'GEV', 'HUBB', 'KVUE', 'OTIS', 'PLTR', 'SOLV', 'VLTO') ")
|
||
|
||
# 失败重跑数据
|
||
#cursor.execute("SELECT code_inner, code_name FROM sp500 where code_inner in ('HUBB') ")
|
||
codes = cursor.fetchall()
|
||
db.close()
|
||
return codes
|
||
|
||
# 插入数据到sp500_his_kline_none表
|
||
def insert_stock_data_to_db(data, code, name):
|
||
try:
|
||
db = connect_to_db()
|
||
with db.cursor() as cursor:
|
||
insert_query = """
|
||
INSERT INTO sp500_his_kline_none (time_key, open, high, low, close, volume, dividends, stock_splits, code, name)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON DUPLICATE KEY UPDATE
|
||
open = VALUES(open),
|
||
high = VALUES(high),
|
||
low = VALUES(low),
|
||
close = VALUES(close),
|
||
volume = VALUES(volume),
|
||
dividends = VALUES(dividends),
|
||
stock_splits = VALUES(stock_splits)
|
||
"""
|
||
for index, row in data.iterrows():
|
||
time_key = index.strftime('%Y-%m-%d %H:%M:%S')
|
||
values = (time_key, row['Open'], row['High'], row['Low'], row['Close'], row['Volume'], row['Dividends'], row['Stock Splits'], code, name)
|
||
cursor.execute(insert_query, values)
|
||
db.commit()
|
||
db.close()
|
||
except pymysql.MySQLError as e:
|
||
logger.error(f"Error occurred while inserting data: {e}")
|
||
|
||
# 拉取股票的历史数据
|
||
def fetch_and_store_stock_data():
|
||
codes = fetch_sp500_codes()
|
||
|
||
for row in codes:
|
||
code_inner = row['code_inner']
|
||
code_name = row['code_name']
|
||
logger.info(f"Fetching data for {code_name} ({code_inner})...")
|
||
|
||
try:
|
||
stock = yf.Ticker(code_inner)
|
||
# 默认获取 十年的数据
|
||
hist_data = stock.history(period="10y", auto_adjust=False)
|
||
|
||
# 有些公司不满10年，不能用 10y，需要改为 max
|
||
#hist_data = stock.history(period="max", auto_adjust=True)
|
||
|
||
if not hist_data.empty:
|
||
logger.info(f"Inserting data for {code_name} ({code_inner})...")
|
||
insert_stock_data_to_db(hist_data, code_inner, code_name)
|
||
else:
|
||
logger.warning(f"No data found for {code_name} ({code_inner})")
|
||
|
||
# 每次请求完后休眠3秒
|
||
time.sleep(3)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error fetching data for {code_name} ({code_inner}): {e}")
|
||
|
||
if __name__ == "__main__":
|
||
fetch_and_store_stock_data() |