modify stockapp.em scripts.

.gitignore

@@ -1,5 +1,25 @@
 # 忽略 log 目录
 log/
+scripts/aabook/log/
+scripts/aabook/local/
+scripts/aabook/data/
+scripts/u9a9/torrents/
+scripts/u9a9/log/
+scripts/javdb/log/
+scripts/javhd/result/tmp/
+scripts/javhd/log/
+scripts/iafd/data/tmp/
+scripts/iafd/result/tmp/
+scripts/iafd/result/bak/
+scripts/iafd/result/performers/
+scripts/iafd/log/
+scripts/thelordofporn/log/
+scripts/vixen_group/log/
+scripts/pornhub/log/
+
+stockapp/data/
+stockapp/log/
+stockapp/result/
 
 # 忽略 Python 编译文件
 *.pyc

gitignore

@@ -0,0 +1,28 @@
+# 忽略 log 目录
+log/
+scripts/aabook/log/
+scripts/aabook/local/
+scripts/aabook/data/
+scripts/u9a9/torrents/
+scripts/u9a9/log/
+scripts/javdb/log/
+scripts/javhd/result/tmp/
+scripts/javhd/log/
+scripts/iafd/data/tmp/
+scripts/iafd/result/tmp/
+scripts/iafd/result/bak/
+scripts/iafd/result/performers/
+scripts/iafd/log/
+scripts/thelordofporn/log/
+scripts/vixen_group/log/
+scripts/pornhub/log/
+
+stockapp/data/
+stockapp/log/
+stockapp/result/
+
+# 忽略 Python 编译文件
+*.pyc
+
+# 忽略环境配置文件
+.env

stockapp/src/config.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 # MySQL 配置
 db_config = {
-    'host': '172.18.0.3',
+    'host': '172.18.0.5',
     'user': 'root',
     'password': 'mysqlpw',
     'database': 'stockdb'
@@ -14,6 +14,9 @@ db_config = {
 
 log_dir_prefix = '../log'
 
+global_share_data_dir = '/root/sharedata'
+global_stock_data_dir = '/root/hostdir/stock_data'
+
 # 获取log目录
 def get_log_directory():
     """

stockapp/src/crawling/stock_hist_em.py

@@ -6,11 +6,96 @@ Desc: 东方财富网-行情首页-沪深京 A 股
 """
 import requests
 import pandas as pd
+import time
 
 from functools import lru_cache
 
 
-def stock_zh_a_spot_em(fs='m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048') -> pd.DataFrame:
+def fetch_with_retries_em(url, params, max_retries=3, delay=2):
+    """带重试机制的 GET 请求"""
+    for attempt in range(max_retries):
+        try:
+            response = requests.get(url, params=params, timeout=5)
+            response.raise_for_status()
+            return response.json()
+        except requests.RequestException as e:
+            print(f"请求失败，第 {attempt + 1} 次重试: {e}")
+            time.sleep(delay)
+    return None
+
+def stock_zh_a_spot_em(fs='m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048', pz=200) -> pd.DataFrame:
+    """
+    东方财富网-沪深京 A 股-实时行情
+    https://quote.eastmoney.com/center/gridlist.html#hs_a_board
+    """
+    url = "http://82.push2.eastmoney.com/api/qt/clist/get"
+    pn = 1  # 初始页数
+    pn_max = 10000  # 设定初始最大页数
+    all_data = []
+    
+    while pn <= pn_max:
+        params = {
+            "pn": str(pn),
+            "pz": str(pz),
+            "po": "1",
+            "np": "1",
+            "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+            "fltt": "2",
+            "invt": "2",
+            "fid": "f3",
+            "fs": fs,
+            "fields": "f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f14,f15,f16,f17,f18,f20,f21,f22,f23,f24,f25,f26,f37,f38,f39,f40,f41,f45,f46,f48,f49,f57,f61,f100,f112,f113,f114,f115,f221",
+            "_": "1623833739532",
+        }
+        
+        data_json = fetch_with_retries_em(url, params)
+        if not data_json or "data" not in data_json or "diff" not in data_json["data"]:
+            break
+        
+        diff_data = data_json["data"]["diff"]
+        if not diff_data:
+            break
+        
+        all_data.extend(diff_data)
+        
+        # 获取 total 数据来更新 pn_max
+        if pn == 1:
+            pn_max = (data_json["data"].get("total", 0) + pz - 1) // pz
+        
+        pn += 1
+        time.sleep(0.5)  # 防止请求过快
+    
+    if not all_data:
+        return pd.DataFrame()
+    
+    temp_df = pd.DataFrame(all_data)
+    column_map = {
+        "f2": "最新价", "f3": "涨跌幅", "f4": "涨跌额", "f5": "成交量", "f6": "成交额", "f7": "振幅", "f8": "换手率",
+        "f9": "市盈率动", "f10": "量比", "f11": "5分钟涨跌", "f12": "代码", "f14": "名称", "f15": "最高", "f16": "最低",
+        "f17": "今开", "f18": "昨收", "f20": "总市值", "f21": "流通市值", "f22": "涨速", "f23": "市净率", "f24": "60日涨跌幅",
+        "f25": "年初至今涨跌幅", "f26": "上市时间", "f37": "加权净资产收益率", "f38": "总股本", "f39": "已流通股份",
+        "f40": "营业收入", "f41": "营业收入同比增长", "f45": "归属净利润", "f46": "归属净利润同比增长", "f48": "每股未分配利润",
+        "f49": "毛利率", "f57": "资产负债率", "f61": "每股公积金", "f100": "所处行业", "f112": "每股收益", "f113": "每股净资产",
+        "f114": "市盈率静", "f115": "市盈率TTM", "f221": "报告期"
+    }
+    temp_df.rename(columns=column_map, inplace=True)
+    
+    numeric_columns = [
+        "最新价", "涨跌幅", "涨跌额", "成交量", "成交额", "振幅", "换手率", "量比", "今开", "最高", "最低", "昨收", "涨速", "5分钟涨跌", "60日涨跌幅",
+        "年初至今涨跌幅", "市盈率动", "市盈率TTM", "市盈率静", "市净率", "每股收益", "每股净资产", "每股公积金", "每股未分配利润",
+        "加权净资产收益率", "毛利率", "资产负债率", "营业收入", "营业收入同比增长", "归属净利润", "归属净利润同比增长", "总股本", "已流通股份",
+        "总市值", "流通市值"
+    ]
+    for col in numeric_columns:
+        temp_df[col] = pd.to_numeric(temp_df[col], errors="coerce")
+    
+    temp_df["报告期"] = pd.to_datetime(temp_df["报告期"], format='%Y%m%d', errors="coerce")
+    temp_df["上市时间"] = pd.to_datetime(temp_df["上市时间"], format='%Y%m%d', errors="coerce")
+    
+    return temp_df
+
+
+def stock_zh_a_spot_em_old(fs='m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048') -> pd.DataFrame:
     """
     东方财富网-沪深京 A 股-实时行情
     https://quote.eastmoney.com/center/gridlist.html#hs_a_board
@@ -308,6 +393,70 @@ def code_id_map_em_older() -> dict:
 
 @lru_cache()
 def code_id_map_em() -> dict:
+    url = "http://80.push2.eastmoney.com/api/qt/clist/get"
+    pz = 200  # 固定每页 200 条
+    pn = 1     # 初始页码
+    pn_max = 10000  # 预设一个较大的初始值
+    
+    params = {
+        "pn": str(pn),
+        "pz": str(pz),
+        "po": "1",
+        "np": "1",
+        "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+        "fltt": "2",
+        "invt": "2",
+        "fid": "f3",
+        "fs": "",
+        "fields": "f12,f13",
+        "_": "1623833739532",
+    }
+    
+    market_fs = {
+        "china_a": "m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048",
+        "hk": "m:128 t:3,m:128 t:4,m:128 t:1,m:128 t:2",
+        "us": "m:105,m:106,m:107"
+    }
+    
+    code_id_dict = {}
+    
+    for market_id, fs in market_fs.items():
+        params["fs"] = fs
+        pn = 1  # 每个市场都从第一页开始
+        total = 0
+        fetched_cnt = 0
+        while pn <= pn_max:
+            params["pn"] = str(pn)
+            data_json = fetch_with_retries_em(url, params)
+            
+            if not data_json or "data" not in data_json or "diff" not in data_json["data"]:
+                print(f"市场 {market_id} 数据获取失败或为空，跳过。")
+                break
+            
+            temp_df = pd.DataFrame(data_json["data"]["diff"])
+            temp_df["market_id"] = 1
+            
+            # 处理 total 以计算 pn_max
+            if pn == 1 and "total" in data_json["data"]:
+                total = int(data_json["data"]["total"])
+                pn_max = (total // pz) + 1  # 计算最大页数
+                print(f"市场 {market_id} 总数据量: {total}, 需要页数: {pn_max}")
+            
+            # 按 f13 进行分组并存入字典
+            grouped = temp_df.groupby('f13')
+            for id, group in grouped:
+                code_id_dict.update(dict.fromkeys(group["f12"], id))
+                fetched_cnt += len(group)
+                # print(f'获取 {market_id} 股票列表，f13: {id}, 股票数: {len(group)}, 已获取总股票数: {fetched_cnt}, 总股票数: {total}')
+            
+            pn += 1  # 翻页继续
+
+        print(f'获取 {market_id} 已获取总股票数: {fetched_cnt}, 总股票数: {total}')
+    
+    return code_id_dict
+
+@lru_cache()
+def code_id_map_em2() -> dict:
     """
     东方财富-股票和市场代码
     http://quote.eastmoney.com/center/gridlist.html#hs_a_board

stockapp/src/stat_growth_em.py

@@ -32,6 +32,8 @@ config.setup_logging()
 current_date = datetime.now().strftime("%Y%m%d")
 current_year = datetime.now().strftime("%Y")
 
+res_dir = config.global_stock_data_dir
+
 # 刷新代码列表，并返回
 def flush_code_map():
     code_id_map_em_df = his_em.code_id_map_em()
@@ -99,20 +101,30 @@ def load_index_codes():
     conn = pymysql.connect(**config.db_config)
     cursor = conn.cursor(pymysql.cursors.DictCursor)
     #沪深300
-    #cursor.execute("SELECT code, code_name FROM hs_index where index_code='000300' ")
+    #cursor.execute("SELECT code, code_name FROM index_hs where index_code='000300' ")
     #中证A500
-    #cursor.execute("SELECT code, code_name FROM hs_index where index_code='000510' ")
+    #cursor.execute("SELECT code, code_name FROM index_hs where index_code='000510' ")
     #沪深300和中证A500的并集，去重
-    #cursor.execute("SELECT DISTINCT CONCAT('index-', code) as code, code_name FROM hs_index where index_code IN ('000300', '000510') ")
+    #cursor.execute("SELECT DISTINCT CONCAT('index-', code) as code, code_name FROM index_hs where index_code IN ('000300', '000510') ")
     #沪深300和中证A500的合并，不去重
-    #cursor.execute("SELECT DISTINCT CONCAT(index_code , '-', code) as code, code_name FROM hs_index where index_code IN ('000300', '000510') ")
+    cursor.execute("SELECT DISTINCT CONCAT(index_code , '-', code) as code, code_name FROM index_hs where index_code IN ('000300', '000510') ")
     #沪深300、中证A500、中证A50、科创芯片、科创创业50，不去重
-    cursor.execute("SELECT DISTINCT CONCAT(index_code , '-', code) as code, code_name FROM hs_index where index_code IN ('000300', '000510', '930050', '000685', '931643') ")
+    #cursor.execute("SELECT DISTINCT CONCAT(index_code , '-', code) as code, code_name FROM index_hs where index_code IN ('000300', '000510', '930050', '000685', '931643') ")
 
     hs300_data = cursor.fetchall()
+
+    #港股国企指数成分股、恒生科技指数成分股等
+    cursor.execute("SELECT DISTINCT CONCAT(index_code , '-', code) as code, code_name FROM index_hk where index_code IN ('HSCEI', 'HSTECH') ")
+    hk_data = cursor.fetchall()
+
+    #美股中概股等
+    cursor.execute("SELECT DISTINCT CONCAT(index_code , '-', code) as code, code_name FROM index_us where index_code IN ('CN_US') ")
+    us_data = cursor.fetchall()
+
+
     cursor.close()
     conn.close()
-    return hs300_data
+    return hs300_data + hk_data + us_data
 
 # 读取富途自选股的指定分类股
 def load_futu_all_codes():
@@ -172,7 +184,14 @@ def calculate_stock_statistics(market, code, code_name):
             logging.warning(f"{code}, {code_name} 未找到0923的数据, 以 {defaut_row['日期']} 的数据来代替")
             row_0923 = defaut_row
 
-        # 获取1008收盘价
+        # 获取0930收盘价
+        try:
+            row_0930 = data[data['日期'] == '2024-09-30'].iloc[0]
+        except IndexError:
+            logging.warning(f"{code}, {code_name} 未找到0930的数据, 以 {defaut_row['日期']} 的数据来代替")
+            row_0930 = defaut_row
+
+        # 获取1008开盘价、收盘价
         try:
             row_1008 = data[data['日期'] == '2024-10-08'].iloc[0]
         except IndexError:
@@ -198,6 +217,7 @@ def calculate_stock_statistics(market, code, code_name):
         try:
             year_increase = (current_row['收盘'] / year_begin_row['收盘'] - 1)
             growth_0923 = (current_row['收盘'] / row_0923['收盘'] - 1) 
+            growth_0930 = (current_row['收盘'] / row_0930['收盘'] - 1) 
             growth_1008 = (current_row['收盘'] / row_1008['收盘'] - 1) 
             growth_1008_open = (current_row['收盘'] / row_1008['开盘'] - 1) 
             year_amplitude = (year_max_row['收盘'] / year_min_row['收盘'] - 1)
@@ -215,6 +235,7 @@ def calculate_stock_statistics(market, code, code_name):
             current_row['日期'], current_row['收盘'],
             year_begin_row['日期'], year_begin_row['收盘'],
             row_0923['日期'], row_0923['收盘'] ,
+            row_0930['日期'], row_0930['收盘'] ,
             row_1008['日期'], row_1008['开盘'] ,row_1008['收盘'] ,
             max_close_row['日期'], max_close_row['收盘'],
             min_close_row['日期'], min_close_row['收盘'],
@@ -222,6 +243,7 @@ def calculate_stock_statistics(market, code, code_name):
             year_min_row['日期'], year_min_row['收盘'],
             year_increase,
             growth_0923 if growth_0923 is not None else 'N/A',
+            growth_0930 if growth_0930 is not None else 'N/A',
             growth_1008 if growth_1008 is not None else 'N/A',
             growth_1008_open if growth_1008_open is not None else 'N/A',
             year_amplitude,
@@ -235,16 +257,18 @@ def calculate_stock_statistics(market, code, code_name):
         return None
 
 # 写入到文件中
-def write_to_csv(results, filename='../stock_statistics.csv'):
+def write_to_csv(results, filename):
     """将所有结果写入CSV文件"""
     try:
         with open(filename, mode='w', newline='', encoding='utf-8') as file:
             writer = csv.writer(file)
             # 写入表头
             writer.writerow([
-                "股市", "股票代码", "股票名称", "当前日期", "当前收盘", "年初日期", "年初收盘", "0923日期", "0923收盘", "1008日期", "1008开盘", "1008收盘",
+                "股市", "股票代码", "股票名称", "当前日期", "当前收盘", "年初日期", "年初收盘", 
-                "最高日期", "最高收盘", "最低日期", "最低收盘", "年内最高日期", "年内最高收盘", "年内最低日期", "年内最低收盘", "年内涨幅", "相比0923收盘价涨幅",
+                "0923日期", "0923收盘", "0930日期", "0930收盘", "1008日期", "1008开盘", "1008收盘",
-                "相比1008收盘价涨幅", "相比1008开盘价涨幅", "年内振幅", "最大振幅", "股价自最高点恢复", "市盈率TTM", "市净率", "总市值"
+                "最高日期", "最高收盘", "最低日期", "最低收盘", "年内最高日期", "年内最高收盘", "年内最低日期", "年内最低收盘", "年内涨幅", 
+                "相比0923收盘价涨幅", "相比0930收盘价涨幅", "相比1008收盘价涨幅", "相比1008开盘价涨幅", 
+                "年内振幅", "最大振幅", "股价自最高点恢复", "市盈率TTM", "市净率", "总市值"
             ])
             # 写入每行数据
             for result in results:
@@ -273,7 +297,7 @@ def main(list, debug):
     if snap_data.empty:
         logging.error(f"fetching snapshot data error!")
         return
-    file_name = f'../result/snapshot_em_{current_date}.csv'
+    file_name = f'{res_dir}/snapshot_em_{current_date}.csv'
     snap_data.to_csv(file_name, index=False, encoding='utf-8')
     logging.info(f"市场快照数据已经写入 CSV 文件 {file_name}\n\n")
 
@@ -304,7 +328,7 @@ def main(list, debug):
             break
     
     if all_results:
-        file_name = f'../result/stock_statistics_{list}_{current_date}'
+        file_name = f'{res_dir}/stock_statistics_{list}_{current_date}'
         if debug:
             file_name = f'{file_name}_debug'
         file_name = f'{file_name}.csv'
@@ -321,4 +345,5 @@ if __name__ == "__main__":
     args = parser.parse_args()
     
     # 调用主函数
+    #flush_code_map()
     main(args.list, args.debug)