add em stat scripts.

stockapp/src/crawling/stock_hist_em.py

@@ -10,7 +10,7 @@ import pandas as pd
 from functools import lru_cache
 
 
-def stock_zh_a_spot_em() -> pd.DataFrame:
+def stock_zh_a_spot_em(fs='m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048') -> pd.DataFrame:
     """
     东方财富网-沪深京 A 股-实时行情
     https://quote.eastmoney.com/center/gridlist.html#hs_a_board
@@ -27,7 +27,7 @@ def stock_zh_a_spot_em() -> pd.DataFrame:
         "fltt": "2",
         "invt": "2",
         "fid": "f3",
-        "fs": "m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048",
+        "fs": fs,
         "fields": "f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f14,f15,f16,f17,f18,f20,f21,f22,f23,f24,f25,f26,f37,f38,f39,f40,f41,f45,f46,f48,f49,f57,f61,f100,f112,f113,f114,f115,f221",
         "_": "1623833739532",
     }
@@ -162,9 +162,9 @@ def stock_zh_a_spot_em() -> pd.DataFrame:
 
     return temp_df
 
-
-@lru_cache()
-def code_id_map_em() -> dict:
+#原有版本，实现的比较繁琐，后面有个简化版本替代它。
+#@lru_cache()
+def code_id_map_em_older() -> dict:
     """
     东方财富-股票和市场代码
     http://quote.eastmoney.com/center/gridlist.html#hs_a_board
@@ -286,7 +286,7 @@ def code_id_map_em() -> dict:
         "invt": "2",
         "fid": "f3",
         "fs": "m:105,m:106,m:107",
-        "fields": "f12",
+        "fields": "f12,f13",
         "_": "1623833739532",
     }
     r = requests.get(url, params=params)
@@ -294,11 +294,62 @@ def code_id_map_em() -> dict:
     if not data_json["data"]["diff"]:
         return dict()
     temp_df_sz = pd.DataFrame(data_json["data"]["diff"])
-    temp_df_sz["us_all"] = 105
-    code_id_dict.update(dict(zip(temp_df_sz["f12"], temp_df_sz["us_all"])))
-    
+    # 把数据保存到字典中。按照f13的值分别存储
+    grouped = temp_df_sz.groupby('f13')
+    for id, group in grouped:
+        temp_df_sz[f"us_all_{id}"] = id
+        code_id_dict.update(dict(zip(group["f12"], str(id))))
+        #print(f"分组 f13 = {id}:")
+        #print(group)
+    #temp_df_sz["us_all"] = 105
+    #code_id_dict.update(dict(zip(temp_df_sz["f12"], temp_df_sz["us_all"])))
+    print(code_id_dict)
     return code_id_dict
 
+@lru_cache()
+def code_id_map_em() -> dict:
+    """
+    东方财富-股票和市场代码
+    http://quote.eastmoney.com/center/gridlist.html#hs_a_board
+    :return: 股票和市场代码
+    :rtype: dict
+    """
+    url = "http://80.push2.eastmoney.com/api/qt/clist/get"
+    params = {
+        "pn": "1",
+        "pz": "50000",
+        "po": "1",
+        "np": "1",
+        "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+        "fltt": "2",
+        "invt": "2",
+        "fid": "f3",
+        "fs": "m:1 t:2,m:1 t:23",
+        "fields": "f12,f13",
+        "_": "1623833739532",
+    }
+    market_fs = {"china_a": "m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048",
+                 "hk":      "m:128 t:3,m:128 t:4,m:128 t:1,m:128 t:2",
+                 "us":      "m:105,m:106,m:107"}
+    code_id_dict = dict()
+
+    for market_id, fs in market_fs.items():
+        params['fs'] = fs
+        r = requests.get(url, params=params)
+        data_json = r.json()
+        if not data_json["data"]["diff"]:
+            return dict()
+        temp_df = pd.DataFrame(data_json["data"]["diff"])
+        temp_df["market_id"] = 1
+        # 把数据保存到字典中。按照f13的值分别存储
+        grouped = temp_df.groupby('f13')
+        for id, group in grouped:
+            temp_df[f"{market_id}_{id}"] = id
+            #code_id_dict.update(dict(zip(group["f12"], str(id))))
+            code_id_dict.update(dict.fromkeys(group["f12"], id))
+            print(f'get {market_id} stock list. f13: {id}, stock count: {len(group)}')
+
+    return code_id_dict
 
 def stock_zh_a_hist(
     symbol: str = "000001",