research/resources
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modify scripts

Browse Source
This commit is contained in:
oscar
2025-03-16 15:19:52 +08:00
parent e136de53f2
commit dab493f8e7
5 changed files with 80 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
stockapp/src/config.py
View File

@ -6,7 +6,7 @@ from pathlib import Path
# MySQL 配置
db_config = {
'host': '172.18.0.4',
'host': 'testdb',
'user': 'root',
'password': 'mysqlpw',
'database': 'stockdb'

5
stockapp/src/crawling/stock_hist_em.py
View File

@ -23,7 +23,7 @@ def fetch_with_retries_em(url, params, max_retries=3, delay=2):
time.sleep(delay)
return None
def stock_zh_a_spot_em(fs='m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048', pz=200) -> pd.DataFrame:
def stock_zh_a_spot_em(fs='m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048', pz=100) -> pd.DataFrame:
"""
东方财富网-沪深京 A 股-实时行情
https://quote.eastmoney.com/center/gridlist.html#hs_a_board
@ -61,6 +61,7 @@ def stock_zh_a_spot_em(fs='m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048', p
# 获取 total 数据来更新 pn_max
if pn == 1:
pn_max = (data_json["data"].get("total", 0) + pz - 1) // pz
print(f'total pages: {pn_max}, total data lines: {data_json["data"].get("total", 0)}, curr lines: {len(diff_data)}, page size: {pz}')
pn += 1
time.sleep(0.5) # 防止请求过快
@ -440,7 +441,7 @@ def code_id_map_em() -> dict:
if pn == 1 and "total" in data_json["data"]:
total = int(data_json["data"]["total"])
pn_max = (total // pz) + 1 # 计算最大页数
print(f"市场 {market_id} 总数据量: {total}, 需要页数: {pn_max}")
print(f"市场 {market_id} 总数据量: {total}, 需要页数: {pn_max}, 当前获取数量: {len(temp_df)}, 每页最大拉取行数: {pz}")
# 按 f13 进行分组并存入字典
grouped = temp_df.groupby('f13')