refactor: 提取数据同步逻辑为抽象基类
新增 base_sync.py 模块,提供三层抽象结构统一数据同步流程: - BaseDataSync: 所有同步类型的基础抽象(客户端、股票代码获取、交易日历) - StockBasedSync: 按股票同步抽象类(适用于 daily, pro_bar) - DateBasedSync: 按日期同步抽象类(适用于 bak_basic)
This commit is contained in:
161
src/data/sync.py
161
src/data/sync.py
@@ -3,7 +3,8 @@
|
||||
该模块作为数据同步的调度中心,统一管理各类型数据的同步流程。
|
||||
具体的同步逻辑已迁移到对应的 api_xxx.py 文件中:
|
||||
- api_daily.py: 日线数据同步 (DailySync 类)
|
||||
- api_bak_basic.py: 历史股票列表同步
|
||||
- api_bak_basic.py: 历史股票列表同步 (BakBasicSync 类)
|
||||
- api_pro_bar.py: Pro Bar 数据同步 (ProBarSync 类)
|
||||
- api_stock_basic.py: 股票基本信息同步
|
||||
- api_trade_cal.py: 交易日历同步
|
||||
|
||||
@@ -30,6 +31,7 @@ import pandas as pd
|
||||
from src.data.api_wrappers import sync_all_stocks
|
||||
from src.data.api_wrappers.api_daily import sync_daily, preview_daily_sync
|
||||
from src.data.api_wrappers.api_pro_bar import sync_pro_bar
|
||||
from src.data.api_wrappers.api_bak_basic import sync_bak_basic
|
||||
|
||||
|
||||
def preview_sync(
|
||||
@@ -135,30 +137,31 @@ def sync_all_data(
|
||||
dry_run: bool = False,
|
||||
) -> Dict[str, pd.DataFrame]:
|
||||
"""同步所有数据类型(每日同步)。
|
||||
该函数按顺序同步所有可用的数据类型:
|
||||
1. 交易日历 (sync_trade_cal_cache)
|
||||
2. 股票基本信息 (sync_all_stocks)
|
||||
3. 日线市场数据 (sync_all)
|
||||
4. 历史股票列表 (sync_bak_basic)
|
||||
|
||||
注意:名称变更 (namechange) 不在自动同步中,如需同步请手动调用。
|
||||
该函数按顺序同步所有可用的数据类型:
|
||||
1. 交易日历 (sync_trade_cal_cache)
|
||||
2. 股票基本信息 (sync_all_stocks)
|
||||
3. Pro Bar 数据 (sync_pro_bar)
|
||||
4. 历史股票列表 (sync_bak_basic)
|
||||
|
||||
Args:
|
||||
force_full: 若为 True,强制所有数据类型完整重载
|
||||
max_workers: 日线数据同步的工作线程数(默认: 10)
|
||||
dry_run: 若为 True,仅显示将要同步的内容,不写入数据
|
||||
注意:名称变更 (namechange) 不在自动同步中,如需同步请手动调用。
|
||||
|
||||
Returns:
|
||||
映射数据类型到同步结果的字典
|
||||
Args:
|
||||
force_full: 若为 True,强制所有数据类型完整重载
|
||||
max_workers: 日线数据同步的工作线程数(默认: 10)
|
||||
dry_run: 若为 True,仅显示将要同步的内容,不写入数据
|
||||
|
||||
Example:
|
||||
>>> result = sync_all_data()
|
||||
>>>
|
||||
>>> # 强制完整重载
|
||||
>>> result = sync_all_data(force_full=True)
|
||||
>>>
|
||||
>>> # Dry run
|
||||
>>> result = sync_all_data(dry_run=True)
|
||||
Returns:
|
||||
映射数据类型到同步结果的字典
|
||||
|
||||
Example:
|
||||
>>> result = sync_all_data()
|
||||
>>>
|
||||
>>> # 强制完整重载
|
||||
>>> result = sync_all_data(force_full=True)
|
||||
>>>
|
||||
>>> # Dry run
|
||||
>>> result = sync_all_data(dry_run=True)
|
||||
"""
|
||||
results: Dict[str, pd.DataFrame] = {}
|
||||
|
||||
@@ -167,47 +170,29 @@ def sync_all_data(
|
||||
print("=" * 60)
|
||||
|
||||
# 1. Sync trade calendar (always needed first)
|
||||
print("\n[1/6] Syncing trade calendar cache...")
|
||||
print("\n[1/4] Syncing trade calendar cache...")
|
||||
try:
|
||||
from src.data.api_wrappers import sync_trade_cal_cache
|
||||
|
||||
sync_trade_cal_cache()
|
||||
results["trade_cal"] = pd.DataFrame()
|
||||
print("[1/6] Trade calendar: OK")
|
||||
print("[1/4] Trade calendar: OK")
|
||||
except Exception as e:
|
||||
print(f"[1/6] Trade calendar: FAILED - {e}")
|
||||
print(f"[1/4] Trade calendar: FAILED - {e}")
|
||||
results["trade_cal"] = pd.DataFrame()
|
||||
|
||||
# 2. Sync stock basic info
|
||||
print("\n[2/6] Syncing stock basic info...")
|
||||
print("\n[2/4] Syncing stock basic info...")
|
||||
try:
|
||||
sync_all_stocks()
|
||||
results["stock_basic"] = pd.DataFrame()
|
||||
print("[2/6] Stock basic: OK")
|
||||
print("[2/4] Stock basic: OK")
|
||||
except Exception as e:
|
||||
print(f"[2/6] Stock basic: FAILED - {e}")
|
||||
print(f"[2/4] Stock basic: FAILED - {e}")
|
||||
results["stock_basic"] = pd.DataFrame()
|
||||
|
||||
# # 3. Sync daily market data
|
||||
# print("\n[3/6] Syncing daily market data...")
|
||||
# try:
|
||||
# daily_result = sync_daily(
|
||||
# force_full=force_full,
|
||||
# max_workers=max_workers,
|
||||
# dry_run=dry_run,
|
||||
# )
|
||||
# results["daily"] = (
|
||||
# pd.concat(daily_result.values(), ignore_index=True)
|
||||
# if daily_result
|
||||
# else pd.DataFrame()
|
||||
# )
|
||||
# print("[3/6] Daily data: OK")
|
||||
# except Exception as e:
|
||||
# print(f"[3/6] Daily data: FAILED - {e}")
|
||||
# results["daily"] = pd.DataFrame()
|
||||
|
||||
# 4. Sync Pro Bar data
|
||||
print("\n[4/6] Syncing Pro Bar data (with adj, tor, vr)...")
|
||||
# 3. Sync Pro Bar data
|
||||
print("\n[3/4] Syncing Pro Bar data (with adj, tor, vr)...")
|
||||
try:
|
||||
pro_bar_result = sync_pro_bar(
|
||||
force_full=force_full,
|
||||
@@ -219,87 +204,19 @@ def sync_all_data(
|
||||
if pro_bar_result
|
||||
else pd.DataFrame()
|
||||
)
|
||||
print(f"[4/6] Pro Bar data: OK ({len(results['pro_bar'])} records)")
|
||||
print(f"[3/4] Pro Bar data: OK ({len(results['pro_bar'])} records)")
|
||||
except Exception as e:
|
||||
print(f"[4/6] Pro Bar data: FAILED - {e}")
|
||||
print(f"[3/4] Pro Bar data: FAILED - {e}")
|
||||
results["pro_bar"] = pd.DataFrame()
|
||||
|
||||
# 5. Sync stock historical list (bak_basic)
|
||||
print("\n[5/6] Syncing stock historical list (bak_basic)...")
|
||||
try:
|
||||
bak_basic_result = sync_bak_basic(force_full=force_full)
|
||||
results["bak_basic"] = bak_basic_result
|
||||
print(f"[5/6] Bak basic: OK ({len(bak_basic_result)} records)")
|
||||
except Exception as e:
|
||||
print(f"[5/6] Bak basic: FAILED - {e}")
|
||||
results["bak_basic"] = pd.DataFrame()
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 60)
|
||||
print("[sync_all_data] Sync Summary")
|
||||
print("=" * 60)
|
||||
for data_type, df in results.items():
|
||||
print(f" {data_type}: {len(df)} records")
|
||||
print("=" * 60)
|
||||
print("\nNote: namechange is NOT in auto-sync. To sync manually:")
|
||||
print(" from src.data.api_wrappers import sync_namechange")
|
||||
print(" sync_namechange(force=True)")
|
||||
|
||||
return results
|
||||
results: Dict[str, pd.DataFrame] = {}
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("[sync_all_data] Starting full data synchronization...")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. Sync trade calendar (always needed first)
|
||||
print("\n[1/5] Syncing trade calendar cache...")
|
||||
try:
|
||||
from src.data.api_wrappers import sync_trade_cal_cache
|
||||
|
||||
sync_trade_cal_cache()
|
||||
results["trade_cal"] = pd.DataFrame()
|
||||
print("[1/5] Trade calendar: OK")
|
||||
except Exception as e:
|
||||
print(f"[1/5] Trade calendar: FAILED - {e}")
|
||||
results["trade_cal"] = pd.DataFrame()
|
||||
|
||||
# 2. Sync stock basic info
|
||||
print("\n[2/5] Syncing stock basic info...")
|
||||
try:
|
||||
sync_all_stocks()
|
||||
results["stock_basic"] = pd.DataFrame()
|
||||
print("[2/5] Stock basic: OK")
|
||||
except Exception as e:
|
||||
print(f"[2/5] Stock basic: FAILED - {e}")
|
||||
results["stock_basic"] = pd.DataFrame()
|
||||
|
||||
# 3. Sync daily market data
|
||||
print("\n[3/5] Syncing daily market data...")
|
||||
try:
|
||||
daily_result = sync_daily(
|
||||
force_full=force_full,
|
||||
max_workers=max_workers,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
results["daily"] = (
|
||||
pd.concat(daily_result.values(), ignore_index=True)
|
||||
if daily_result
|
||||
else pd.DataFrame()
|
||||
)
|
||||
print("[3/5] Daily data: OK")
|
||||
except Exception as e:
|
||||
print(f"[3/5] Daily data: FAILED - {e}")
|
||||
results["daily"] = pd.DataFrame()
|
||||
|
||||
# 4. Sync stock historical list (bak_basic)
|
||||
print("\n[4/5] Syncing stock historical list (bak_basic)...")
|
||||
print("\n[4/4] Syncing stock historical list (bak_basic)...")
|
||||
try:
|
||||
bak_basic_result = sync_bak_basic(force_full=force_full)
|
||||
results["bak_basic"] = bak_basic_result
|
||||
print(f"[4/5] Bak basic: OK ({len(bak_basic_result)} records)")
|
||||
print(f"[4/4] Bak basic: OK ({len(bak_basic_result)} records)")
|
||||
except Exception as e:
|
||||
print(f"[4/5] Bak basic: FAILED - {e}")
|
||||
print(f"[4/4] Bak basic: FAILED - {e}")
|
||||
results["bak_basic"] = pd.DataFrame()
|
||||
|
||||
# Summary
|
||||
@@ -316,10 +233,6 @@ def sync_all_data(
|
||||
return results
|
||||
|
||||
|
||||
# 保留向后兼容的导入
|
||||
from src.data.api_wrappers import sync_bak_basic
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=" * 60)
|
||||
print("Data Sync Module")
|
||||
|
||||
Reference in New Issue
Block a user