refactor: 将表结构定义从 storage 迁移到各 API 文件
- 移除 storage.py 集中式建表逻辑，改为各 API 文件自管理 - base_sync.py 新增 ensure_table_exists() 和表探测机制 - api_daily/api_pro_bar/api_bak_basic 添加 TABLE_SCHEMA 定义 - api_financial_sync 添加完整利润表字段定义 - sync.py 更新职责文档，明确仅同步每日更新数据 - AGENTS.md 添加 v2.1 架构变更历史和 AI 行为准则
This commit is contained in:
@@ -62,164 +62,17 @@ class Storage:
|
||||
self._initialized = True
|
||||
|
||||
def _init_db(self):
|
||||
"""Initialize database connection and schema."""
|
||||
"""Initialize database connection and schema.
|
||||
|
||||
注意：建表语句已迁移到对应的 API 文件中，
|
||||
每个同步类负责自己的表结构定义和创建。
|
||||
参见：
|
||||
- api_daily.py: DailySync.TABLE_SCHEMA
|
||||
- api_pro_bar.py: ProBarSync.TABLE_SCHEMA
|
||||
- api_bak_basic.py: BakBasicSync.TABLE_SCHEMA
|
||||
- api_financial_sync.py: FinancialSync.TABLE_SCHEMAS
|
||||
"""
|
||||
self._connection = duckdb.connect(str(self.db_path))
|
||||
|
||||
# Create tables with schema validation
|
||||
self._connection.execute("""
|
||||
CREATE TABLE IF NOT EXISTS daily (
|
||||
ts_code VARCHAR(16) NOT NULL,
|
||||
trade_date DATE NOT NULL,
|
||||
open DOUBLE,
|
||||
high DOUBLE,
|
||||
low DOUBLE,
|
||||
close DOUBLE,
|
||||
pre_close DOUBLE,
|
||||
change DOUBLE,
|
||||
pct_chg DOUBLE,
|
||||
vol DOUBLE,
|
||||
amount DOUBLE,
|
||||
turnover_rate DOUBLE,
|
||||
volume_ratio DOUBLE,
|
||||
PRIMARY KEY (ts_code, trade_date)
|
||||
)
|
||||
""")
|
||||
|
||||
# Create composite index for query optimization (trade_date, ts_code)
|
||||
self._connection.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_date_code ON daily(trade_date, ts_code)
|
||||
""")
|
||||
|
||||
# Create financial_income table for income statement data
|
||||
# 完整的利润表字段（94列全部）
|
||||
self._connection.execute("""
|
||||
CREATE TABLE IF NOT EXISTS financial_income (
|
||||
ts_code VARCHAR(16) NOT NULL,
|
||||
ann_date DATE,
|
||||
f_ann_date DATE,
|
||||
end_date DATE NOT NULL,
|
||||
report_type INTEGER,
|
||||
comp_type INTEGER,
|
||||
end_type VARCHAR(10),
|
||||
basic_eps DOUBLE,
|
||||
diluted_eps DOUBLE,
|
||||
total_revenue DOUBLE,
|
||||
revenue DOUBLE,
|
||||
int_income DOUBLE,
|
||||
prem_earned DOUBLE,
|
||||
comm_income DOUBLE,
|
||||
n_commis_income DOUBLE,
|
||||
n_oth_income DOUBLE,
|
||||
n_oth_b_income DOUBLE,
|
||||
prem_income DOUBLE,
|
||||
out_prem DOUBLE,
|
||||
une_prem_reser DOUBLE,
|
||||
reins_income DOUBLE,
|
||||
n_sec_tb_income DOUBLE,
|
||||
n_sec_uw_income DOUBLE,
|
||||
n_asset_mg_income DOUBLE,
|
||||
oth_b_income DOUBLE,
|
||||
fv_value_chg_gain DOUBLE,
|
||||
invest_income DOUBLE,
|
||||
ass_invest_income DOUBLE,
|
||||
forex_gain DOUBLE,
|
||||
total_cogs DOUBLE,
|
||||
oper_cost DOUBLE,
|
||||
int_exp DOUBLE,
|
||||
comm_exp DOUBLE,
|
||||
biz_tax_surchg DOUBLE,
|
||||
sell_exp DOUBLE,
|
||||
admin_exp DOUBLE,
|
||||
fin_exp DOUBLE,
|
||||
assets_impair_loss DOUBLE,
|
||||
prem_refund DOUBLE,
|
||||
compens_payout DOUBLE,
|
||||
reser_insur_liab DOUBLE,
|
||||
div_payt DOUBLE,
|
||||
reins_exp DOUBLE,
|
||||
oper_exp DOUBLE,
|
||||
compens_payout_refu DOUBLE,
|
||||
insur_reser_refu DOUBLE,
|
||||
reins_cost_refund DOUBLE,
|
||||
other_bus_cost DOUBLE,
|
||||
operate_profit DOUBLE,
|
||||
non_oper_income DOUBLE,
|
||||
non_oper_exp DOUBLE,
|
||||
nca_disploss DOUBLE,
|
||||
total_profit DOUBLE,
|
||||
income_tax DOUBLE,
|
||||
n_income DOUBLE,
|
||||
n_income_attr_p DOUBLE,
|
||||
minority_gain DOUBLE,
|
||||
oth_compr_income DOUBLE,
|
||||
t_compr_income DOUBLE,
|
||||
compr_inc_attr_p DOUBLE,
|
||||
compr_inc_attr_m_s DOUBLE,
|
||||
ebit DOUBLE,
|
||||
ebitda DOUBLE,
|
||||
insurance_exp DOUBLE,
|
||||
undist_profit DOUBLE,
|
||||
distable_profit DOUBLE,
|
||||
rd_exp DOUBLE,
|
||||
fin_exp_int_exp DOUBLE,
|
||||
fin_exp_int_inc DOUBLE,
|
||||
transfer_surplus_rese DOUBLE,
|
||||
transfer_housing_imprest DOUBLE,
|
||||
transfer_oth DOUBLE,
|
||||
adj_lossgain DOUBLE,
|
||||
withdra_legal_surplus DOUBLE,
|
||||
withdra_legal_pubfund DOUBLE,
|
||||
withdra_biz_devfund DOUBLE,
|
||||
withdra_rese_fund DOUBLE,
|
||||
withdra_oth_ersu DOUBLE,
|
||||
workers_welfare DOUBLE,
|
||||
distr_profit_shrhder DOUBLE,
|
||||
prfshare_payable_dvd DOUBLE,
|
||||
comshare_payable_dvd DOUBLE,
|
||||
capit_comstock_div DOUBLE,
|
||||
net_after_nr_lp_correct DOUBLE,
|
||||
credit_impa_loss DOUBLE,
|
||||
net_expo_hedging_benefits DOUBLE,
|
||||
oth_impair_loss_assets DOUBLE,
|
||||
total_opcost DOUBLE,
|
||||
amodcost_fin_assets DOUBLE,
|
||||
oth_income DOUBLE,
|
||||
asset_disp_income DOUBLE,
|
||||
continued_net_profit DOUBLE,
|
||||
end_net_profit DOUBLE,
|
||||
update_flag VARCHAR(1),
|
||||
PRIMARY KEY (ts_code, end_date)
|
||||
update_flag VARCHAR(1),
|
||||
PRIMARY KEY (ts_code, end_date)
|
||||
)
|
||||
""")
|
||||
|
||||
# Create pro_bar table for pro bar data (with adj, tor, vr)
|
||||
self._connection.execute("""
|
||||
CREATE TABLE IF NOT EXISTS pro_bar (
|
||||
ts_code VARCHAR(16) NOT NULL,
|
||||
trade_date DATE NOT NULL,
|
||||
open DOUBLE,
|
||||
high DOUBLE,
|
||||
low DOUBLE,
|
||||
close DOUBLE,
|
||||
pre_close DOUBLE,
|
||||
change DOUBLE,
|
||||
pct_chg DOUBLE,
|
||||
vol DOUBLE,
|
||||
amount DOUBLE,
|
||||
tor DOUBLE,
|
||||
vr DOUBLE,
|
||||
adj_factor DOUBLE,
|
||||
PRIMARY KEY (ts_code, trade_date)
|
||||
)
|
||||
""")
|
||||
|
||||
# Create index for financial_income
|
||||
self._connection.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_financial_ann ON financial_income(ts_code, ann_date)
|
||||
""")
|
||||
|
||||
def save(self, name: str, data: pd.DataFrame, mode: str = "append") -> dict:
|
||||
"""Save data to DuckDB.
|
||||
|
||||
@@ -271,6 +124,11 @@ class Storage:
|
||||
self._connection.execute(f"DELETE FROM {name}")
|
||||
|
||||
# UPSERT: INSERT OR REPLACE
|
||||
columns = ', '.join(f'"{col}"' for col in data.columns)
|
||||
self._connection.execute(f"""
|
||||
INSERT OR REPLACE INTO {name} ({columns})
|
||||
SELECT {columns} FROM temp_data
|
||||
""")
|
||||
columns = ", ".join(data.columns)
|
||||
self._connection.execute(f"""
|
||||
INSERT OR REPLACE INTO {name} ({columns})
|
||||
|
||||
Reference in New Issue
Block a user