liaozhaorun/ProStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(experiment): 新增因子排除机制并优化模型训练参数

Browse Source 
- 添加 EXCLUDED_FACTORS 列表支持批量排除效果不佳的因子
- 修复 LightGBM 树结构冲突，调整正则化和采样策略防过拟合
- 调整数据处理器配置，关闭模型自动保存
This commit is contained in:
liaozhaorun
2026-03-18 20:57:02 +08:00
parent 16f82d3458
commit 0a29506f45
3 changed files with 131 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
src/experiment/common.py
View File

@@ -11,7 +11,6 @@ import polars as pl
from src.factors import FactorEngine
# =============================================================================
# 日期范围配置（正确的 train/val/test 三分法）
# =============================================================================
@@ -22,7 +21,6 @@ VAL_END = "20241231"
TEST_START = "20250101"
TEST_END = "20261231"
# =============================================================================
# 因子配置
# =============================================================================
@@ -257,6 +255,49 @@ SELECTED_FACTORS = [
# 因子定义字典（完整因子库，用于存放尚未注册到metadata的因子）
FACTOR_DEFINITIONS = {}
# 需要排除的因子列表（这些因子不会被计算和使用）
# 用于临时屏蔽效果不好的因子，无需从 SELECTED_FACTORS 中删除
EXCLUDED_FACTORS: List[str] = [
'GTJA_alpha005',
'GTJA_alpha028',
'GTJA_alpha023',
'GTJA_alpha002',
'GTJA_alpha010',
'GTJA_alpha011',
'GTJA_alpha044',
'GTJA_alpha036',
'GTJA_alpha027',
'GTJA_alpha109',
'GTJA_alpha104',
'GTJA_alpha103',
'GTJA_alpha085',
'GTJA_alpha111',
'GTJA_alpha092',
'GTJA_alpha067',
'GTJA_alpha060',
'GTJA_alpha062',
'GTJA_alpha063',
'GTJA_alpha079',
'GTJA_alpha073',
'GTJA_alpha087',
'GTJA_alpha117',
'GTJA_alpha113',
'GTJA_alpha138',
'GTJA_alpha121',
'GTJA_alpha124',
'GTJA_alpha133',
'GTJA_alpha131',
'GTJA_alpha118',
'GTJA_alpha164',
'GTJA_alpha162',
'GTJA_alpha157',
'GTJA_alpha171',
'GTJA_alpha177',
'GTJA_alpha180',
'GTJA_alpha188',
'GTJA_alpha191',
]
def get_label_factor(label_name: str) -> dict:
"""获取Label因子定义字典。
@@ -276,52 +317,84 @@ def get_label_factor(label_name: str) -> dict:
# 辅助函数
# =============================================================================
def register_factors(
engine: FactorEngine,
selected_factors: List[str],
factor_definitions: dict,
label_factor: dict,
engine: FactorEngine,
selected_factors: List[str],
factor_definitions: dict,
label_factor: dict,
excluded_factors: Optional[List[str]] = None,
) -> List[str]:
"""注册因子。
selected_factors 从 metadata 查询，factor_definitions 用 DSL 表达式注册。
excluded_factors 中的因子会被排除，不参与计算。
Args:
engine: FactorEngine实例
selected_factors: 从metadata中选择的因子名称列表
factor_definitions: 通过表达式定义的因子字典
label_factor: label因子定义字典
excluded_factors: 需要排除的因子名称列表，默认为None
Returns:
特征列名称列表
特征列名称列表（已排除excluded_factors中的因子）
"""
print("=" * 80)
print("注册因子")
print("=" * 80)
# 处理排除列表
excluded = set(excluded_factors) if excluded_factors else set()
if excluded:
print(f"\n[排除因子] 以下 {len(excluded)} 个因子将被排除:")
for name in sorted(excluded):
print(f" - {name}")
# 过滤 SELECTED_FACTORS 中的因子（排除excluded_factors）
filtered_selected = [name for name in selected_factors if name not in excluded]
excluded_from_selected = set(selected_factors) - set(filtered_selected)
if excluded_from_selected:
print(
f"\n[排除详情] 从 SELECTED_FACTORS 排除 {len(excluded_from_selected)} 个因子"
)
# 注册 SELECTED_FACTORS 中的因子（已在 metadata 中）
print("\n注册特征因子（从 metadata）:")
for name in selected_factors:
for name in filtered_selected:
engine.add_factor(name)
print(f" - {name}")
# 过滤 FACTOR_DEFINITIONS 中的因子（排除excluded_factors）
filtered_definitions = {
name: expr for name, expr in factor_definitions.items() if name not in excluded
}
excluded_from_definitions = set(factor_definitions.keys()) - set(
filtered_definitions.keys()
)
if excluded_from_definitions:
print(
f"\n[排除详情] 从 FACTOR_DEFINITIONS 排除 {len(excluded_from_definitions)} 个因子"
)
# 注册 FACTOR_DEFINITIONS 中的因子（通过表达式，尚未在 metadata 中）
print("\n注册特征因子（表达式）:")
for name, expr in factor_definitions.items():
for name, expr in filtered_definitions.items():
engine.add_factor(name, expr)
print(f" - {name}: {expr}")
# 注册 label 因子（通过表达式）
# 注册 label 因子（通过表达式，label因子不受excluded_factors影响）
print("\n注册 Label 因子（表达式）:")
for name, expr in label_factor.items():
engine.add_factor(name, expr)
print(f" - {name}: {expr}")
# 特征列 = SELECTED_FACTORS + FACTOR_DEFINITIONS 的 keys
feature_cols = selected_factors + list(factor_definitions.keys())
# 特征列 = 过滤后的 SELECTED_FACTORS + 过滤后的 FACTOR_DEFINITIONS 的 keys
feature_cols = filtered_selected + list(filtered_definitions.keys())
print(f"\n特征因子数: {len(feature_cols)}")
print(f" - 来自 metadata: {len(selected_factors)}")
print(f" - 来自表达式: {len(factor_definitions)}")
print(f" - 来自 metadata: {len(filtered_selected)}")
print(f" - 来自表达式: {len(filtered_definitions)}")
if excluded:
print(f" - 已排除: {len(excluded)}")
print(f"Label: {list(label_factor.keys())[0]}")
print(f"已注册因子总数: {len(engine.list_registered())}")
@@ -329,11 +402,11 @@ def register_factors(
def prepare_data(
engine: FactorEngine,
feature_cols: List[str],
start_date: str,
end_date: str,
label_name: str,
engine: FactorEngine,
feature_cols: List[str],
start_date: str,
end_date: str,
label_name: str,
) -> pl.DataFrame:
"""准备数据。
@@ -391,11 +464,11 @@ def stock_pool_filter(df: pl.DataFrame) -> pl.Series:
"""
# 代码筛选（排除创业板、科创板、北交所）
code_filter = (
~df["ts_code"].str.starts_with("30") # 排除创业板
& ~df["ts_code"].str.starts_with("68") # 排除科创板
& ~df["ts_code"].str.starts_with("8") # 排除北交所
& ~df["ts_code"].str.starts_with("9") # 排除北交所
& ~df["ts_code"].str.starts_with("4") # 排除北交所
~df["ts_code"].str.starts_with("30") # 排除创业板
& ~df["ts_code"].str.starts_with("68") # 排除科创板
& ~df["ts_code"].str.starts_with("8") # 排除北交所
& ~df["ts_code"].str.starts_with("9") # 排除北交所
& ~df["ts_code"].str.starts_with("4") # 排除北交所
)
# 在已筛选的股票中，选取市值最小的500只
@@ -410,7 +483,6 @@ def stock_pool_filter(df: pl.DataFrame) -> pl.Series:
# 定义筛选所需的基础列
STOCK_FILTER_REQUIRED_COLUMNS = ["total_mv"]
# =============================================================================
# 输出配置
# =============================================================================
@@ -418,7 +490,7 @@ OUTPUT_DIR = "output"
SAVE_PREDICTIONS = True
# 模型保存配置
SAVE_MODEL = True # 是否保存模型
SAVE_MODEL = False # 是否保存模型
MODEL_SAVE_DIR = "models" # 模型保存目录
# Top N 配置：每日推荐股票数量
@@ -451,7 +523,7 @@ def get_output_path(model_type: str, test_start: str, test_end: str) -> str:
def get_model_save_path(
model_type: str, model_name: Optional[str] = None
model_type: str, model_name: Optional[str] = None
) -> Optional[str]:
"""生成模型保存路径。
@@ -477,10 +549,10 @@ def get_model_save_path(
def save_model_with_factors(
model,
model_path: str,
selected_factors: List[str],
factor_definitions: dict,
model,
model_path: str,
selected_factors: List[str],
factor_definitions: dict,
) -> None:
"""保存模型及关联的因子信息。

3
src/experiment/learn_to_rank.py
View File

@@ -41,6 +41,7 @@ from src.training.config import TrainingConfig
from src.experiment.common import (
SELECTED_FACTORS,
FACTOR_DEFINITIONS,
EXCLUDED_FACTORS,
get_label_factor,
register_factors,
prepare_data,
@@ -260,7 +261,7 @@ engine = FactorEngine()
# 2. 使用 metadata 定义因子
print("\n[2] 定义因子（从 metadata 注册）")
feature_cols = register_factors(
engine, SELECTED_FACTORS, FACTOR_DEFINITIONS, LABEL_FACTOR
engine, SELECTED_FACTORS, FACTOR_DEFINITIONS, LABEL_FACTOR, EXCLUDED_FACTORS
)
# 3. 准备数据

49
src/experiment/regression.py
View File

@@ -17,6 +17,7 @@ from src.training import (
Winsorizer,
NullFiller,
check_data_quality,
CrossSectionalStandardScaler,
)
from src.training.config import TrainingConfig
@@ -24,6 +25,7 @@ from src.training.config import TrainingConfig
from src.experiment.common import (
SELECTED_FACTORS,
FACTOR_DEFINITIONS,
EXCLUDED_FACTORS,
get_label_factor,
register_factors,
prepare_data,
@@ -60,24 +62,25 @@ LABEL_FACTOR = get_label_factor(LABEL_NAME)
# 模型参数配置
MODEL_PARAMS = {
"objective": "regression",
"metric": "mae", # 改为 MAE，对异常值更稳健
# 树结构控制（防过拟合核心）
# "num_leaves": 20, # 从31降为20，降低模型复杂度
# "max_depth": 16, # 显式限制深度，防止过度拟合噪声
# "min_child_samples": 50, # 叶子最小样本数，防止学习极端样本
# "min_child_weight": 0.001,
# 学习参数
"learning_rate": 0.01, # 降低学习率，配合更多树
"n_estimators": 1000, # 增加树数量，配合早停
# 采样策略（关键防过拟合）
"subsample": 0.8, # 每棵树随机采样80%数据（行采样）
"subsample_freq": 5, # 每5轮迭代进行一次 subsample
"colsample_bytree": 0.8, # 每棵树随机选择80%特征（列采样）
# 正则化
"reg_alpha": 0.1, # L1正则，增加稀疏性
"reg_lambda": 1.0, # L2正则，平滑权重
# 数值稳定性
# 基础设置
"objective": "regression_l1", # LightGBM 中 MAE 对应的目标函数推荐写 regression_l1
"metric": "mae",
# 1. 修复树结构冲突：深度设为5，叶子数必须<=32。
# 推荐设定为稍微小于满二叉树的数值（如 15~31），以增加树的不对称性，防止过拟合
"max_depth": 5,
"num_leaves": 24, # 修改：从 63 降为 24
"min_data_in_leaf": 100, # 修改：适当增大，金融数据噪音大，叶子节点数据越多越抗噪
# 2. 学习参数
"learning_rate": 0.01,
"n_estimators": 1500, # 修改：配合小学习率，树可以再多一点
# 3. 修复采样抖动：改为每棵树都重新采样
"subsample": 0.8,
"subsample_freq": 1, # 【关键修改】：从 5 改为 1。每轮都重采样，让抖动均匀化，而不是5轮来一次大抖动
"colsample_bytree": 0.8,
# 正则化（金融量化等高噪场景可适当加大）
"reg_alpha": 0.5, # 修改：适当提高L1，强迫模型只选最有效的因子
"reg_lambda": 1.0,
# 杂项
"verbose": -1,
"random_state": 42,
}
@@ -92,12 +95,12 @@ print("=" * 80)
# 1. 创建 FactorEngine（启用 metadata 功能）
print("\n[1] 创建 FactorEngine")
engine = FactorEngine(metadata_path="data/factors.jsonl")
engine = FactorEngine()
# 2. 使用 metadata 定义因子
print("\n[2] 定义因子（从 metadata 注册）")
feature_cols = register_factors(
engine, SELECTED_FACTORS, FACTOR_DEFINITIONS, LABEL_FACTOR
engine, SELECTED_FACTORS, FACTOR_DEFINITIONS, LABEL_FACTOR, EXCLUDED_FACTORS
)
target_col = LABEL_NAME
@@ -126,7 +129,7 @@ model = LightGBMModel(params=MODEL_PARAMS)
processors = [
NullFiller(feature_cols=feature_cols, strategy="mean"),
Winsorizer(feature_cols=feature_cols, lower=0.01, upper=0.99),
StandardScaler(feature_cols=feature_cols),
StandardScaler(feature_cols=feature_cols + [LABEL_NAME]),
]
# 7. 创建数据划分器（正确的 train/val/test 三分法）
@@ -230,7 +233,7 @@ print("-" * 60)
print(" [说明] 此检查在 fillna 等处理之前执行，用于发现数据问题")
print("\n 检查训练集...")
check_data_quality(train_data, feature_cols, raise_on_error=True)
check_data_quality(train_data, feature_cols, raise_on_error=False)
if "val_data" in locals() and val_data is not None:
print("\n 检查验证集...")
@@ -579,7 +582,7 @@ zero_importance = importance_gain[importance_gain == 0].index.tolist()
if zero_importance:
print(f"\n[低重要性特征] 以下{len(zero_importance)}个特征重要性为0，可考虑删除:")
for feat in zero_importance:
print(f" - {feat}")
print(f"'{feat}',")
else:
print("\n所有特征都有一定重要性")