liaozhaorun/NewStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
cecbef02f6f9b88780d59830547a40a5cfd7f0dd
NewStock/main/train/Regression2.ipynb

2152 lines
228 KiB
Plaintext
Raw Normal View History

parallel测试
2025-05-26 21:34:36 +08:00
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "79a7758178bafdd3",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T12:46:06.987506Z",
"start_time": "2025-04-03T12:46:06.259551Z"
},
"jupyter": {
"source_hidden": true
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"e:\\PyProject\\NewStock\\main\\train\n"
]
}
],
"source": [
"%load_ext autoreload\n",
"%autoreload 2\n",
"\n",
"import gc\n",
"import os\n",
"import sys\n",
"sys.path.append('../../')\n",
"print(os.getcwd())\n",
"import pandas as pd\n",
"from main.factor.factor import get_rolling_factor, get_simple_factor\n",
"from main.utils.factor import read_industry_data\n",
"from main.utils.factor_processor import calculate_score\n",
"from main.utils.utils import read_and_merge_h5_data, merge_with_industry_data\n",
"\n",
"import warnings\n",
"\n",
"warnings.filterwarnings(\"ignore\")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "a79cafb06a7e0e43",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T12:47:00.212859Z",
"start_time": "2025-04-03T12:46:06.998047Z"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"daily data\n",
"daily basic\n",
"inner merge on ['ts_code', 'trade_date']\n",
"stk limit\n",
"left merge on ['ts_code', 'trade_date']\n",
"money flow\n",
"left merge on ['ts_code', 'trade_date']\n",
"cyq perf\n",
"left merge on ['ts_code', 'trade_date']\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 8665405 entries, 0 to 8665404\n",
"Data columns (total 32 columns):\n",
" # Column Dtype \n",
"--- ------ ----- \n",
" 0 ts_code object \n",
" 1 trade_date datetime64[ns]\n",
" 2 open float64 \n",
" 3 close float64 \n",
" 4 high float64 \n",
" 5 low float64 \n",
" 6 vol float64 \n",
" 7 pct_chg float64 \n",
" 8 turnover_rate float64 \n",
" 9 pe_ttm float64 \n",
" 10 circ_mv float64 \n",
" 11 total_mv float64 \n",
" 12 volume_ratio float64 \n",
" 13 is_st bool \n",
" 14 up_limit float64 \n",
" 15 down_limit float64 \n",
" 16 buy_sm_vol float64 \n",
" 17 sell_sm_vol float64 \n",
" 18 buy_lg_vol float64 \n",
" 19 sell_lg_vol float64 \n",
" 20 buy_elg_vol float64 \n",
" 21 sell_elg_vol float64 \n",
" 22 net_mf_vol float64 \n",
" 23 his_low float64 \n",
" 24 his_high float64 \n",
" 25 cost_5pct float64 \n",
" 26 cost_15pct float64 \n",
" 27 cost_50pct float64 \n",
" 28 cost_85pct float64 \n",
" 29 cost_95pct float64 \n",
" 30 weight_avg float64 \n",
" 31 winner_rate float64 \n",
"dtypes: bool(1), datetime64[ns](1), float64(29), object(1)\n",
"memory usage: 2.0+ GB\n",
"None\n"
]
}
],
"source": [
"from main.utils.utils import read_and_merge_h5_data\n",
"\n",
"print('daily data')\n",
"df = read_and_merge_h5_data('../../data/daily_data.h5', key='daily_data',\n",
" columns=['ts_code', 'trade_date', 'open', 'close', 'high', 'low', 'vol', 'pct_chg'],\n",
" df=None)\n",
"\n",
"print('daily basic')\n",
"df = read_and_merge_h5_data('../../data/daily_basic.h5', key='daily_basic',\n",
" columns=['ts_code', 'trade_date', 'turnover_rate', 'pe_ttm', 'circ_mv', 'total_mv', 'volume_ratio',\n",
" 'is_st'], df=df, join='inner')\n",
"\n",
"print('stk limit')\n",
"df = read_and_merge_h5_data('../../data/stk_limit.h5', key='stk_limit',\n",
" columns=['ts_code', 'trade_date', 'pre_close', 'up_limit', 'down_limit'],\n",
" df=df)\n",
"print('money flow')\n",
"df = read_and_merge_h5_data('../../data/money_flow.h5', key='money_flow',\n",
" columns=['ts_code', 'trade_date', 'buy_sm_vol', 'sell_sm_vol', 'buy_lg_vol', 'sell_lg_vol',\n",
" 'buy_elg_vol', 'sell_elg_vol', 'net_mf_vol'],\n",
" df=df)\n",
"print('cyq perf')\n",
"df = read_and_merge_h5_data('../../data/cyq_perf.h5', key='cyq_perf',\n",
" columns=['ts_code', 'trade_date', 'his_low', 'his_high', 'cost_5pct', 'cost_15pct',\n",
" 'cost_50pct',\n",
" 'cost_85pct', 'cost_95pct', 'weight_avg', 'winner_rate'],\n",
" df=df)\n",
"print(df.info())"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "cac01788dac10678",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T12:47:10.527104Z",
"start_time": "2025-04-03T12:47:00.488715Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"industry\n"
]
}
],
"source": [
"print('industry')\n",
"industry_df = read_and_merge_h5_data('../../data/industry_data.h5', key='industry_data',\n",
" columns=['ts_code', 'l2_code', 'in_date'],\n",
" df=None, on=['ts_code'], join='left')\n",
"\n",
"\n",
"def merge_with_industry_data(df, industry_df):\n",
" # 确保日期字段是 datetime 类型\n",
" df['trade_date'] = pd.to_datetime(df['trade_date'])\n",
" industry_df['in_date'] = pd.to_datetime(industry_df['in_date'])\n",
"\n",
" # 对 industry_df 按 ts_code 和 in_date 排序\n",
" industry_df_sorted = industry_df.sort_values(['in_date', 'ts_code'])\n",
"\n",
" # 对原始 df 按 ts_code 和 trade_date 排序\n",
" df_sorted = df.sort_values(['trade_date', 'ts_code'])\n",
"\n",
" # 使用 merge_asof 进行向后合并\n",
" merged = pd.merge_asof(\n",
" df_sorted,\n",
" industry_df_sorted,\n",
" by='ts_code', # 按 ts_code 分组\n",
" left_on='trade_date',\n",
" right_on='in_date',\n",
" direction='backward'\n",
" )\n",
"\n",
" # 获取每个 ts_code 的最早 in_date 记录\n",
" min_in_date_per_ts = (industry_df_sorted\n",
" .groupby('ts_code')\n",
" .first()\n",
" .reset_index()[['ts_code', 'l2_code']])\n",
"\n",
" # 填充未匹配到的记录（trade_date 早于所有 in_date 的情况）\n",
" merged['l2_code'] = merged['l2_code'].fillna(\n",
" merged['ts_code'].map(min_in_date_per_ts.set_index('ts_code')['l2_code'])\n",
" )\n",
"\n",
" # 保留需要的列并重置索引\n",
" result = merged.reset_index(drop=True)\n",
" return result\n",
"\n",
"\n",
"# 使用示例\n",
"df = merge_with_industry_data(df, industry_df)\n",
"# print(mdf[mdf['ts_code'] == '600751.SH'][['ts_code', 'trade_date', 'l2_code']])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "c4e9e1d31da6dba6",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T12:47:10.719252Z",
"start_time": "2025-04-03T12:47:10.541247Z"
},
"jupyter": {
"source_hidden": true
}
},
"outputs": [],
"source": [
"from main.factor.factor import *\n",
"\n",
"def calculate_indicators(df):\n",
" \"\"\"\n",
" 计算四个指标：当日涨跌幅、5日移动平均、RSI、MACD。\n",
" \"\"\"\n",
" df = df.sort_values('trade_date')\n",
" df['daily_return'] = (df['close'] - df['pre_close']) / df['pre_close'] * 100\n",
" # df['5_day_ma'] = df['close'].rolling(window=5).mean()\n",
" delta = df['close'].diff()\n",
" gain = delta.where(delta > 0, 0)\n",
" loss = -delta.where(delta < 0, 0)\n",
" avg_gain = gain.rolling(window=14).mean()\n",
" avg_loss = loss.rolling(window=14).mean()\n",
" rs = avg_gain / avg_loss\n",
" df['RSI'] = 100 - (100 / (1 + rs))\n",
"\n",
" # 计算MACD\n",
" ema12 = df['close'].ewm(span=12, adjust=False).mean()\n",
" ema26 = df['close'].ewm(span=26, adjust=False).mean()\n",
" df['MACD'] = ema12 - ema26\n",
" df['Signal_line'] = df['MACD'].ewm(span=9, adjust=False).mean()\n",
" df['MACD_hist'] = df['MACD'] - df['Signal_line']\n",
"\n",
" # 4. 情绪因子1：市场上涨比例（Up Ratio）\n",
" df['up_ratio'] = df['daily_return'].apply(lambda x: 1 if x > 0 else 0)\n",
" df['up_ratio_20d'] = df['up_ratio'].rolling(window=20).mean() # 过去20天上涨比例\n",
"\n",
" # 5. 情绪因子2：成交量变化率（Volume Change Rate）\n",
" df['volume_mean'] = df['vol'].rolling(window=20).mean() # 过去20天的平均成交量\n",
" df['volume_change_rate'] = (df['vol'] - df['volume_mean']) / df['volume_mean'] * 100 # 成交量变化率\n",
"\n",
" # 6. 情绪因子3：波动率（Volatility）\n",
" df['volatility'] = df['daily_return'].rolling(window=20).std() # 过去20天的日收益率标准差\n",
"\n",
" # 7. 情绪因子4：成交额变化率（Amount Change Rate）\n",
" df['amount_mean'] = df['amount'].rolling(window=20).mean() # 过去20天的平均成交额\n",
" df['amount_change_rate'] = (df['amount'] - df['amount_mean']) / df['amount_mean'] * 100 # 成交额变化率\n",
"\n",
" # df = sentiment_panic_greed_index(df)\n",
" # df = sentiment_market_breadth_proxy(df)\n",
" # df = sentiment_reversal_indicator(df)\n",
"\n",
" return df\n",
"\n",
"\n",
"def generate_index_indicators(h5_filename):\n",
" df = pd.read_hdf(h5_filename, key='index_data')\n",
" df['trade_date'] = pd.to_datetime(df['trade_date'], format='%Y%m%d')\n",
" df = df.sort_values('trade_date')\n",
"\n",
" # 计算每个ts_code的相关指标\n",
" df_indicators = []\n",
" for ts_code in df['ts_code'].unique():\n",
" df_index = df[df['ts_code'] == ts_code].copy()\n",
" df_index = calculate_indicators(df_index)\n",
" df_indicators.append(df_index)\n",
"\n",
" # 合并所有指数的结果\n",
" df_all_indicators = pd.concat(df_indicators, ignore_index=True)\n",
"\n",
" # 保留trade_date列，并将同一天的数据按ts_code合并成一行\n",
" df_final = df_all_indicators.pivot_table(\n",
" index='trade_date',\n",
" columns='ts_code',\n",
" values=['daily_return', \n",
" 'RSI', 'MACD', 'Signal_line', 'MACD_hist', \n",
" # 'sentiment_panic_greed_index',\n",
" 'up_ratio_20d', 'volume_change_rate', 'volatility',\n",
" 'amount_change_rate', 'amount_mean'],\n",
" aggfunc='last'\n",
" )\n",
"\n",
" df_final.columns = [f\"{col[1]}_{col[0]}\" for col in df_final.columns]\n",
" df_final = df_final.reset_index()\n",
"\n",
" return df_final\n",
"\n",
"\n",
"# 使用函数\n",
"h5_filename = '../../data/index_data.h5'\n",
"index_data = generate_index_indicators(h5_filename)\n",
"index_data = index_data.dropna()\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "a735bc02ceb4d872",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T12:47:10.821169Z",
"start_time": "2025-04-03T12:47:10.751831Z"
}
},
"outputs": [],
"source": [
"import talib\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "53f86ddc0677a6d7",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T12:47:15.944254Z",
"start_time": "2025-04-03T12:47:10.826179Z"
},
"jupyter": {
"source_hidden": true
},
"scrolled": true
},
"outputs": [],
"source": [
"from main.utils.factor import get_act_factor\n",
"\n",
"\n",
"def read_industry_data(h5_filename):\n",
" # 读取 H5 文件中所有的行业数据\n",
" industry_data = pd.read_hdf(h5_filename, key='sw_daily', columns=[\n",
" 'ts_code', 'trade_date', 'open', 'close', 'high', 'low', 'pe', 'pb', 'vol'\n",
" ]) # 假设 H5 文件的键是 'industry_data'\n",
" industry_data = industry_data.sort_values(by=['ts_code', 'trade_date'])\n",
" industry_data = industry_data.reindex()\n",
" industry_data['trade_date'] = pd.to_datetime(industry_data['trade_date'], format='%Y%m%d')\n",
"\n",
" grouped = industry_data.groupby('ts_code', group_keys=False)\n",
" industry_data['obv'] = grouped.apply(\n",
" lambda x: pd.Series(talib.OBV(x['close'].values, x['vol'].values), index=x.index)\n",
" )\n",
" industry_data['return_5'] = grouped['close'].apply(lambda x: x / x.shift(5) - 1)\n",
" industry_data['return_20'] = grouped['close'].apply(lambda x: x / x.shift(20) - 1)\n",
"\n",
" industry_data = get_act_factor(industry_data, cat=False)\n",
" industry_data = industry_data.sort_values(by=['trade_date', 'ts_code'])\n",
"\n",
" # # 计算每天每个 ts_code 的因子和当天所有 ts_code 的中位数的偏差\n",
" # factor_columns = ['obv', 'return_5', 'return_20', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4'] # 因子列\n",
" # \n",
" # for factor in factor_columns:\n",
" # if factor in industry_data.columns:\n",
" # # 计算每天每个 ts_code 的因子值与当天所有 ts_code 的中位数的偏差\n",
" # industry_data[f'{factor}_deviation'] = industry_data.groupby('trade_date')[factor].transform(\n",
" # lambda x: x - x.mean())\n",
"\n",
" industry_data['return_5_percentile'] = industry_data.groupby('trade_date')['return_5'].transform(\n",
" lambda x: x.rank(pct=True))\n",
" industry_data['return_20_percentile'] = industry_data.groupby('trade_date')['return_20'].transform(\n",
" lambda x: x.rank(pct=True))\n",
"\n",
" # cs_rank_intraday_range(industry_data)\n",
" # cs_rank_close_pos_in_range(industry_data)\n",
"\n",
" industry_data = industry_data.drop(columns=['open', 'close', 'high', 'low', 'pe', 'pb', 'vol'])\n",
"\n",
" industry_data = industry_data.rename(\n",
" columns={col: f'industry_{col}' for col in industry_data.columns if col not in ['ts_code', 'trade_date']})\n",
"\n",
" industry_data = industry_data.rename(columns={'ts_code': 'cat_l2_code'})\n",
" return industry_data\n",
"\n",
"\n",
"industry_df = read_industry_data('../../data/sw_daily.h5')\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "dbe2fd8021b9417f",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T12:47:15.969344Z",
"start_time": "2025-04-03T12:47:15.963327Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['ts_code', 'open', 'close', 'high', 'low', 'circ_mv', 'total_mv', 'is_st', 'up_limit', 'down_limit', 'buy_sm_vol', 'sell_sm_vol', 'buy_lg_vol', 'sell_lg_vol', 'buy_elg_vol', 'sell_elg_vol', 'net_mf_vol', 'his_low', 'his_high', 'cost_5pct', 'cost_15pct', 'cost_50pct', 'cost_85pct', 'cost_95pct', 'weight_avg', 'in_date']\n"
]
}
],
"source": [
"origin_columns = df.columns.tolist()\n",
"origin_columns = [col for col in origin_columns if\n",
" col not in ['turnover_rate', 'pe_ttm', 'volume_ratio', 'vol', 'pct_chg', 'l2_code', 'winner_rate']]\n",
"origin_columns = [col for col in origin_columns if col not in index_data.columns]\n",
"origin_columns = [col for col in origin_columns if 'cyq' not in col]\n",
"print(origin_columns)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "85c3e3d0235ffffa",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T12:47:16.089879Z",
"start_time": "2025-04-03T12:47:15.990101Z"
}
},
"outputs": [],
"source": [
"fina_indicator_df = read_and_merge_h5_data('../../data/fina_indicator.h5', key='fina_indicator',\n",
" columns=['ts_code', 'ann_date', 'undist_profit_ps', 'ocfps', 'bps'],\n",
" df=None)\n",
"cashflow_df = read_and_merge_h5_data('../../data/cashflow.h5', key='cashflow',\n",
" columns=['ts_code', 'ann_date', 'n_cashflow_act'],\n",
" df=None)\n",
"balancesheet_df = read_and_merge_h5_data('../../data/balancesheet.h5', key='balancesheet',\n",
" columns=['ts_code', 'ann_date', 'money_cap', 'total_liab'],\n",
" df=None)\n",
"top_list_df = read_and_merge_h5_data('../../data/top_list.h5', key='top_list',\n",
" columns=['ts_code', 'trade_date', 'reason'],\n",
" df=None)\n",
"\n",
"top_list_df = top_list_df.sort_values(by='trade_date', ascending=False).drop_duplicates(subset=['ts_code', 'trade_date'], keep='first').sort_values(by='trade_date')\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "92d84ce15a562ec6",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T13:08:01.612695Z",
"start_time": "2025-04-03T12:47:16.121802Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"使用 'ann_date' 作为财务数据生效日期。\n",
"警告: 从 financial_data_subset 中移除了 366 行，因为其 'ts_code' 或 'ann_date' 列存在空值。\n",
"使用 'ann_date' 作为财务数据生效日期。\n",
"警告: 从 financial_data_subset 中移除了 366 行，因为其 'ts_code' 或 'ann_date' 列存在空值。\n",
"开始计算因子: AR, BR (原地修改)...\n",
"因子 AR, BR 计算成功。\n",
"因子 AR, BR 计算流程结束。\n",
"使用 'ann_date' 作为财务数据生效日期。\n",
"使用 'ann_date' 作为财务数据生效日期。\n",
"使用 'ann_date' 作为财务数据生效日期。\n",
"使用 'ann_date' 作为财务数据生效日期。\n",
"警告: 从 financial_data_subset 中移除了 366 行，因为其 'ts_code' 或 'ann_date' 列存在空值。\n",
"计算 BBI...\n",
"--- 计算日级别偏离度 (使用 pct_chg) ---\n",
"--- 计算日级别动量基准 (使用 pct_chg) ---\n",
"日级别动量基准计算完成 (使用 pct_chg)。\n",
"日级别偏离度计算完成 (使用 pct_chg)。\n",
"--- 计算日级别行业偏离度 (使用 pct_chg 和行业基准) ---\n",
"--- 计算日级别行业动量基准 (使用 pct_chg 和 cat_l2_code) ---\n",
"错误: 计算日级别行业动量基准需要以下列: ['pct_chg', 'cat_l2_code', 'trade_date', 'ts_code']。\n",
"错误: 计算日级别行业偏离度需要以下列: ['pct_chg', 'daily_industry_positive_benchmark', 'daily_industry_negative_benchmark']。请先运行 daily_industry_momentum_benchmark(df)。\n",
"Index(['ts_code', 'trade_date', 'open', 'close', 'high', 'low', 'vol',\n",
" 'pct_chg', 'turnover_rate', 'pe_ttm', 'circ_mv', 'total_mv',\n",
" 'volume_ratio', 'is_st', 'up_limit', 'down_limit', 'buy_sm_vol',\n",
" 'sell_sm_vol', 'buy_lg_vol', 'sell_lg_vol', 'buy_elg_vol',\n",
" 'sell_elg_vol', 'net_mf_vol', 'his_low', 'his_high', 'cost_5pct',\n",
" 'cost_15pct', 'cost_50pct', 'cost_85pct', 'cost_95pct', 'weight_avg',\n",
" 'winner_rate', 'l2_code', 'undist_profit_ps', 'ocfps', 'AR', 'BR',\n",
" 'AR_BR', 'log_circ_mv', 'cashflow_to_ev_factor', 'book_to_price_ratio',\n",
" 'turnover_rate_mean_5', 'variance_20', 'bbi_ratio_factor',\n",
" 'daily_deviation', 'lg_elg_net_buy_vol', 'flow_lg_elg_intensity',\n",
" 'sm_net_buy_vol', 'flow_divergence_diff', 'flow_divergence_ratio',\n",
" 'total_buy_vol', 'lg_elg_buy_prop', 'flow_struct_buy_change',\n",
" 'lg_elg_net_buy_vol_change', 'flow_lg_elg_accel',\n",
" 'chip_concentration_range', 'chip_skewness', 'floating_chip_proxy',\n",
" 'cost_support_15pct_change', 'cat_winner_price_zone',\n",
" 'flow_chip_consistency', 'profit_taking_vs_absorb', '_is_positive',\n",
" '_is_negative', 'cat_is_positive', '_pos_returns', '_neg_returns',\n",
" '_pos_returns_sq', '_neg_returns_sq', 'upside_vol', 'downside_vol',\n",
" 'vol_ratio', 'return_skew', 'return_kurtosis', 'volume_change_rate',\n",
" 'cat_volume_breakout', 'turnover_deviation', 'cat_turnover_spike',\n",
" 'avg_volume_ratio', 'cat_volume_ratio_breakout', 'vol_spike',\n",
" 'vol_std_5', 'atr_14', 'atr_6', 'obv'],\n",
" dtype='object')\n",
"Calculating lg_flow_mom_corr_20_60...\n",
"Finished lg_flow_mom_corr_20_60.\n",
"Calculating lg_flow_accel...\n",
"Finished lg_flow_accel.\n",
"Calculating profit_pressure...\n",
"Finished profit_pressure.\n",
"Calculating underwater_resistance...\n",
"Finished underwater_resistance.\n",
"Calculating cost_conc_std_20...\n",
"Finished cost_conc_std_20.\n",
"Calculating profit_decay_20...\n",
"Finished profit_decay_20.\n",
"Calculating vol_amp_loss_20...\n",
"Finished vol_amp_loss_20.\n",
"Calculating vol_drop_profit_cnt_5...\n",
"Finished vol_drop_profit_cnt_5.\n",
"Calculating lg_flow_vol_interact_20...\n",
"Finished lg_flow_vol_interact_20.\n",
"Calculating cost_break_confirm_cnt_5...\n",
"Finished cost_break_confirm_cnt_5.\n",
"Calculating atr_norm_channel_pos_14...\n",
"Finished atr_norm_channel_pos_14.\n",
"Calculating turnover_diff_skew_20...\n",
"Finished turnover_diff_skew_20.\n",
"Calculating lg_sm_flow_diverge_20...\n",
"Finished lg_sm_flow_diverge_20.\n",
"Calculating pullback_strong_20_20...\n",
"Finished pullback_strong_20_20.\n",
"Calculating vol_wgt_hist_pos_20...\n",
"Finished vol_wgt_hist_pos_20.\n",
"Calculating vol_adj_roc_20...\n",
"Finished vol_adj_roc_20.\n",
"Calculating cs_rank_net_lg_flow_val...\n",
"Finished cs_rank_net_lg_flow_val.\n",
"Calculating cs_rank_flow_divergence...\n",
"Finished cs_rank_flow_divergence.\n",
"Calculating cs_rank_ind_adj_lg_flow...\n",
"Finished cs_rank_ind_adj_lg_flow.\n",
"Calculating cs_rank_elg_buy_ratio...\n",
"Finished cs_rank_elg_buy_ratio.\n",
"Calculating cs_rank_rel_profit_margin...\n",
"Finished cs_rank_rel_profit_margin.\n",
"Calculating cs_rank_cost_breadth...\n",
"Finished cs_rank_cost_breadth.\n",
"Calculating cs_rank_dist_to_upper_cost...\n",
"Finished cs_rank_dist_to_upper_cost.\n",
"Calculating cs_rank_winner_rate...\n",
"Finished cs_rank_winner_rate.\n",
"Calculating cs_rank_intraday_range...\n",
"Finished cs_rank_intraday_range.\n",
"Calculating cs_rank_close_pos_in_range...\n",
"Finished cs_rank_close_pos_in_range.\n",
"Calculating cs_rank_opening_gap...\n",
"Error calculating cs_rank_opening_gap: Missing 'pre_close' column. Assigning NaN.\n",
"Calculating cs_rank_pos_in_hist_range...\n",
"Finished cs_rank_pos_in_hist_range.\n",
"Calculating cs_rank_vol_x_profit_margin...\n",
"Finished cs_rank_vol_x_profit_margin.\n",
"Calculating cs_rank_lg_flow_price_concordance...\n",
"Finished cs_rank_lg_flow_price_concordance.\n",
"Calculating cs_rank_turnover_per_winner...\n",
"Finished cs_rank_turnover_per_winner.\n",
"Calculating cs_rank_ind_cap_neutral_pe (Placeholder - requires statsmodels)...\n",
"Finished cs_rank_ind_cap_neutral_pe (Placeholder).\n",
"Calculating cs_rank_volume_ratio...\n",
"Finished cs_rank_volume_ratio.\n",
"Calculating cs_rank_elg_buy_sell_sm_ratio...\n",
"Finished cs_rank_elg_buy_sell_sm_ratio.\n",
"Calculating cs_rank_cost_dist_vol_ratio...\n",
"Finished cs_rank_cost_dist_vol_ratio.\n",
"Calculating cs_rank_size...\n",
"Finished cs_rank_size.\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 4539678 entries, 0 to 4539677\n",
"Columns: 178 entries, ts_code to cs_rank_size\n",
"dtypes: bool(10), datetime64[ns](1), float64(162), int32(3), object(2)\n",
"memory usage: 5.7+ GB\n",
"None\n",
"['ts_code', 'trade_date', 'open', 'close', 'high', 'low', 'vol', 'pct_chg', 'turnover_rate', 'pe_ttm', 'circ_mv', 'total_mv', 'volume_ratio', 'is_st', 'up_limit', 'down_limit', 'buy_sm_vol', 'sell_sm_vol', 'buy_lg_vol', 'sell_lg_vol', 'buy_elg_vol', 'sell_elg_vol', 'net_mf_vol', 'his_low', 'his_high', 'cost_5pct', 'cost_15pct', 'cost_50pct', 'cost_85pct', 'cost_95pct', 'weight_avg', 'winner_rate', 'cat_l2_code', 'undist_profit_ps', 'ocfps', 'AR', 'BR', 'AR_BR', 'log_circ_mv', 'cashflow_to_ev_factor', 'book_to_price_ratio', 'turnover_rate_mean_5', 'variance_20', 'bbi_ratio_factor', 'daily_deviation', 'lg_elg_net_buy_vol', 'flow_lg_elg_intensity', 'sm_net_buy_vol', 'flow_divergence_diff', 'flow_divergence_ratio', 'total_buy_vol', 'lg_elg_buy_prop', 'flow_struct_buy_change', 'lg_elg_net_buy_vol_change', 'flow_lg_elg_accel', 'chip_concentration_range', 'chip_skewness', 'floating_chip_proxy', 'cost_support_15pct_change', 'cat_winner_price_zone', 'flow_chip_consistency', 'profit_taking_vs_absorb', 'cat_is_positive', 'upside_vol', 'downside_vol', 'vol_ratio', 'return_skew', 'return_kurtosis', 'volume_change_rate', 'cat_volume_breakout', 'turnover_deviation', 'cat_turnover_spike', 'avg_volume_ratio', 'cat_volume_ratio_breakout', 'vol_spike', 'vol_std_5', 'atr_14', 'atr_6', 'obv', 'maobv_6', 'rsi_3', 'return_5', 'return_20', 'std_return_5', 'std_return_90', 'std_return_90_2', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4', 'rank_act_factor1', 'rank_act_factor2', 'rank_act_factor3', 'cov', 'delta_cov', 'alpha_22_improved', 'alpha_003', 'alpha_007', 'alpha_013', 'vol_break', 'weight_roc5', 'price_cost_divergence', 'smallcap_concentration', 'cost_stability', 'high_cost_break_days', 'liquidity_risk', 'turnover_std', 'mv_volatility', 'volume_growth', 'mv_growth', 'momentum_factor', 'resonance_factor', 'log_close', 'cat_vol_spike', 'up', 'down', 'obv_maobv_6', 'std_return_5_over_std_return_90', 'std_return_90_minus_std_return_90_2', 'cat_af2', 'cat_af3', 'cat_af4', 'act_factor5', 'act_factor6', 'active_buy_volume_large', 'active_buy_volume_big', 'active_buy_volume_small', 'buy_lg_vol_minus_sell_lg_vol', 'buy_elg_vol_minus_sell_elg_vol', 'ctrl_strength', 'low_cost_dev', 'asymmetry', 'lock_factor', 'cat_vol_break', 'cost_atr_adj', 'cat_golden_resonance', 'mv_turnover_ratio', 'mv_adjusted_volume', 'mv_weighted_turnover', 'nonlinear_mv_volume', 'mv_volume_ratio', 'mv_momentum', 'lg_flow_mom_corr_20_60', 'lg_flow_accel', 'profit_pressure', 'underwater_resistance', 'cost_conc_std_20', 'profit_decay_20', 'vol_amp_loss_20', 'vol_drop_profit_cnt_5', 'lg_flow_vol_interact_20', 'cost_break_confirm_cnt_5', 'atr_norm_channel_pos_14', 'turnover_diff_skew_20', 'lg_sm_flow_diverge_20', 'pullback_strong_20_20', 'vol_wgt_hist_pos_20', 'vol_adj_roc_20', 'cs_rank_net_lg_flow_val', 'cs_rank_flow_divergence', 'cs_rank_ind_adj_lg_flow', 'cs_rank_elg_buy_ratio', 'cs_rank_rel_profit_margin', 'cs_rank_cost_breadth', 'cs_rank_dist_to_upper_cost', 'cs_rank_winner_rate', 'cs_rank_intraday_range', 'cs_rank_close_pos_in_range', 'cs_rank_opening_gap', 'cs_rank_pos_in_hist_range', 'cs_rank_vol_x_profit_margin', 'cs_rank_lg_flow_price_concordance', 'cs_rank_turnover_per_winner', 'cs_rank_ind_cap_neutral_pe', 'cs_rank_volume_ratio', 'cs_rank_elg_buy_sell_sm_ratio', 'cs_rank_cost_dist_vol_ratio', 'cs_rank_size']\n"
]
}
],
"source": [
"\n",
"import numpy as np\n",
"from main.factor.factor import *\n",
"\n",
"def filter_data(df):\n",
" # df = df.groupby('trade_date').apply(lambda x: x.nlargest(1000, 'act_factor1'))\n",
" df = df[~df['is_st']]\n",
" df = df[~df['ts_code'].str.endswith('BJ')]\n",
" df = df[~df['ts_code'].str.startswith('30')]\n",
" df = df[~df['ts_code'].str.startswith('68')]\n",
" df = df[~df['ts_code'].str.startswith('8')]\n",
" df = df[df['trade_date'] >= '2019-01-01']\n",
" if 'in_date' in df.columns:\n",
" df = df.drop(columns=['in_date'])\n",
" df = df.reset_index(drop=True)\n",
" return df\n",
"\n",
"gc.collect()\n",
"\n",
"df = filter_data(df)\n",
"df = df.sort_values(by=['ts_code', 'trade_date'])\n",
"\n",
"# df = price_minus_deduction_price(df, n=120)\n",
"# df = price_deduction_price_diff_ratio_to_sma(df, n=120)\n",
"# df = cat_price_vs_sma_vs_deduction_price(df, n=120)\n",
"# df = cat_reason(df, top_list_df)\n",
"# df = cat_is_on_top_list(df, top_list_df)\n",
"\n",
"df = add_financial_factor(df, fina_indicator_df, factor_value_col='undist_profit_ps')\n",
"df = add_financial_factor(df, fina_indicator_df, factor_value_col='ocfps')\n",
"calculate_arbr(df, N=26)\n",
"df['log_circ_mv'] = np.log(df['circ_mv'])\n",
"df = calculate_cashflow_to_ev_factor(df, cashflow_df, balancesheet_df)\n",
"df = caculate_book_to_price_ratio(df, fina_indicator_df)\n",
"df = turnover_rate_n(df, n=5)\n",
"df = variance_n(df, n=20)\n",
"df = bbi_ratio_factor(df)\n",
"df = daily_deviation(df)\n",
"df = daily_industry_deviation(df)\n",
"df, _ = get_rolling_factor(df)\n",
"df, _ = get_simple_factor(df)\n",
"\n",
"df = df.rename(columns={'l1_code': 'cat_l1_code'})\n",
"df = df.rename(columns={'l2_code': 'cat_l2_code'})\n",
"\n",
"lg_flow_mom_corr(df, N=20, M=60)\n",
"lg_flow_accel(df)\n",
"profit_pressure(df)\n",
"underwater_resistance(df)\n",
"cost_conc_std(df, N=20)\n",
"profit_decay(df, N=20)\n",
"vol_amp_loss(df, N=20)\n",
"vol_drop_profit_cnt(df, N=20, M=5)\n",
"lg_flow_vol_interact(df, N=20)\n",
"cost_break_confirm_cnt(df, M=5)\n",
"atr_norm_channel_pos(df, N=14)\n",
"turnover_diff_skew(df, N=20)\n",
"lg_sm_flow_diverge(df, N=20)\n",
"pullback_strong(df, N=20, M=20)\n",
"vol_wgt_hist_pos(df, N=20)\n",
"vol_adj_roc(df, N=20)\n",
"\n",
"cs_rank_net_lg_flow_val(df)\n",
"cs_rank_flow_divergence(df)\n",
"cs_rank_industry_adj_lg_flow(df) # Needs cat_l2_code\n",
"cs_rank_elg_buy_ratio(df)\n",
"cs_rank_rel_profit_margin(df)\n",
"cs_rank_cost_breadth(df)\n",
"cs_rank_dist_to_upper_cost(df)\n",
"cs_rank_winner_rate(df)\n",
"cs_rank_intraday_range(df)\n",
"cs_rank_close_pos_in_range(df)\n",
"cs_rank_opening_gap(df) # Needs pre_close\n",
"cs_rank_pos_in_hist_range(df) # Needs his_low, his_high\n",
"cs_rank_vol_x_profit_margin(df)\n",
"cs_rank_lg_flow_price_concordance(df)\n",
"cs_rank_turnover_per_winner(df)\n",
"cs_rank_ind_cap_neutral_pe(df) # Placeholder - needs external libraries\n",
"cs_rank_volume_ratio(df) # Needs volume_ratio\n",
"cs_rank_elg_buy_sell_sm_ratio(df)\n",
"cs_rank_cost_dist_vol_ratio(df) # Needs volume_ratio\n",
"cs_rank_size(df) # Needs circ_mv\n",
"\n",
"\n",
"# df = df.merge(index_data, on='trade_date', how='left')\n",
"\n",
"print(df.info())\n",
"print(df.columns.tolist())"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "b87b938028afa206",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T13:08:03.658725Z",
"start_time": "2025-04-03T13:08:02.469611Z"
}
},
"outputs": [],
"source": [
"from scipy.stats import ks_2samp, wasserstein_distance\n",
"\n",
"\n",
"def remove_shifted_features(train_data, test_data, feature_columns, ks_threshold=0.05, wasserstein_threshold=0.1,\n",
" importance_threshold=0.05):\n",
" dropped_features = []\n",
"\n",
" # **统计数据漂移**\n",
" numeric_columns = train_data.select_dtypes(include=['float64', 'int64']).columns\n",
" numeric_columns = [col for col in numeric_columns if col in feature_columns]\n",
" for feature in numeric_columns:\n",
" ks_stat, p_value = ks_2samp(train_data[feature], test_data[feature])\n",
" wasserstein_dist = wasserstein_distance(train_data[feature], test_data[feature])\n",
"\n",
" if p_value < ks_threshold or wasserstein_dist > wasserstein_threshold:\n",
" dropped_features.append(feature)\n",
"\n",
" print(f\"检测到 {len(dropped_features)} 个可能漂移的特征: {dropped_features}\")\n",
"\n",
" # **应用阈值进行最终筛选**\n",
" filtered_features = [f for f in feature_columns if f not in dropped_features]\n",
"\n",
" return filtered_features, dropped_features\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "f4f16d63ad18d1bc",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T13:08:03.670700Z",
"start_time": "2025-04-03T13:08:03.665739Z"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import statsmodels.api as sm # 用于中性化回归\n",
"from tqdm import tqdm # 可选，用于显示进度条\n",
"\n",
"# --- 常量 ---\n",
"epsilon = 1e-10 # 防止除零\n",
"\n",
"# --- 1. 中位数去极值 (MAD) ---\n",
"\n",
"def cs_mad_filter(df: pd.DataFrame,\n",
" features: list,\n",
" k: float = 3.0,\n",
" scale_factor: float = 1.4826):\n",
" \"\"\"\n",
" 对指定特征列进行截面 MAD 去极值处理 (原地修改)。\n",
"\n",
" 方法: 对每日截面数据，计算 median 和 MAD，\n",
" 将超出 [median - k * scale * MAD, median + k * scale * MAD] 范围的值\n",
" 替换为边界值 (Winsorization)。\n",
" scale_factor=1.4826 使得 MAD 约等于正态分布的标准差。\n",
"\n",
" Args:\n",
" df (pd.DataFrame): 输入 DataFrame，需包含 'trade_date' 和 features 列。\n",
" features (list): 需要处理的特征列名列表。\n",
" k (float): MAD 的倍数，用于确定边界。默认为 3.0。\n",
" scale_factor (float): MAD 的缩放因子。默认为 1.4826。\n",
"\n",
" WARNING: 此函数会原地修改输入的 DataFrame 'df'。\n",
" \"\"\"\n",
" print(f\"开始截面 MAD 去极值处理 (k={k})...\")\n",
" if not all(col in df.columns for col in features):\n",
" missing = [col for col in features if col not in df.columns]\n",
" print(f\"错误: DataFrame 中缺少以下特征列: {missing}。跳过去极值处理。\")\n",
" return\n",
"\n",
" grouped = df.groupby('trade_date')\n",
"\n",
" for col in tqdm(features, desc=\"MAD Filtering\"):\n",
" try:\n",
" # 计算截面中位数\n",
" median = grouped[col].transform('median')\n",
" # 计算截面 MAD (Median Absolute Deviation from Median)\n",
" mad = (df[col] - median).abs().groupby(df['trade_date']).transform('median')\n",
"\n",
" # 计算上下边界\n",
" lower_bound = median - k * scale_factor * mad\n",
" upper_bound = median + k * scale_factor * mad\n",
"\n",
" # 原地应用 clip\n",
" df[col] = np.clip(df[col], lower_bound, upper_bound)\n",
"\n",
" except KeyError:\n",
" print(f\"警告: 列 '{col}' 可能不存在或在分组中出错，跳过此列的 MAD 处理。\")\n",
" except Exception as e:\n",
" print(f\"警告: 处理列 '{col}' 时发生错误: {e}，跳过此列的 MAD 处理。\")\n",
"\n",
" print(\"截面 MAD 去极值处理完成。\")\n",
"\n",
"\n",
"# --- 2. 行业市值中性化 ---\n",
"\n",
"def cs_neutralize_industry_cap(df: pd.DataFrame,\n",
" features: list,\n",
" industry_col: str = 'cat_l2_code',\n",
" market_cap_col: str = 'circ_mv'):\n",
" \"\"\"\n",
" 对指定特征列进行截面行业和对数市值中性化 (原地修改)。\n",
" 使用 OLS 回归: feature ~ 1 + log(market_cap) + C(industry)\n",
" 将回归残差写回原特征列。\n",
"\n",
" Args:\n",
" df (pd.DataFrame): 输入 DataFrame，需包含 'trade_date', features 列,\n",
" industry_col, market_cap_col。\n",
" features (list): 需要处理的特征列名列表。\n",
" industry_col (str): 行业分类列名。\n",
" market_cap_col (str): 流通市值列名。\n",
"\n",
" WARNING: 此函数会原地修改输入的 DataFrame 'df' 的 features 列。\n",
" 计算量较大，可能耗时较长。\n",
" 需要安装 statsmodels 库 (pip install statsmodels)。\n",
" \"\"\"\n",
" print(\"开始截面行业市值中性化...\")\n",
" required_cols = features + ['trade_date', industry_col, market_cap_col]\n",
" if not all(col in df.columns for col in required_cols):\n",
" missing = [col for col in required_cols if col not in df.columns]\n",
" print(f\"错误: DataFrame 中缺少必需列: {missing}。无法进行中性化。\")\n",
" return\n",
"\n",
" # 预处理：计算 log 市值，处理 industry code 可能的 NaN\n",
" log_cap_col = '_log_market_cap'\n",
" df[log_cap_col] = np.log1p(df[market_cap_col]) # log1p 处理 0 值\n",
" # df[industry_col] = df[industry_col].cat.add_categories('UnknownIndustry')\n",
" # df[industry_col] = df[industry_col].fillna('UnknownIndustry') # 填充行业 NaN\n",
" # df[industry_col] = df[industry_col].astype('category') # 转为类别，ols 会自动处理\n",
"\n",
" dates = df['trade_date'].unique()\n",
" all_residuals = [] # 用于收集所有日期的残差\n",
"\n",
" for date in tqdm(dates, desc=\"Neutralizing\"):\n",
" daily_data = df.loc[df['trade_date'] == date, features + [log_cap_col, industry_col]].copy() # 使用 .loc 获取副本\n",
"\n",
" # 准备自变量 X (常数项 + log市值 + 行业哑变量)\n",
" X = daily_data[[log_cap_col]]\n",
" X = sm.add_constant(X, prepend=True) # 添加常数项\n",
" # 创建行业哑变量 (drop_first=True 避免共线性)\n",
" industry_dummies = pd.get_dummies(daily_data[industry_col], prefix=industry_col, drop_first=True)\n",
" industry_dummies = industry_dummies.astype(int)\n",
" X = pd.concat([X, industry_dummies], axis=1)\n",
"\n",
" daily_residuals = daily_data[[col for col in features]].copy() # 创建用于存储残差的df\n",
"\n",
" for col in features:\n",
" Y = daily_data[col]\n",
"\n",
" # 处理 NaN 值，确保 X 和 Y 在相同位置有有效值\n",
" valid_mask = Y.notna() & X.notna().all(axis=1)\n",
" if valid_mask.sum() < (X.shape[1] + 1): # 数据点不足以估计模型\n",
" print(f\"警告: 日期 {date}, 特征 {col} 有效数据不足 ({valid_mask.sum()}个)，无法中性化，填充 NaN。\")\n",
" daily_residuals[col] = np.nan\n",
" continue\n",
"\n",
" Y_valid = Y[valid_mask]\n",
" X_valid = X[valid_mask]\n",
"\n",
" # 执行 OLS 回归\n",
" try:\n",
" model = sm.OLS(Y_valid.to_numpy(), X_valid.to_numpy())\n",
" results = model.fit()\n",
" # 将残差填回对应位置\n",
" daily_residuals.loc[valid_mask, col] = results.resid\n",
" daily_residuals.loc[~valid_mask, col] = np.nan # 原本无效的位置填充 NaN\n",
" except Exception as e:\n",
" print(f\"警告: 日期 {date}, 特征 {col} 回归失败: {e}，填充 NaN。\")\n",
" daily_residuals[col] = np.nan\n",
" break\n",
"\n",
" all_residuals.append(daily_residuals)\n",
"\n",
" # 合并所有日期的残差结果\n",
" if all_residuals:\n",
" residuals_df = pd.concat(all_residuals)\n",
" # 将残差结果更新回原始 df (原地修改)\n",
" # 使用 update 比 merge 更适合基于索引的原地更新\n",
" # 确保 residuals_df 的索引与 df 中对应部分一致\n",
" df.update(residuals_df)\n",
" else:\n",
" print(\"没有有效的残差结果可以合并。\")\n",
"\n",
"\n",
" # 清理临时列\n",
" df.drop(columns=[log_cap_col], inplace=True)\n",
" print(\"截面行业市值中性化完成。\")\n",
"\n",
"\n",
"# --- 3. Z-Score 标准化 ---\n",
"\n",
"def cs_zscore_standardize(df: pd.DataFrame, features: list, epsilon: float = 1e-10):\n",
" \"\"\"\n",
" 对指定特征列进行截面 Z-Score 标准化 (原地修改)。\n",
" 方法: Z = (value - cross_sectional_mean) / (cross_sectional_std + epsilon)\n",
"\n",
" Args:\n",
" df (pd.DataFrame): 输入 DataFrame，需包含 'trade_date' 和 features 列。\n",
" features (list): 需要处理的特征列名列表。\n",
" epsilon (float): 防止除以零的小常数。\n",
"\n",
" WARNING: 此函数会原地修改输入的 DataFrame 'df'。\n",
" \"\"\"\n",
" print(\"开始截面 Z-Score 标准化...\")\n",
" if not all(col in df.columns for col in features):\n",
" missing = [col for col in features if col not in df.columns]\n",
" print(f\"错误: DataFrame 中缺少以下特征列: {missing}。跳过标准化处理。\")\n",
" return\n",
"\n",
" grouped = df.groupby('trade_date')\n",
"\n",
" for col in tqdm(features, desc=\"Standardizing\"):\n",
" try:\n",
" # 使用 transform 计算截面均值和标准差\n",
" mean = grouped[col].transform('mean')\n",
" std = grouped[col].transform('std')\n",
"\n",
" # 计算 Z-Score 并原地赋值\n",
" df[col] = (df[col] - mean) / (std + epsilon)\n",
"\n",
" except KeyError:\n",
" print(f\"警告: 列 '{col}' 可能不存在或在分组中出错，跳过此列的标准化处理。\")\n",
" except Exception as e:\n",
" print(f\"警告: 处理列 '{col}' 时发生错误: {e}，跳过此列的标准化处理。\")\n",
"\n",
" print(\"截面 Z-Score 标准化完成。\")\n",
"\n",
"def fill_nan_with_daily_median(df: pd.DataFrame, feature_columns: list[str]) -> pd.DataFrame:\n",
" \"\"\"\n",
" 对指定特征列进行每日截面中位数填充缺失值 (NaN)。\n",
"\n",
" 参数:\n",
" df (pd.DataFrame): 包含多日数据的DataFrame，需要包含 'trade_date' 和 feature_columns 中的列。\n",
" feature_columns (list[str]): 需要进行缺失值填充的特征列名称列表。\n",
"\n",
" 返回:\n",
" pd.DataFrame: 包含缺失值填充后特征列的DataFrame。在输入DataFrame的副本上操作。\n",
" \"\"\"\n",
" processed_df = df.copy() # 在副本上操作，保留原始数据\n",
"\n",
" # 确保 trade_date 是 datetime 类型以便正确分组\n",
" processed_df['trade_date'] = pd.to_datetime(processed_df['trade_date'])\n",
"\n",
" def _fill_daily_nan(group):\n",
" # group 是某一个交易日的 DataFrame\n",
"\n",
" # 遍历指定的特征列\n",
" for feature_col in feature_columns:\n",
" # 检查列是否存在于当前分组中\n",
" if feature_col in group.columns:\n",
" # 计算当日该特征的中位数\n",
" median_val = group[feature_col].median()\n",
"\n",
" # 使用当日中位数填充该特征列的 NaN 值\n",
" # inplace=True 会直接修改 group DataFrame\n",
" group[feature_col].fillna(median_val, inplace=True)\n",
" # else:\n",
" # print(f\"Warning: Feature column '{feature_col}' not found in daily group for {group['trade_date'].iloc[0]}. Skipping.\")\n",
"\n",
" return group\n",
"\n",
" # 按交易日期分组，并应用每日填充函数\n",
" # group_keys=False 避免将分组键添加到结果索引中\n",
" filled_df = processed_df.groupby('trade_date', group_keys=False).apply(_fill_daily_nan)\n",
"\n",
" return filled_df"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "40e6b68a91b30c79",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T13:08:04.694262Z",
"start_time": "2025-04-03T13:08:03.694904Z"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"\n",
"def remove_outliers_label_percentile(label: pd.Series, lower_percentile: float = 0.01, upper_percentile: float = 0.99,\n",
" log=True):\n",
" if not (0 <= lower_percentile < upper_percentile <= 1):\n",
" raise ValueError(\"Percentile values must satisfy 0 <= lower_percentile < upper_percentile <= 1.\")\n",
"\n",
" # Calculate lower and upper bounds based on percentiles\n",
" lower_bound = label.quantile(lower_percentile)\n",
" upper_bound = label.quantile(upper_percentile)\n",
"\n",
" # Filter out values outside the bounds\n",
" filtered_label = label[(label >= lower_bound) & (label <= upper_bound)]\n",
"\n",
" # Print the number of removed outliers\n",
" if log:\n",
" print(f\"Removed {len(label) - len(filtered_label)} outliers.\")\n",
" return filtered_label\n",
"\n",
"\n",
"def calculate_risk_adjusted_target(df, days=5):\n",
" df = df.sort_values(by=['ts_code', 'trade_date'])\n",
"\n",
" df['future_close'] = df.groupby('ts_code')['close'].shift(-days)\n",
" df['future_open'] = df.groupby('ts_code')['open'].shift(-1)\n",
" df['future_return'] = (df['future_close'] - df['future_open']) / df['future_open']\n",
"\n",
" df['future_volatility'] = df.groupby('ts_code')['future_return'].rolling(days, min_periods=1).std().reset_index(\n",
" level=0, drop=True)\n",
" sharpe_ratio = df['future_return'] * df['future_volatility']\n",
" sharpe_ratio.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
"\n",
" return sharpe_ratio\n",
"\n",
"\n",
"def calculate_score(df, days=5, lambda_param=1.0):\n",
" def calculate_max_drawdown(prices):\n",
" peak = prices.iloc[0] # 初始化峰值\n",
" max_drawdown = 0 # 初始化最大回撤\n",
"\n",
" for price in prices:\n",
" if price > peak:\n",
" peak = price # 更新峰值\n",
" else:\n",
" drawdown = (peak - price) / peak # 计算当前回撤\n",
" max_drawdown = max(max_drawdown, drawdown) # 更新最大回撤\n",
"\n",
" return max_drawdown\n",
"\n",
" def compute_stock_score(stock_df):\n",
" stock_df = stock_df.sort_values(by=['trade_date'])\n",
" future_return = stock_df['future_return']\n",
" # 使用已有的 pct_chg 字段计算波动率\n",
" volatility = stock_df['pct_chg'].rolling(days).std().shift(-days)\n",
" max_drawdown = stock_df['close'].rolling(days).apply(calculate_max_drawdown, raw=False).shift(-days)\n",
" score = future_return - lambda_param * max_drawdown\n",
" return score\n",
"\n",
" # # 确保 DataFrame 按照股票代码和交易日期排序\n",
" # df = df.sort_values(by=['ts_code', 'trade_date'])\n",
"\n",
" # 对每个股票分别计算 score\n",
" df['score'] = df.groupby('ts_code').apply(compute_stock_score).reset_index(level=0, drop=True)\n",
"\n",
" return df['score']\n",
"\n",
"\n",
"def remove_highly_correlated_features(df, feature_columns, threshold=0.9):\n",
" numeric_features = df[feature_columns].select_dtypes(include=[np.number]).columns.tolist()\n",
" if not numeric_features:\n",
" raise ValueError(\"No numeric features found in the provided data.\")\n",
"\n",
" corr_matrix = df[numeric_features].corr().abs()\n",
" upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n",
" to_drop = [column for column in upper.columns if any(upper[column] > threshold)]\n",
" remaining_features = [col for col in feature_columns if col not in to_drop\n",
" or 'act' in col or 'af' in col]\n",
" return remaining_features\n",
"\n",
"\n",
"def cross_sectional_standardization(df, features):\n",
" df_sorted = df.sort_values(by='trade_date') # 按时间排序\n",
" df_standardized = df_sorted.copy()\n",
"\n",
" for date in df_sorted['trade_date'].unique():\n",
" # 获取当前时间点的数据\n",
" current_data = df_standardized[df_standardized['trade_date'] == date]\n",
"\n",
" # 只对指定特征进行标准化\n",
" scaler = StandardScaler()\n",
" standardized_values = scaler.fit_transform(current_data[features])\n",
"\n",
" # 将标准化结果重新赋值回去\n",
" df_standardized.loc[df_standardized['trade_date'] == date, features] = standardized_values\n",
"\n",
" return df_standardized\n",
"\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
"\n",
"\n",
"def neutralize_manual_revised(df: pd.DataFrame, features: list, industry_col: str, mkt_cap_col: str) -> pd.DataFrame:\n",
" \"\"\"\n",
" 手动实现简单回归以提升速度，通过构建 Series 确保索引对齐。\n",
" 对特征在行业内部进行市值中性化。\n",
"\n",
" Args:\n",
" df: 输入的 DataFrame，包含特征、行业分类和市值列。\n",
" features: 需要进行中性化的特征列名列表。\n",
" industry_col: 行业分类列的列名。\n",
" mkt_cap_col: 市值列的列名。\n",
"\n",
" Returns:\n",
" 中性化后的 DataFrame。\n",
" \"\"\"\n",
"\n",
" df[mkt_cap_col] = pd.to_numeric(df[mkt_cap_col], errors='coerce')\n",
" df_cleaned = df.dropna(subset=[mkt_cap_col]).copy()\n",
" df_cleaned = df_cleaned[df_cleaned[mkt_cap_col] > 0].copy()\n",
"\n",
" if df_cleaned.empty:\n",
" print(\"警告: 清理市值异常值后 DataFrame 为空。\")\n",
" return df # 返回原始或空df，取决于清理前的状态\n",
"\n",
" processed_df = df\n",
"\n",
" for col in features:\n",
" if col not in df_cleaned.columns:\n",
" print(f\"警告: 特征列 '{col}' 不存在于清理后的 DataFrame 中，已跳过。\")\n",
" # 对于原始 df 中该列不存在的，在结果 df 中也保持原样（可能全是NaN）\n",
" processed_df[col] = df[col] if col in df.columns else np.nan\n",
" continue\n",
"\n",
" # 跳过对控制变量本身进行中性化\n",
" if col == mkt_cap_col or col == industry_col:\n",
" print(f\"警告: 特征列 '{col}' 是控制变量或内部使用的列，跳过中性化。\")\n",
" # 在结果 df 中也保持原样\n",
" processed_df[col] = df[col] if col in df.columns else np.nan\n",
" continue\n",
"\n",
" residual_series = pd.Series(index=df_cleaned.index, dtype=float)\n",
"\n",
" # 在分组前处理特征列的 NaN，只对有因子值的行进行回归计算\n",
" df_subset_factor = df_cleaned.dropna(subset=[col]).copy()\n",
"\n",
" if not df_subset_factor.empty:\n",
" for industry, group in df_subset_factor.groupby(industry_col):\n",
" x = group[mkt_cap_col] # 市值对数\n",
" y = group[col] # 因子值\n",
"\n",
" # 确保有足够的数据点 (>1) 且市值对数有方差 (>0) 进行回归计算\n",
" # 检查 np.var > 一个很小的正数，避免浮点数误差导致的零方差判断问题\n",
" if len(group) > 1 and np.var(x) > 1e-9:\n",
" try:\n",
" beta = np.cov(y, x)[0, 1] / np.var(x)\n",
" alpha = np.mean(y) - beta * np.mean(x)\n",
"\n",
" # 计算残差\n",
" resid = y - (alpha + beta * x)\n",
"\n",
" # 将计算出的残差存储到 residual_series 中，通过索引自动对齐\n",
" residual_series.loc[resid.index] = resid\n",
"\n",
" except Exception as e:\n",
" # 捕获可能的计算异常，例如np.cov或np.var因为极端数据报错\n",
" print(f\"警告: 在行业 {industry} 计算回归时发生错误: {e}。该组残差将设为原始值或 NaN。\")\n",
" # 此时该组的残差会保持 residual_series 初始化时的 NaN 或后续处理\n",
" # 也可以选择保留原始值：residual_series.loc[group.index] = group[col]\n",
"\n",
" else:\n",
" residual_series.loc[group.index] = group[col] # 保留原始因子值\n",
" processed_df.loc[residual_series.index, col] = residual_series\n",
"\n",
"\n",
" else:\n",
" processed_df[col] = np.nan # 或 df[col] if col in df.columns else np.nan\n",
"\n",
" return processed_df\n",
"\n",
"\n",
"import gc\n",
"\n",
"gc.collect()\n",
"\n",
"\n",
"def mad_filter(df, features, n=3):\n",
" for col in features:\n",
" median = df[col].median()\n",
" mad = np.median(np.abs(df[col] - median))\n",
" upper = median + n * mad\n",
" lower = median - n * mad\n",
" df[col] = np.clip(df[col], lower, upper) # 截断极值\n",
" return df\n",
"\n",
"\n",
"def percentile_filter(df, features, lower_percentile=0.01, upper_percentile=0.99):\n",
" for col in features:\n",
" # 按日期分组计算上下百分位数\n",
" lower_bound = df.groupby('trade_date')[col].transform(\n",
" lambda x: x.quantile(lower_percentile)\n",
" )\n",
" upper_bound = df.groupby('trade_date')[col].transform(\n",
" lambda x: x.quantile(upper_percentile)\n",
" )\n",
" # 截断超出范围的值\n",
" df[col] = np.clip(df[col], lower_bound, upper_bound)\n",
" return df\n",
"\n",
"\n",
"from scipy.stats import iqr\n",
"\n",
"\n",
"def iqr_filter(df, features):\n",
" for col in features:\n",
" df[col] = df.groupby('trade_date')[col].transform(\n",
" lambda x: (x - x.median()) / iqr(x) if iqr(x) != 0 else x\n",
" )\n",
" return df\n",
"\n",
"\n",
"def quantile_filter(df, features, lower_quantile=0.01, upper_quantile=0.99, window=60):\n",
" df = df.copy()\n",
" for col in features:\n",
" # 计算 rolling 统计量，需要按日期进行 groupby\n",
" rolling_lower = df.groupby('trade_date')[col].transform(lambda x: x.rolling(window=min(len(x), window)).quantile(lower_quantile))\n",
" rolling_upper = df.groupby('trade_date')[col].transform(lambda x: x.rolling(window=min(len(x), window)).quantile(upper_quantile))\n",
"\n",
" # 对数据进行裁剪\n",
" df[col] = np.clip(df[col], rolling_lower, rolling_upper)\n",
" \n",
" return df\n",
"\n",
"def select_top_features_by_rankic(df: pd.DataFrame, feature_columns: list, n: int, target_column: str = 'future_return') -> list:\n",
" \"\"\"\n",
" 计算给定特征与目标列的 RankIC，并返回 RankIC 绝对值最高的 n 个特征。\n",
"\n",
" Args:\n",
" df: 包含特征列和目标列的 Pandas DataFrame。\n",
" feature_columns: 包含所有待评估特征列名的列表。\n",
" n: 希望选取的 RankIC 绝对值最高的特征数量。\n",
" target_column: 目标列的名称，用于计算 RankIC。默认为 'future_return'。\n",
"\n",
" Returns:\n",
" 包含 RankIC 绝对值最高的 n 个特征列名的列表。\n",
" \"\"\"\n",
" numeric_columns = df.select_dtypes(include=['float64', 'int64']).columns\n",
" numeric_columns = [col for col in numeric_columns if col in feature_columns]\n",
" if target_column not in df.columns:\n",
" raise ValueError(f\"目标列 '{target_column}' 不存在于 DataFrame 中。\")\n",
"\n",
" rankic_scores = {}\n",
" for feature in numeric_columns:\n",
" if feature not in df.columns:\n",
" print(f\"警告: 特征列 '{feature}' 不存在于 DataFrame 中，已跳过。\")\n",
" continue\n",
"\n",
" # 计算特征与目标列的 RankIC (斯皮尔曼相关系数)\n",
" # dropna() 是为了处理缺失值，确保相关性计算不失败\n",
" valid_data = df[[feature, target_column]].dropna()\n",
" if len(valid_data) > 1: # 确保有足够的数据点进行相关性计算\n",
" # 计算斯皮尔曼相关性\n",
" correlation = valid_data[feature].corr(valid_data[target_column], method='spearman')\n",
" rankic_scores[feature] = abs(correlation) # 使用绝对值来衡量相关性强度\n",
" else:\n",
" rankic_scores[feature] = 0 # 数据不足，RankIC设为0或跳过\n",
"\n",
" # 将 RankIC 分数转换为 Series 便于排序\n",
" rankic_series = pd.Series(rankic_scores)\n",
"\n",
" # 按 RankIC 绝对值降序排序，选取前 n 个特征\n",
" # handle case where n might be larger than available features\n",
" n_actual = min(n, len(rankic_series))\n",
" top_features = rankic_series.sort_values(ascending=False).head(n_actual).index.tolist()\n",
" top_features = [col for col in feature_columns if col in top_features or col not in numeric_columns]\n",
" return top_features\n",
"\n",
"def create_deviation_within_dates(df, feature_columns):\n",
" groupby_col = 'cat_l2_code' # 使用 trade_date 进行分组\n",
" new_columns = {}\n",
" ret_feature_columns = feature_columns[:]\n",
"\n",
" # 自动选择所有数值型特征\n",
" num_features = [col for col in feature_columns if 'cat' not in col and 'index' not in col]\n",
"\n",
" # num_features = ['vol', 'pct_chg', 'turnover_rate', 'volume_ratio', 'cat_vol_spike', 'obv', 'maobv_6', 'return_5', 'return_10', 'return_20', 'std_return_5', 'std_return_15', 'std_return_90', 'std_return_90_2', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4', 'act_factor5', 'act_factor6', 'rank_act_factor1', 'rank_act_factor2', 'rank_act_factor3', 'active_buy_volume_large', 'active_buy_volume_big', 'active_buy_volume_small', 'alpha_022', 'alpha_003', 'alpha_007', 'alpha_013']\n",
" num_features = [col for col in num_features if 'cat' not in col and 'industry' not in col]\n",
" num_features = [col for col in num_features if 'limit' not in col]\n",
" num_features = [col for col in num_features if 'cyq' not in col]\n",
"\n",
" # 遍历所有数值型特征\n",
" for feature in num_features:\n",
" if feature == 'trade_date': # 不需要对 'trade_date' 计算偏差\n",
" continue\n",
"\n",
" # grouped_mean = df.groupby(['trade_date'])[feature].transform('mean')\n",
" # deviation_col_name = f'deviation_mean_{feature}'\n",
" # new_columns[deviation_col_name] = df[feature] - grouped_mean\n",
" # ret_feature_columns.append(deviation_col_name)\n",
"\n",
" grouped_mean = df.groupby(['trade_date', groupby_col])[feature].transform('mean')\n",
" deviation_col_name = f'deviation_mean_{feature}'\n",
" new_columns[deviation_col_name] = df[feature] - grouped_mean\n",
" ret_feature_columns.append(deviation_col_name)\n",
"\n",
" # 将新计算的偏差特征与原始 DataFrame 合并\n",
" df = pd.concat([df, pd.DataFrame(new_columns)], axis=1)\n",
"\n",
" # for feature in ['obv', 'return_20', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4']:\n",
" # df[f'deviation_industry_{feature}'] = df[feature] - df[f'industry_{feature}']\n",
"\n",
" return df, ret_feature_columns\n"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "47c12bb34062ae7a",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T14:57:50.841165Z",
"start_time": "2025-04-03T14:49:25.889057Z"
}
},
"outputs": [],
"source": [
"days = 5\n",
"validation_days = 120\n",
"\n",
"import gc\n",
"\n",
"gc.collect()\n",
"\n",
"df = df.sort_values(by=['ts_code', 'trade_date'])\n",
"df['future_return'] = df.groupby('ts_code', group_keys=False)['close'].apply(lambda x: x.shift(-days) / x - 1)\n",
"# df['future_return'] = (df.groupby('ts_code')['close'].shift(-days) - df.groupby('ts_code')['open'].shift(-1)) / \\\n",
"# df.groupby('ts_code')['open'].shift(-1)\n",
"df['future_return'] = np.log1p(df['future_return'])\n",
"\n",
"df['future_return_2'] = df.groupby('ts_code', group_keys=False)['close'].apply(lambda x: x.shift(-2) / x - 1)\n",
"# df['future_return'] = (df.groupby('ts_code')['close'].shift(-days) - df.groupby('ts_code')['open'].shift(-1)) / \\\n",
"# df.groupby('ts_code')['open'].shift(-1)\n",
"df['future_return_2'] = np.log1p(df['future_return_2'])\n",
"\n",
"df['cat_up_limit'] = df['pct_chg'] > 5\n",
"# df['label'] = (df.groupby('ts_code')['cat_up_limit']\n",
"# .rolling(window=5, min_periods=1).sum()\n",
"# .groupby('ts_code') # 再次按 ts_code 分组\n",
"# .shift(-5)\n",
"# .fillna(0) # 填充每个股票组最后的 NaN\n",
"# .astype(int)\n",
"# .reset_index(level=0, drop=True))\n",
"df['label'] = df.groupby('trade_date', group_keys=False)['future_return'].transform(\n",
" lambda x: pd.qcut(x, q=20, labels=False, duplicates='drop')\n",
")\n",
"filter_index = df['future_return'].between(df['future_return'].quantile(0.01), df['future_return'].quantile(0.99))\n",
"\n",
"# for col in [col for col in df.columns]:\n",
"# train_data[col] = train_data[col].astype('str')\n",
"# test_data[col] = test_data[col].astype('str')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "8f4dc587",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" ts_code trade_date open close high low vol pct_chg \\\n",
"976171 002193.SZ 2019-01-02 18.02 17.82 18.21 17.73 22513.62 -1.00 \n",
"976172 002193.SZ 2019-01-03 17.82 17.78 17.94 17.65 19523.21 -0.22 \n",
"976173 002193.SZ 2019-01-04 17.47 18.04 18.10 17.34 28094.69 1.46 \n",
"976174 002193.SZ 2019-01-07 18.17 18.31 18.39 18.00 23866.88 1.50 \n",
"976175 002193.SZ 2019-01-08 18.25 18.52 18.66 18.13 22853.46 1.15 \n",
"... ... ... ... ... ... ... ... ... \n",
"977704 002193.SZ 2025-05-19 11.74 11.95 11.97 11.61 94249.32 2.49 \n",
"977705 002193.SZ 2025-05-20 11.99 12.20 12.22 11.84 103040.57 2.09 \n",
"977706 002193.SZ 2025-05-21 12.20 11.97 12.28 11.82 83112.00 -1.89 \n",
"977707 002193.SZ 2025-05-22 11.95 11.80 12.17 11.72 88811.00 -1.42 \n",
"977708 002193.SZ 2025-05-23 11.74 11.51 11.90 11.51 93013.00 -2.46 \n",
"\n",
" turnover_rate pe_ttm ... cs_rank_lg_flow_price_concordance \\\n",
"976171 0.9771 23.0129 ... 0.621118 \n",
"976172 0.8473 22.9598 ... 0.230095 \n",
"976173 1.2193 23.3049 ... 0.507119 \n",
"976174 1.0358 23.6499 ... 0.315078 \n",
"976175 0.9918 23.9154 ... 0.248631 \n",
"... ... ... ... ... \n",
"977704 3.6012 NaN ... 0.224402 \n",
"977705 3.9371 NaN ... 0.124834 \n",
"977706 3.1757 NaN ... 0.803720 \n",
"977707 3.3934 NaN ... 0.667220 \n",
"977708 3.5540 NaN ... 0.308867 \n",
"\n",
" cs_rank_turnover_per_winner cs_rank_ind_cap_neutral_pe \\\n",
"976171 0.990869 NaN \n",
"976172 0.934283 NaN \n",
"976173 0.925182 NaN \n",
"976174 0.843796 NaN \n",
"976175 0.834672 NaN \n",
"... ... ... \n",
"977704 0.624335 NaN \n",
"977705 0.641102 NaN \n",
"977706 0.624709 NaN \n",
"977707 0.563268 NaN \n",
"977708 0.661242 NaN \n",
"\n",
" cs_rank_volume_ratio cs_rank_elg_buy_sell_sm_ratio \\\n",
"976171 0.710344 0.341855 \n",
"976172 0.444444 0.318912 \n",
"976173 0.489226 0.260036 \n",
"976174 0.250000 0.251095 \n",
"976175 0.510588 0.286679 \n",
"... ... ... \n",
"977704 0.234043 0.397274 \n",
"977705 0.344124 0.116534 \n",
"977706 0.221189 0.126370 \n",
"977707 0.412155 0.130521 \n",
"977708 0.534540 0.134175 \n",
"\n",
" cs_rank_cost_dist_vol_ratio cs_rank_size future_return \\\n",
"976171 0.261603 0.262235 0.033658 \n",
"976172 0.185342 0.264695 0.029373 \n",
"976173 0.211959 0.259489 0.025179 \n",
"976174 0.145266 0.255474 0.014638 \n",
"976175 0.202299 0.259854 0.003235 \n",
"... ... ... ... \n",
"977704 0.299535 0.031250 NaN \n",
"977705 0.339641 0.032205 NaN \n",
"977706 0.291597 0.030555 NaN \n",
"977707 0.372634 0.030555 NaN \n",
"977708 0.430090 0.027898 NaN \n",
"\n",
" cat_up_limit label \n",
"976171 False 8.0 \n",
"976172 False 7.0 \n",
"976173 False 10.0 \n",
"976174 False 13.0 \n",
"976175 False 6.0 \n",
"... ... ... \n",
"977704 False NaN \n",
"977705 False NaN \n",
"977706 False NaN \n",
"977707 False NaN \n",
"977708 False NaN \n",
"\n",
"[1538 rows x 181 columns]\n"
]
}
],
"source": [
"print(df[df['ts_code'] == '002193.SZ'])"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "29221dde",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"162\n"
]
}
],
"source": [
"feature_columns = [col for col in df.head(10)\n",
" .merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
" # .merge(index_data, on='trade_date', how='left')\n",
" .columns\n",
" ]\n",
"feature_columns = [col for col in feature_columns if col not in ['trade_date',\n",
" 'ts_code',\n",
" 'label']]\n",
"feature_columns = [col for col in feature_columns if 'future' not in col]\n",
"feature_columns = [col for col in feature_columns if 'label' not in col]\n",
"feature_columns = [col for col in feature_columns if 'score' not in col]\n",
"feature_columns = [col for col in feature_columns if 'gen' not in col]\n",
"feature_columns = [col for col in feature_columns if 'is_st' not in col]\n",
"feature_columns = [col for col in feature_columns if 'pe_ttm' not in col]\n",
"# feature_columns = [col for col in feature_columns if 'volatility' not in col]\n",
"# feature_columns = [col for col in feature_columns if 'circ_mv' not in col]\n",
"feature_columns = [col for col in feature_columns if 'code' not in col]\n",
"feature_columns = [col for col in feature_columns if col not in origin_columns]\n",
"feature_columns = [col for col in feature_columns if not col.startswith('_')]\n",
"# feature_columns = [col for col in feature_columns if col not in ['ts_code', 'trade_date', 'vol_std_5', 'cov', 'delta_cov', 'alpha_22_improved', 'alpha_007', 'consecutive_up_limit', 'mv_volatility', 'volume_growth', 'mv_growth', 'arbr']]\n",
"feature_columns = [col for col in feature_columns if col not in ['intraday_lg_flow_corr_20', \n",
" 'cap_neutral_cost_metric', \n",
" 'hurst_net_mf_vol_60', \n",
" 'complex_factor_deap_1', \n",
" 'lg_buy_consolidation_20',\n",
" 'cs_rank_ind_cap_neutral_pe',\n",
" 'cs_rank_opening_gap',\n",
" 'cs_rank_ind_adj_lg_flow']]\n",
"feature_columns = [col for col in feature_columns if col not in ['cat_reason', 'cat_is_on_top_list']]\n",
"print(len(feature_columns))"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "03ee5daf",
"metadata": {},
"outputs": [],
"source": [
"# df = fill_nan_with_daily_median(df, feature_columns)\n",
"for feature_col in [col for col in feature_columns if col in df.columns]:\n",
" # median_val = df[feature_col].median()\n",
" df[feature_col].fillna(0, inplace=True)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"id": "b76ea08a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" ts_code trade_date log_circ_mv\n",
"0 000001.SZ 2019-01-02 16.574219\n",
"1 000001.SZ 2019-01-03 16.583965\n",
"2 000001.SZ 2019-01-04 16.633371\n",
"['vol', 'pct_chg', 'turnover_rate', 'volume_ratio', 'winner_rate', 'undist_profit_ps', 'ocfps', 'AR', 'BR', 'AR_BR', 'log_circ_mv', 'cashflow_to_ev_factor', 'book_to_price_ratio', 'turnover_rate_mean_5', 'variance_20', 'bbi_ratio_factor', 'daily_deviation', 'lg_elg_net_buy_vol', 'flow_lg_elg_intensity', 'sm_net_buy_vol', 'total_buy_vol', 'lg_elg_buy_prop', 'flow_struct_buy_change', 'lg_elg_net_buy_vol_change', 'flow_lg_elg_accel', 'chip_concentration_range', 'chip_skewness', 'floating_chip_proxy', 'cost_support_15pct_change', 'cat_winner_price_zone', 'flow_chip_consistency', 'profit_taking_vs_absorb', 'cat_is_positive', 'upside_vol', 'downside_vol', 'vol_ratio', 'return_skew', 'return_kurtosis', 'volume_change_rate', 'cat_volume_breakout', 'turnover_deviation', 'cat_turnover_spike', 'avg_volume_ratio', 'cat_volume_ratio_breakout', 'vol_spike', 'vol_std_5', 'atr_14', 'atr_6', 'obv', 'maobv_6', 'rsi_3', 'return_5', 'return_20', 'std_return_5', 'std_return_90', 'std_return_90_2', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4', 'rank_act_factor1', 'rank_act_factor2', 'rank_act_factor3', 'cov', 'delta_cov', 'alpha_22_improved', 'alpha_003', 'alpha_007', 'alpha_013', 'vol_break', 'weight_roc5', 'smallcap_concentration', 'cost_stability', 'high_cost_break_days', 'liquidity_risk', 'turnover_std', 'mv_volatility', 'volume_growth', 'mv_growth', 'momentum_factor', 'resonance_factor', 'log_close', 'cat_vol_spike', 'up', 'down', 'obv_maobv_6', 'std_return_5_over_std_return_90', 'std_return_90_minus_std_return_90_2', 'cat_af2', 'cat_af3', 'cat_af4', 'act_factor5', 'act_factor6', 'active_buy_volume_large', 'active_buy_volume_big', 'active_buy_volume_small', 'buy_lg_vol_minus_sell_lg_vol', 'buy_elg_vol_minus_sell_elg_vol', 'ctrl_strength', 'low_cost_dev', 'asymmetry', 'lock_factor', 'cat_vol_break', 'cost_atr_adj', 'cat_golden_resonance', 'mv_turnover_ratio', 'mv_adjusted_volume', 'mv_weighted_turnover', 'nonlinear_mv_volume', 'mv_volume_ratio', 'mv_momentum', 'lg_flow_mom_corr_20_60', 'lg_flow_accel', 'profit_pressure', 'underwater_resistance', 'cost_conc_std_20', 'profit_decay_20', 'vol_amp_loss_20', 'vol_drop_profit_cnt_5', 'lg_flow_vol_interact_20', 'cost_break_confirm_cnt_5', 'atr_norm_channel_pos_14', 'turnover_diff_skew_20', 'lg_sm_flow_diverge_20', 'pullback_strong_20_20', 'vol_wgt_hist_pos_20', 'vol_adj_roc_20', 'cs_rank_net_lg_flow_val', 'cs_rank_elg_buy_ratio', 'cs_rank_rel_profit_margin', 'cs_rank_cost_breadth', 'cs_rank_dist_to_upper_cost', 'cs_rank_winner_rate', 'cs_rank_intraday_range', 'cs_rank_close_pos_in_range', 'cs_rank_pos_in_hist_range', 'cs_rank_vol_x_profit_margin', 'cs_rank_lg_flow_price_concordance', 'cs_rank_turnover_per_winner', 'cs_rank_volume_ratio', 'cs_rank_elg_buy_sell_sm_ratio', 'cs_rank_cost_dist_vol_ratio', 'cs_rank_size', 'cat_up_limit', 'industry_obv', 'industry_return_5', 'industry_return_20', 'industry__ema_5', 'industry__ema_13', 'industry__ema_20', 'industry__ema_60', 'industry_act_factor1', 'industry_act_factor2', 'industry_act_factor3', 'industry_act_factor4', 'industry_act_factor5', 'industry_act_factor6', 'industry_rank_act_factor1', 'industry_rank_act_factor2', 'industry_rank_act_factor3', 'industry_return_5_percentile', 'industry_return_20_percentile']\n",
"去除极值\n",
"开始截面 MAD 去极值处理 (k=3.0)...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"MAD Filtering: 100%|██████████| 132/132 [00:28<00:00, 4.57it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"截面 MAD 去极值处理完成。\n",
"开始截面 MAD 去极值处理 (k=3.0)...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"MAD Filtering: 100%|██████████| 132/132 [00:24<00:00, 5.46it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"截面 MAD 去极值处理完成。\n",
"开始截面 MAD 去极值处理 (k=3.0)...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"MAD Filtering: 0it [00:00, ?it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"截面 MAD 去极值处理完成。\n",
"开始截面 MAD 去极值处理 (k=3.0)...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"MAD Filtering: 0it [00:00, ?it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"截面 MAD 去极值处理完成。\n",
"feature_columns: ['vol', 'pct_chg', 'turnover_rate', 'volume_ratio', 'winner_rate', 'undist_profit_ps', 'ocfps', 'AR', 'BR', 'AR_BR', 'log_circ_mv', 'cashflow_to_ev_factor', 'book_to_price_ratio', 'turnover_rate_mean_5', 'variance_20', 'bbi_ratio_factor', 'daily_deviation', 'lg_elg_net_buy_vol', 'flow_lg_elg_intensity', 'sm_net_buy_vol', 'total_buy_vol', 'lg_elg_buy_prop', 'flow_struct_buy_change', 'lg_elg_net_buy_vol_change', 'flow_lg_elg_accel', 'chip_concentration_range', 'chip_skewness', 'floating_chip_proxy', 'cost_support_15pct_change', 'cat_winner_price_zone', 'flow_chip_consistency', 'profit_taking_vs_absorb', 'cat_is_positive', 'upside_vol', 'downside_vol', 'vol_ratio', 'return_skew', 'return_kurtosis', 'volume_change_rate', 'cat_volume_breakout', 'turnover_deviation', 'cat_turnover_spike', 'avg_volume_ratio', 'cat_volume_ratio_breakout', 'vol_spike', 'vol_std_5', 'atr_14', 'atr_6', 'obv', 'maobv_6', 'rsi_3', 'return_5', 'return_20', 'std_return_5', 'std_return_90', 'std_return_90_2', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4', 'rank_act_factor1', 'rank_act_factor2', 'rank_act_factor3', 'cov', 'delta_cov', 'alpha_22_improved', 'alpha_003', 'alpha_007', 'alpha_013', 'vol_break', 'weight_roc5', 'smallcap_concentration', 'cost_stability', 'high_cost_break_days', 'liquidity_risk', 'turnover_std', 'mv_volatility', 'volume_growth', 'mv_growth', 'momentum_factor', 'resonance_factor', 'log_close', 'cat_vol_spike', 'up', 'down', 'obv_maobv_6', 'std_return_5_over_std_return_90', 'std_return_90_minus_std_return_90_2', 'cat_af2', 'cat_af3', 'cat_af4', 'act_factor5', 'act_factor6', 'active_buy_volume_large', 'active_buy_volume_big', 'active_buy_volume_small', 'buy_lg_vol_minus_sell_lg_vol', 'buy_elg_vol_minus_sell_elg_vol', 'ctrl_strength', 'low_cost_dev', 'asymmetry', 'lock_factor', 'cat_vol_break', 'cost_atr_adj', 'cat_golden_resonance', 'mv_turnover_ratio', 'mv_adjusted_volume', 'mv_weighted_turnover', 'nonlinear_mv_volume', 'mv_volume_ratio', 'mv_momentum', 'lg_flow_mom_corr_20_60', 'lg_flow_accel', 'profit_pressure', 'underwater_resistance', 'cost_conc_std_20', 'profit_decay_20', 'vol_amp_loss_20', 'vol_drop_profit_cnt_5', 'lg_flow_vol_interact_20', 'cost_break_confirm_cnt_5', 'atr_norm_channel_pos_14', 'turnover_diff_skew_20', 'lg_sm_flow_diverge_20', 'pullback_strong_20_20', 'vol_wgt_hist_pos_20', 'vol_adj_roc_20', 'cs_rank_net_lg_flow_val', 'cs_rank_elg_buy_ratio', 'cs_rank_rel_profit_margin', 'cs_rank_cost_breadth', 'cs_rank_dist_to_upper_cost', 'cs_rank_winner_rate', 'cs_rank_intraday_range', 'cs_rank_close_pos_in_range', 'cs_rank_pos_in_hist_range', 'cs_rank_vol_x_profit_margin', 'cs_rank_lg_flow_price_concordance', 'cs_rank_turnover_per_winner', 'cs_rank_volume_ratio', 'cs_rank_elg_buy_sell_sm_ratio', 'cs_rank_cost_dist_vol_ratio', 'cs_rank_size', 'cat_up_limit', 'industry_obv', 'industry_return_5', 'industry_return_20', 'industry__ema_5', 'industry__ema_13', 'industry__ema_20', 'industry__ema_60', 'industry_act_factor1', 'industry_act_factor2', 'industry_act_factor3', 'industry_act_factor4', 'industry_act_factor5', 'industry_act_factor6', 'industry_rank_act_factor1', 'industry_rank_act_factor2', 'industry_rank_act_factor3', 'industry_return_5_percentile', 'industry_return_20_percentile']\n",
"df最小日期: 2019-01-02\n",
"df最大日期: 2025-05-23\n",
"2057539\n",
"train_data最小日期: 2020-01-02\n",
"train_data最大日期: 2022-12-30\n",
"1766694\n",
"test_data最小日期: 2023-01-03\n",
"test_data最大日期: 2025-05-23\n",
" ts_code trade_date log_circ_mv\n",
"0 000001.SZ 2019-01-02 16.574219\n",
"1 000001.SZ 2019-01-03 16.583965\n",
"2 000001.SZ 2019-01-04 16.633371\n"
]
}
],
"source": [
"split_date = '2023-01-01'\n",
"train_data = df[filter_index & (df['trade_date'] <= split_date) & (df['trade_date'] >= '2020-01-01')]\n",
"test_data = df[(df['trade_date'] >= split_date)]\n",
"\n",
"print(df[['ts_code', 'trade_date', 'log_circ_mv']].head(3))\n",
"\n",
"industry_df = industry_df.sort_values(by=['trade_date'])\n",
"index_data = index_data.sort_values(by=['trade_date'])\n",
"\n",
"# train_data = train_data.merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
"# train_data = train_data.merge(index_data, on='trade_date', how='left')\n",
"# test_data = test_data.merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
"# test_data = test_data.merge(index_data, on='trade_date', how='left')\n",
"\n",
"train_data, test_data = train_data.replace([np.inf, -np.inf], np.nan), test_data.replace([np.inf, -np.inf], np.nan)\n",
"\n",
"# feature_columns_new = feature_columns[:]\n",
"# train_data, _ = create_deviation_within_dates(train_data, [col for col in feature_columns if col in train_data.columns])\n",
"# test_data, _ = create_deviation_within_dates(test_data, [col for col in feature_columns if col in train_data.columns])\n",
"\n",
"# feature_columns = [\n",
"# 'undist_profit_ps', \n",
"# 'AR_BR',\n",
"# 'pe_ttm',\n",
"# 'alpha_22_improved', \n",
"# 'alpha_003', \n",
"# 'alpha_007', \n",
"# 'alpha_013', \n",
"# 'cat_up_limit', \n",
"# 'cat_down_limit', \n",
"# 'up_limit_count_10d', \n",
"# 'down_limit_count_10d', \n",
"# 'consecutive_up_limit', \n",
"# 'vol_break', \n",
"# 'weight_roc5', \n",
"# 'price_cost_divergence', \n",
"# 'smallcap_concentration', \n",
"# 'cost_stability', \n",
"# 'high_cost_break_days', \n",
"# 'liquidity_risk', \n",
"# 'turnover_std', \n",
"# 'mv_volatility', \n",
"# 'volume_growth', \n",
"# 'mv_growth', \n",
"# 'lg_flow_mom_corr_20_60', \n",
"# 'lg_flow_accel', \n",
"# 'profit_pressure', \n",
"# 'underwater_resistance', \n",
"# 'cost_conc_std_20', \n",
"# 'profit_decay_20', \n",
"# 'vol_amp_loss_20', \n",
"# 'vol_drop_profit_cnt_5', \n",
"# 'lg_flow_vol_interact_20', \n",
"# 'cost_break_confirm_cnt_5', \n",
"# 'atr_norm_channel_pos_14', \n",
"# 'turnover_diff_skew_20', \n",
"# 'lg_sm_flow_diverge_20', \n",
"# 'pullback_strong_20_20', \n",
"# 'vol_wgt_hist_pos_20', \n",
"# 'vol_adj_roc_20',\n",
"# 'cashflow_to_ev_factor',\n",
"# 'ocfps',\n",
"# 'book_to_price_ratio',\n",
"# 'turnover_rate_mean_5',\n",
"# 'variance_20',\n",
"# 'bbi_ratio_factor'\n",
"# ]\n",
"# feature_columns = [col for col in feature_columns if col in train_data.columns]\n",
"# feature_columns = [col for col in feature_columns if not col.startswith('_')]\n",
"\n",
"numeric_columns = df.select_dtypes(include=['float64', 'int64']).columns\n",
"numeric_columns = [col for col in numeric_columns if col in feature_columns]\n",
"# feature_columns = select_top_features_by_rankic(df, numeric_columns, n=10)\n",
"print(feature_columns)\n",
"\n",
"# train_data = fill_nan_with_daily_median(train_data, feature_columns)\n",
"# test_data = fill_nan_with_daily_median(test_data, feature_columns)\n",
"\n",
"train_data = train_data.dropna(subset=[col for col in feature_columns if col in train_data.columns])\n",
"train_data = train_data.dropna(subset=['label'])\n",
"train_data = train_data.reset_index(drop=True)\n",
"# print(test_data.tail())\n",
"test_data = test_data.dropna(subset=[col for col in feature_columns if col in train_data.columns])\n",
"# test_data = test_data.dropna(subset=['label'])\n",
"test_data = test_data.reset_index(drop=True)\n",
"\n",
"transform_feature_columns = feature_columns\n",
"transform_feature_columns = [col for col in transform_feature_columns if col in feature_columns and not col.startswith('cat') and col in train_data.columns]\n",
"# transform_feature_columns.remove('undist_profit_ps')\n",
"print('去除极值')\n",
"cs_mad_filter(train_data, transform_feature_columns)\n",
"# print('中性化')\n",
"# cs_neutralize_industry_cap(train_data, transform_feature_columns)\n",
"# print('标准化')\n",
"# cs_zscore_standardize(train_data, transform_feature_columns)\n",
"\n",
"cs_mad_filter(test_data, transform_feature_columns)\n",
"# cs_neutralize_industry_cap(test_data, transform_feature_columns)\n",
"# cs_zscore_standardize(test_data, transform_feature_columns)\n",
"\n",
"mad_filter_feature_columns = [col for col in feature_columns if col not in transform_feature_columns and not col.startswith('cat') and col in train_data.columns]\n",
"cs_mad_filter(train_data, mad_filter_feature_columns)\n",
"cs_mad_filter(test_data, mad_filter_feature_columns)\n",
"\n",
"\n",
"print(f'feature_columns: {feature_columns}')\n",
"\n",
"\n",
"print(f\"df最小日期: {df['trade_date'].min().strftime('%Y-%m-%d')}\")\n",
"print(f\"df最大日期: {df['trade_date'].max().strftime('%Y-%m-%d')}\")\n",
"print(len(train_data))\n",
"print(f\"train_data最小日期: {train_data['trade_date'].min().strftime('%Y-%m-%d')}\")\n",
"print(f\"train_data最大日期: {train_data['trade_date'].max().strftime('%Y-%m-%d')}\")\n",
"print(len(test_data))\n",
"print(f\"test_data最小日期: {test_data['trade_date'].min().strftime('%Y-%m-%d')}\")\n",
"print(f\"test_data最大日期: {test_data['trade_date'].max().strftime('%Y-%m-%d')}\")\n",
"\n",
"cat_columns = [col for col in feature_columns if col.startswith('cat')]\n",
"for col in cat_columns:\n",
" train_data[col] = train_data[col].astype('category')\n",
" test_data[col] = test_data[col].astype('category')\n",
"\n",
"print(df[['ts_code', 'trade_date', 'log_circ_mv']].head(3))\n"
]
},
{
"cell_type": "code",
"execution_count": 37,
"id": "2e4b027e",
"metadata": {},
"outputs": [],
"source": [
"class RmseObjective(object):\n",
"\n",
" def is_max_optimal(self):\n",
" return False\n",
"\n",
" def get_final_error(self, error, weight):\n",
" return np.sqrt(error / (weight + 1e-38))\n",
"\n",
"\n",
" def evaluate(self, approxes, target, weight):\n",
" assert len(approxes) == 1\n",
" assert len(target) == len(approxes[0])\n",
"\n",
" approx = approxes[0]\n",
"\n",
" error_sum = 0.0\n",
" weight_sum = 0.0\n",
"\n",
" for i in range(len(approx)):\n",
" w = 1.0 if weight is None else weight[i]\n",
" weight_sum += w\n",
" error_sum += w * ((approx[i] - target[i])**2)\n",
"\n",
" return error_sum, weight_sum\n",
"\n",
" def calc_ders_range(self, approxes, targets, weights):\n",
" assert len(approxes) == len(targets)\n",
" if weights is not None:\n",
" assert len(weights) == len(approxes)\n",
"\n",
" result = []\n",
" for index in range(len(targets)):\n",
" der1 = targets[index] - approxes[index]\n",
" der2 = -1\n",
"\n",
" if weights is not None:\n",
" der1 *= weights[index]\n",
" der2 *= weights[index]\n",
"\n",
" result.append((der1, der2))\n",
" return result\n"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "3ff2d1c5",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.linear_model import LogisticRegression\n",
"import matplotlib.pyplot as plt # 保持 matplotlib 导入，尽管LightGBM的绘图功能已移除\n",
"from sklearn.decomposition import PCA\n",
"import pandas as pd\n",
"import numpy as np\n",
"import datetime # 用于日期计算\n",
"from catboost import CatBoostClassifier, CatBoostRanker, CatBoostRegressor\n",
"from catboost import Pool\n",
"import lightgbm as lgb\n",
"from lightgbm import LGBMClassifier, LGBMRegressor\n",
"\n",
"def train_model(train_data_df, feature_columns,\n",
" print_info=True, # 调整参数名，更通用\n",
" validation_days=180, use_pca=False, split_date=None,\n",
" target_column='label', type='light'): # 增加目标列参数\n",
"\n",
" print('train data size: ', len(train_data_df))\n",
" print(train_data_df[['ts_code', 'trade_date', 'log_circ_mv']])\n",
" # 确保数据按时间排序\n",
" train_data_df = train_data_df.sort_values(by='trade_date')\n",
"\n",
" # 识别数值型特征列\n",
" numeric_feature_columns = train_data_df[feature_columns].select_dtypes(include=['float64', 'int64']).columns.tolist()\n",
"\n",
" # 去除标签为空的样本\n",
" initial_len = len(train_data_df)\n",
" train_data_df = train_data_df.dropna(subset=[target_column])\n",
"\n",
" if print_info:\n",
" print(f'原始样本数: {initial_len}, 去除标签为空后样本数: {len(train_data_df)}')\n",
"\n",
" # 提取特征和标签，只取数值型特征用于线性回归\n",
" \n",
" if split_date is None:\n",
" all_dates = train_data_df['trade_date'].unique() # 获取所有唯一的 trade_date\n",
" split_date = all_dates[-validation_days] # 划分点为倒数第 validation_days 天\n",
" train_data_split = train_data_df[train_data_df['trade_date'] < split_date] # 训练集\n",
" val_data_split = train_data_df[train_data_df['trade_date'] >= split_date] # 验证集\n",
" \n",
" X_train = train_data_split[feature_columns]\n",
" y_train = train_data_split[target_column]\n",
" \n",
" X_val = val_data_split[feature_columns]\n",
" y_val = val_data_split[target_column]\n",
"\n",
"\n",
" # # 标准化数值特征 (使用 StandardScaler 对训练集fit并transform, 对验证集只transform)\n",
" scaler = StandardScaler()\n",
" # X_train = scaler.fit_transform(X_train)\n",
"\n",
" # 训练线性回归模型\n",
" # model = LogisticRegression(random_state=42)\n",
" \n",
" # # 使用处理后的特征和样本权重进行训练\n",
" # model.fit(X_train, y_train)\n",
"\n",
"\n",
" if type == 'cat':\n",
" params = {\n",
" 'loss_function': 'RMSE', # 适用于二分类\n",
" 'eval_metric': 'RMSE', # 评估指标\n",
" 'iterations': 1500,\n",
" 'learning_rate': 0.01,\n",
" 'depth': 8, # 控制模型复杂度\n",
" 'l2_leaf_reg': 1, # L2 正则化\n",
" 'verbose': 5000,\n",
" 'early_stopping_rounds': 300,\n",
" 'one_hot_max_size': 50,\n",
" # 'class_weights': [0.6, 1.2],\n",
" 'task_type': 'GPU',\n",
" 'has_time': True,\n",
" 'random_seed': 7\n",
" }\n",
" cat_features = [i for i, col in enumerate(feature_columns) if col.startswith('cat')]\n",
" group_train = train_data_split['trade_date'].factorize()[0]\n",
" group_val = val_data_split['trade_date'].factorize()[0]\n",
" train_pool = Pool(\n",
" data=X_train,\n",
" label=y_train,\n",
" # group_id=group_train,\n",
" cat_features=cat_features\n",
" )\n",
" val_pool = Pool(\n",
" data=X_val,\n",
" label=y_val,\n",
" # group_id=group_val,\n",
" cat_features=cat_features\n",
" )\n",
"\n",
" model = CatBoostRegressor(**params)\n",
" model.fit(train_pool,\n",
" eval_set=val_pool, \n",
" plot=True, \n",
" use_best_model=True\n",
" )\n",
" elif type == 'light':\n",
" label_gain = list(range(len(train_data_split['label'].unique())))\n",
" params = {\n",
" 'label_gain': [gain * gain for gain in label_gain],\n",
" 'objective': 'rmse',\n",
" 'metric': 'rmse',\n",
" 'learning_rate': 0.01,\n",
" 'num_leaves': 1024,\n",
" # 'min_data_in_leaf': 128,\n",
" 'max_depth': 100,\n",
" 'max_bin': 1024,\n",
" 'feature_fraction': 0.7,\n",
" 'bagging_fraction': 0.7,\n",
" 'bagging_freq': 5,\n",
" 'lambda_l1': 1,\n",
" 'lambda_l2': 1,\n",
" 'boosting': 'gbdt',\n",
" 'verbosity': -1,\n",
" 'extra_trees': True,\n",
" # 'max_position': 5,\n",
" # 'ndcg_at': 1,\n",
" 'quant_train_renew_leaf': True,\n",
" # 'lambdarank_truncation_level': 1,\n",
" # 'lambdarank_position_bias_regularization': 1,\n",
" 'seed': 7\n",
" }\n",
" train_groups = train_data_split.groupby('trade_date').size().tolist()\n",
" val_groups = val_data_split.groupby('trade_date').size().tolist()\n",
"\n",
" categorical_feature = [col for col in feature_columns if 'cat' in col]\n",
" train_dataset = lgb.Dataset(\n",
" X_train, label=y_train, \n",
" group=train_groups,\n",
" categorical_feature=categorical_feature\n",
" )\n",
" val_dataset = lgb.Dataset(\n",
" X_val, label=y_val, \n",
" group=val_groups,\n",
" categorical_feature=categorical_feature\n",
" )\n",
"\n",
" evals = {}\n",
" callbacks = [lgb.log_evaluation(period=1000),\n",
" lgb.callback.record_evaluation(evals),\n",
" lgb.early_stopping(100, first_metric_only=True)\n",
" ]\n",
" # 训练模型\n",
" model = lgb.train(\n",
" params, train_dataset, num_boost_round=1000,\n",
" valid_sets=[train_dataset, val_dataset], valid_names=['train', 'valid'],\n",
" callbacks=callbacks\n",
" )\n",
" from flaml import AutoML\n",
" from sklearn.datasets import fetch_california_housing\n",
"\n",
" # # Initialize an AutoML instance\n",
" # model = AutoML()\n",
" # # Specify automl goal and constraint\n",
" # automl_settings = {\n",
" # \"time_budget\": 3000, # in seconds\n",
" # \"metric\": \"r2\",\n",
" # \"task\": \"regression\",\n",
" # \"estimator_list\": [\n",
" # # \"catboost\",\n",
" # \"lgbm\",\n",
" # # \"xgboost\"\n",
" # ], \n",
" # # \"ensemble\": {\n",
" # # \"final_estimator\": LGBMRegressor(),\n",
" # # \"passthrough\": False,\n",
" # # },\n",
" # }\n",
" # model.fit(X_train=X_train, y_train=y_train, X_val=X_val, y_val=y_val, mlflow_logging=False, **automl_settings)\n",
"\n",
" # 打印特征重要性（如果需要）\n",
" if True:\n",
" lgb.plot_metric(evals)\n",
" lgb.plot_importance(model, importance_type='split', max_num_features=20)\n",
" plt.show()\n",
"\n",
"\n",
" return model, scaler, None # 返回训练好的模型、scaler 和 pca 对象"
]
},
{
"cell_type": "code",
"execution_count": 39,
"id": "c6eb5cd4-e714-420a-ac48-39af3e11ee81",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T15:03:18.426481Z",
"start_time": "2025-04-03T15:02:19.926352Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"train data size: 728000\n",
" ts_code trade_date log_circ_mv\n",
"0 600306.SH 2020-01-02 11.552040\n",
"1 603269.SH 2020-01-02 11.324801\n",
"2 002633.SZ 2020-01-02 11.759023\n",
"3 603991.SH 2020-01-02 11.181150\n",
"4 000691.SZ 2020-01-02 11.677910\n",
"... ... ... ...\n",
"727995 002235.SZ 2022-12-30 12.914708\n",
"727996 605598.SH 2022-12-30 11.783580\n",
"727997 002613.SZ 2022-12-30 12.489464\n",
"727998 600800.SH 2022-12-30 12.571911\n",
"727999 603068.SH 2022-12-30 12.967134\n",
"\n",
"[728000 rows x 3 columns]\n",
"原始样本数: 728000, 去除标签为空后样本数: 728000\n",
"Training until validation scores don't improve for 100 rounds\n",
"Early stopping, best iteration is:\n",
"[58]\ttrain's rmse: 0.0373704\tvalid's rmse: 0.040525\n",
"Evaluated only: rmse\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAHHCAYAAABEEKc/AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAYSpJREFUeJzt3XlcVPX+P/DX7MMwwLAjiqCCG+6ghnUzC8MsyxYt9Zpa2W2x8nL13mwxzW9Z3TRNLduz+9Ob125a11xCXNJyBdHcNVkU2bcZGAaGmfP7Y2BgDqCIyADzej4e8xjmc86c+bxHnHnxOZ9zjkQQBAFEREREZCd1dgeIiIiI2hoGJCIiIiIRBiQiIiIiEQYkIiIiIhEGJCIiIiIRBiQiIiIiEQYkIiIiIhEGJCIiIiIRBiQiIiIiEQYkImoxX3/9NSQSCdLS0m7aayxYsAASiaTdbNfZ0tLSIJFI8PXXXzfr+RKJBAsWLGjRPhG1BwxIRO1QTRCRSCTYt29fveWCICAkJAQSiQT33Xdfs17jo48+avaXKl2fdevWYdmyZc7uBhHVwYBE1I6p1WqsW7euXvuePXtw+fJlqFSqZm+7OQFp6tSpKC8vR2hoaLNf11lee+01lJeXO+W1b2ZACg0NRXl5OaZOndqs55eXl+O1115r4V4RtX0MSETt2NixY7FhwwZUVVU5tK9btw5RUVEICgpqlX6UlZUBAGQyGdRqdbvaVVXTd7lcDrVa7eTeXJvJZILVam3y+hKJBGq1GjKZrFmvp1arIZfLm/VcovaMAYmoHZs0aRIKCgqQkJBgb6usrMR3332HyZMnN/gcq9WKZcuWITIyEmq1GoGBgfjLX/6CoqIi+zphYWE4efIk9uzZY9+Vd8cddwCo3b23Z88ePPfccwgICECXLl0clonnIG3duhUjR46Eh4cHPD09MXTo0AZHvsT27duHoUOHQq1Wo0ePHvjkk0/qrXO1OTbi+TM184xOnTqFyZMnw9vbG7fddpvDMvHzZ82ahU2bNqFfv35QqVSIjIzEtm3b6r3W7t27ER0d7dDXpsxruuOOO/DTTz8hPT3d/l6HhYXZtymRSPDtt9/itddeQ+fOnaHRaKDX61FYWIg5c+agf//+0Gq18PT0xD333INjx45d8/2ZPn06tFotMjMzMX78eGi1Wvj7+2POnDmwWCxNeg8vXLiA6dOnQ6fTwcvLCzNmzIDRaHR4bnl5OV588UX4+fnBw8MD999/PzIzMzmvidoF/llA1I6FhYUhJiYG//73v3HPPfcAsIWRkpISPPbYY/jwww/rPecvf/kLvv76a8yYMQMvvvgiUlNTsXLlShw9ehS//vorFAoFli1bhhdeeAFarRavvvoqACAwMNBhO8899xz8/f0xf/58+yhMQ77++ms88cQTiIyMxLx586DT6XD06FFs27at0RAHAL///jvuvvtu+Pv7Y8GCBaiqqsIbb7xRrx/NMWHCBERERODtt9+GIAhXXXffvn34/vvv8dxzz8HDwwMffvghHn74YWRkZMDX1xcAcPToUYwZMwadOnXCwoULYbFY8Oabb8Lf3/+afXn11VdRUlKCy5cv44MPPgAAaLVah3UWLVoEpVKJOXPmoKKiAkqlEqdOncKmTZswYcIEdOvWDTk5Ofjkk08wcuRInDp1CsHBwVd9XYvFgri4OAwfPhzvv/8+duzYgSVLlqBHjx549tlnr9nviRMnolu3bli8eDGSk5Px+eefIyAgAO+++659nenTp+M///kPpk6diltuuQV79uzBvffee81tE7UJAhG1O1999ZUAQDh8+LCwcuVKwcPDQzAajYIgCMKECROEUaNGCYIgCKGhocK9995rf97evXsFAMLatWsdtrdt27Z67ZGRkcLIkSMbfe3bbrtNqKqqanBZamqqIAiCUFxcLHh4eAjDhw8XysvLHda1Wq1XrXH8+PGCWq0W0tPT7W2nTp0SZDKZUPejKzU1VQAgfPXVV/W2AUB444037I/feOMNAYAwadKkeuvWLBM/X6lUChcuXLC3HTt2TAAgrFixwt42btw4QaPRCJmZmfa28+fPC3K5vN42G3LvvfcKoaGh9dp37dolABC6d+9u//etYTKZBIvF4tCWmpoqqFQq4c0333RoE78/06ZNEwA4rCcIgjB48GAhKiqq3nvQ0Hv4xBNPOKz34IMPCr6+vvbHSUlJAgBh9uzZDutNnz693jaJ2iLuYiNq5yZOnIjy8nJs3rwZBoMBmzdvbnRkZsOGDfDy8sLo0aORn59vv0VFRUGr1WLXrl1Nft2ZM2dec15LQkICDAYDXn755Xrze66268lisWD79u0YP348unbtam/v06cP4uLimtzHxjzzzDNNXjc2NhY9evSwPx4wYAA8PT1x8eJFe1937NiB8ePHO4zahIeH20f1btS0adPg5ubm0KZSqSCVSu19KCgogFarRa9evZCcnNyk7Yrfhz/96U/2uprz3IKCAuj1egCw74Z87rnnHNZ74YUXmrR9ImfjLjaids7f3x+xsbFYt24djEYjLBYLHnnkkQbXPX/+PEpKShAQENDg8tzc3Ca/brdu3a65zh9//AEA6NevX5O3CwB5eXkoLy9HREREvWW9evXCli1brmt7Yk3pe426Aa2Gt7e3fc5Wbm4uysvLER4eXm+9htqao6H+Wq1WLF++HB999BFSU1Md5g7V7Pq7GrVaXW8XYN26rkX8vnh7ewMAioqK4OnpifT0dEil0np9b6n3hOhmY0Ai6gAmT56MmTNnIjs7G/fccw90Ol2D61mtVgQEBGDt2rUNLm/KnJka4hENZ2lsJEo82biu6+l7Y6NkwjXmLrWkhvr79ttv4/XXX8cTTzyBRYsWwcfHB1KpFLNnz27SUW7NPartWs9vzfeF6GZiQCLqAB588EH85S9/wYEDB7B+/fpG1+vRowd27NiBW2+99ZohoSUO1a/ZNXXixInrGjnw9/eHm5sbzp8/X2/Z2bNnHR7XjFwUFxc7tKenp19nb5snICAAarUaFy5cqLesobaGNOe9/u677zBq1Ch88cUXDu3FxcXw8/O77u21tNDQUFitVqSmpjqMBDb1PSFyNs5BIuoAtFotPv74YyxYsADjxo1rdL2JEyfCYrFg0aJF9ZZVVVU5hAx3d/d6oeN63X333fDw8MDixYthMpkcll1tpEEmkyEuLg6bNm1CRkaGvf306dPYvn27w7qenp7w8/PDL7/84tD+0Ucf3VDfm0omkyE2NhabNm3ClStX7O0XLlzA1q1bm7QNd3d3lJSUXPfrit/DDRs2IDMz87q2c7PUzBUT/zusWLHCGd0hum4cQSLqIKZNm3bNdUaOHIm//OUvWLx4MVJSUnD33XdDoVDg/Pnz2LBhA5YvX26fvxQVFYWPP/4Y//d//4fw8HAEBATgzjvvvK4+eXp64oMPPsBTTz2FoUOH2s89dOzYMRiNRqxZs6bR5y5cuBDbtm3Dn/70Jzz33HOoqqrCihUrEBkZiePHjzus+9RTT+Gdd97BU089hejoaPzyyy84d+7cdfX1RixYsAA///wzbr31Vjz77LOwWCxYuXIl+vXrh5SUlGs+PyoqCuvXr0d8fDyGDh0KrVZ71aALAPfddx/efPNNzJgxAyNGjMDvv/+OtWvXonv37i1U1Y2JiorCww8/jGXLlqGgoMB+mH/Nv0t7OpkouSYGJCIXs3r1akRFReGTTz7BK6+8ArlcjrCwMPz5z3/Grbfeal9v/vz5SE9Px3vvvQeDwYCRI0ded0ACgCeffBIBAQF45513sGjRIigUCvTu3Rt//etfr/q8AQMGYPv27YiPj8f8+fPRpUsXLFy4EFlZWfUC0vz585GXl4fvvvsO//nPf3DPPfdg69atjU5Gb2lRUVHYunUr5syZg9dffx0hISF48803cfr0aZw5c+aaz3/uueeQkpKCr776Ch988AFCQ0OvGZBeeeUVlJWVYd26dVi/fj2GDBmCn376CS+//HJLlXXDvvnmGwQFBeHf//43Nm7ciNjYWKxfvx69evVqF2ctJ9cmETijjojophg/fjxOnjz
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAygAAAHHCAYAAABUeXyPAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3XlcTun/+PHX3Z42MtlLlpAlNRgiZEn2dWTwQdYZNCNbJoWy7/s2jKmxJIxtZoSJscwIg5nssoxkZpgwSBlpOb8//Dpft4pK6Wbez8fjfnSf61znOtd1ruq+r3MtR6MoioIQQgghhBBC6AC9ws6AEEIIIYQQQmSQBooQQgghhBBCZ0gDRQghhBBCCKEzpIEihBBCCCGE0BnSQBFCCCGEEELoDGmgCCGEEEIIIXSGNFCEEEIIIYQQOkMaKEIIIYQQQgidIQ0UIYQQQgghhM6QBooQQgghCkxoaCgajYbY2NjCzooQ4i0hDRQhhBAiH2V8Ic/q9fnnnxfIOaOioggKCuLBgwcFkv5/2ePHjwkKCuLgwYOFnRUh/jMMCjsDQgghxLto8uTJVKhQQSusZs2aBXKuqKgogoOD8fb2pmjRogVyjrzq06cPH330EcbGxoWdlTx5/PgxwcHBALi7uxduZoT4j5AGihBCCFEA2rRpQ926dQs7G68lKSkJMzOz10pDX18ffX39fMrRm5Oens7Tp08LOxtC/CfJEC8hhBCiEOzevZvGjRtjZmaGhYUF7dq14/z581pxzpw5g7e3NxUrVsTExIRSpUoxYMAA7t27p8YJCgpi7NixAFSoUEEdThYbG0tsbCwajYbQ0NBM59doNAQFBWmlo9FouHDhAr169aJYsWK4ubmp+9evX0+dOnUwNTXF2tqajz76iJs3b76ynFnNQbG3t6d9+/YcPHiQunXrYmpqSq1atdRhVNu2baNWrVqYmJhQp04dfvvtN600vb29MTc35/fff8fT0xMzMzPKlCnD5MmTURRFK25SUhKjR4/G1tYWY2Njqlatyty5czPF02g0+Pj4sGHDBmrUqIGxsTErV67ExsYGgODgYPXaZly3nNTP89f26tWrai+XlZUV/fv35/Hjx5mu2fr16/nggw8oUqQIxYoVo0mTJvzwww9acXLy+yPE20p6UIQQQogC8PDhQ+7evasV9t577wGwbt06+vXrh6enJ7NmzeLx48esWLECNzc3fvvtN+zt7QGIjIzk999/p3///pQqVYrz58+zatUqzp8/z7Fjx9BoNHTt2pXLly+zceNGFixYoJ7DxsaGO3fu5Drf3bt3x8HBgenTp6tf4qdNm8aECRPw8vJi0KBB3LlzhyVLltCkSRN+++23PA0ru3r1Kr169eLjjz/mf//7H3PnzqVDhw6sXLmS8ePHM2zYMABmzJiBl5cXMTEx6On9333VtLQ0WrduTYMGDZg9ezZ79uxh0qRJpKamMnnyZAAURaFjx44cOHCAgQMH4uzszN69exk7dix//vknCxYs0MrTjz/+yObNm/Hx8eG9996jdu3arFixgqFDh9KlSxe6du0KgJOTE5Cz+nmel5cXFSpUYMaMGfz66698+eWXlChRglmzZqlxgoODCQoKomHDhkyePBkjIyOOHz/Ojz/+SKtWrYCc//4I8dZShBBCCJFvQkJCFCDLl6IoyqNHj5SiRYsqgwcP1jru9u3bipWVlVb448ePM6W/ceNGBVAOHz6shs2ZM0cBlOvXr2vFvX79ugIoISEhmdIBlEmTJqnbkyZNUgClZ8+eWvFiY2MVfX19Zdq0aVrhZ8+eVQwMDDKFZ3c9ns9b+fLlFUCJiopSw/bu3asAiqmpqXLjxg01/IsvvlAA5cCBA2pYv379FED59NNP1bD09HSlXbt2ipGRkXLnzh1FURRlx44dCqBMnTpVK08ffvihotFolKtXr2pdDz09PeX8+fNace/cuZPpWmXIaf1kXNsBAwZoxe3SpYtSvHhxdfvKlSuKnp6e0qVLFyUtLU0rbnp6uqIoufv9EeJtJUO8hBBCiAKwbNkyIiMjtV7w7K77gwcP6NmzJ3fv3lVf+vr61K9fnwMHDqhpmJqaqu+fPHnC3bt3adCgAQC//vprgeT7k08+0dretm0b6enpeHl5aeW3VKlSODg4aOU3N6pXr46rq6u6Xb9+fQCaN2+OnZ1dpvDff/89Uxo+Pj7q+4whWk+fPmXfvn0AREREoK+vz2effaZ13OjRo1EUhd27d2uFN23alOrVq+e4DLmtnxevbePGjbl37x4JCQkA7Nixg/T0dCZOnKjVW5RRPsjd748QbysZ4iWEEEIUgA8++CDLSfJXrlwBnn0Rz4qlpaX6/p9//iE4OJjw8HDi4+O14j18+DAfc/t/Xlx57MqVKyiKgoODQ5bxDQ0N83Se5xshAFZWVgDY2tpmGX7//n2tcD09PSpWrKgVVqVKFQB1vsuNGzcoU6YMFhYWWvEcHR3V/c97seyvktv6ebHMxYoVA56VzdLSkmvXrqGnp/fSRlJufn+EeFtJA0UIIYR4g9LT04Fn8whKlSqVab+Bwf99NHt5eREVFcXYsWNxdnbG3Nyc9PR0WrdurabzMi/OgciQlpaW7THP9wpk5Fej0bB79+4sV+MyNzd/ZT6ykt3KXtmFKy9Mai8IL5b9VXJbP/lRttz8/gjxtpLfYiGEEOINqlSpEgAlSpSgZcuW2ca7f/8++/fvJzg4mIkTJ6rhGXfQn5ddQyTjDv2LD3B8sefgVflVFIUKFSqoPRS6ID09nd9//10rT5cvXwZQJ4mXL1+effv28ejRI61elEuXLqn7XyW7a5ub+smpSpUqkZ6ezoULF3B2ds42Drz690eIt5nMQRFCCCHeIE9PTywtLZk+fTopKSmZ9mesvJVxt/3Fu+sLFy7MdEzGs0pebIhYWlry3nvvcfjwYa3w5cuX5zi/Xbt2RV9fn+Dg4Ex5URQl05K6b9LSpUu18rJ06VIMDQ1p0aIFAG3btiUtLU0rHsCCBQvQaDS0adPmlecoUqQIkPna5qZ+cqpz587o6ekxefLkTD0wGefJ6e+PEG8z6UERQggh3iBLS0tWrFhBnz59eP/99/noo4+wsbEhLi6OXbt20ahRI5YuXYqlpSVNmjRh9uzZpKSkULZsWX744QeuX7+eKc06deoAEBAQwEcffYShoSEdOnTAzMyMQYMGMXPmTAYNGkTdunU5fPiw2tOQE5UqVWLq1Kn4+/sTGxtL586dsbCw4Pr162zfvp0hQ4YwZsyYfLs+OWViYsKePXvo168f9evXZ/fu3ezatYvx48erzy7p0KEDzZo1IyAggNjYWGrXrs0PP/zAzp078fX1VXsjXsbU1JTq1auzadMmqlSpgrW1NTVr1qRmzZo5rp+cqly5MgEBAUyZMoXGjRvTtWtXjI2NOXHiBGXKlGHGjBk5/v0R4q1WSKuHCSGEEO+kjGV1T5w48dJ4Bw4cUDw9PRUrKyvFxMREqVSpkuLt7a2cPHlSjfPHH38oXbp0UYoWLapYWVkp3bt3V/76668sl72dMmWKUrZsWUVPT09rWd/Hjx8rAwcOVKysrBQLCwvFy8tLiY+Pz3aZ4Ywlel+0detWxc3NTTEzM1PMzMyUatWqKcOHD1diYmJydD1eXGa4Xbt2meICyvDhw7XCMpZKnjNnjhrWr18/xczMTLl27ZrSqlUrpUiRIkrJkiWVSZMmZVqe99GjR8rIkSOVMmXKKIaGhoqDg4MyZ84cddnel507Q1RUlFKnTh3FyMhI67rltH6yu7ZZXRtFUZSvvvpKcXFxUYyNjZVixYopTZs2VSIjI7Xi5OT3R4i3lUZR3sCsMyGEEEKIfOLt7c0333xDYmJiYWdFCFEAZA6KEEIIIYQQQmdIA0UIIYQQQgihM6SBIoQQQgghhNAZMgdFCCGEEEIIoTOkB0UIIYQQQgihM6SBIoQQQgghhNAZ8qBGIcRbJT09nb/++gsLCws0Gk1hZ0cIIYQQOaAoCo8ePaJMmTLo6b28j0QaKEKIt8pff/2Fra1tYWdDCCGEEHlw8+ZNypUr99I40kARQrxVLCw
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"\n",
"gc.collect()\n",
"\n",
"use_pca = False\n",
"type = 'light'\n",
"# feature_contri = [2 if feat.startswith('act_factor') or 'buy' in feat or 'sell' in feat else 1 for feat in feature_columns]\n",
"# light_params['feature_contri'] = feature_contri\n",
"# print(f'feature_contri: {feature_contri}')\n",
"model, scaler, pca = train_model(train_data\n",
" .dropna(subset=['label']).groupby('trade_date', group_keys=False)\n",
" .apply(lambda x: x.nsmallest(1000, 'total_mv'))\n",
" .merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
" .merge(index_data, on='trade_date', how='left'), \n",
" feature_columns, type=type, target_column='future_return_2')\n"
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "5d1522a7538db91b",
"metadata": {
"ExecuteTime": {
"end_time": "2025-04-03T15:04:39.656944Z",
"start_time": "2025-04-03T15:04:39.298483Z"
}
},
"outputs": [],
"source": [
"score_df = test_data.groupby('trade_date', group_keys=False).apply(lambda x: x.nsmallest(1000, 'total_mv'))\n",
"# score_df = fill_nan_with_daily_median(score_df, ['pe_ttm'])\n",
"# score_df = score_df[score_df['pe_ttm'] > 0]\n",
"score_df = score_df.merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
"score_df = score_df.merge(index_data, on='trade_date', how='left')\n",
"# score_df = score_df.groupby('trade_date', group_keys=False).apply(lambda x: x.nsmallest(50, 'total_mv')).reset_index()\n",
"numeric_columns = score_df.select_dtypes(include=['float64', 'int64']).columns\n",
"numeric_columns = [col for col in feature_columns if col in numeric_columns]\n",
"\n",
"import pandas as pd\n",
"\n",
"def filter_high_volatility_daily(df: pd.DataFrame, threshold: float, window: int = 5,\n",
" date_col: str = 'trade_date', code_col: str = 'ts_code',\n",
" close_col: str = 'close') -> pd.DataFrame:\n",
" df = df.sort_values([code_col, date_col])\n",
"\n",
" def get_daily_std_rank(group):\n",
" rolling_std = group[close_col].rolling(window=window, min_periods=window).std()\n",
" # 在每个日期内计算标准差的百分比排名\n",
" return rolling_std.groupby(group[date_col]).rank(pct=True, ascending=False)\n",
"\n",
" # 直接在 apply 中计算并添加 'std_rank' 列\n",
" df_with_std = df.groupby(code_col, group_keys=False).apply(\n",
" lambda x: x.assign(std_rank=get_daily_std_rank(x))\n",
" )\n",
"\n",
" # 过滤掉每天标准差排名在前 top_percent 的股票\n",
" threshold_rank = threshold / 100.0\n",
" df_filtered = df_with_std[df_with_std['std_rank'] > threshold_rank].drop(\n",
" columns=['std_rank']\n",
" )\n",
"\n",
" return df_filtered\n",
"\n",
"# threshold = 10\n",
"# score_df = filter_high_volatility_daily(score_df, threshold=threshold)\n",
"# print(len(score_df))\n",
"\n",
"if type == 'cat':\n",
" score_df['score'] = model.predict(score_df[feature_columns])\n",
"elif type == 'light':\n",
" score_df['score'] = model.predict(score_df[feature_columns])\n",
"score_df['score_ranks'] = score_df.groupby('trade_date')['score'].rank(ascending=True)\n",
"\n",
"score_df = score_df.groupby('trade_date', group_keys=False).apply(\n",
" lambda x: \n",
" x[\n",
" # (x['score'] <= x['score'].quantile(0.99)) & \n",
" (x['score'] >= x['score'].quantile(0.90))\n",
" ] # 计算90%分位数作为阈值，筛选分数>=阈值的行\n",
").reset_index(drop=True) # drop=True 避免添加旧索引列\n",
"# df_to_drop = score_df.loc[score_df.groupby('trade_date')['score'].idxmax()]\n",
"# score_df = score_df.drop(df_to_drop.index)\n",
"save_df = score_df.groupby('trade_date', group_keys=False).apply(lambda x: x.nlargest(2, 'score')).reset_index()\n",
"# save_df = score_df.groupby('trade_date', group_keys=False).apply(lambda x: x.nsmallest(2, 'total_mv')).reset_index()\n",
"save_df = save_df.sort_values(['trade_date', 'score'])\n",
"save_df[['trade_date', 'score', 'ts_code']].to_csv('predictions_test.tsv', index=False)\n"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "09b1799e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"162\n",
"['vol', 'pct_chg', 'turnover_rate', 'volume_ratio', 'winner_rate', 'undist_profit_ps', 'ocfps', 'AR', 'BR', 'AR_BR', 'log_circ_mv', 'cashflow_to_ev_factor', 'book_to_price_ratio', 'turnover_rate_mean_5', 'variance_20', 'bbi_ratio_factor', 'daily_deviation', 'lg_elg_net_buy_vol', 'flow_lg_elg_intensity', 'sm_net_buy_vol', 'total_buy_vol', 'lg_elg_buy_prop', 'flow_struct_buy_change', 'lg_elg_net_buy_vol_change', 'flow_lg_elg_accel', 'chip_concentration_range', 'chip_skewness', 'floating_chip_proxy', 'cost_support_15pct_change', 'cat_winner_price_zone', 'flow_chip_consistency', 'profit_taking_vs_absorb', 'cat_is_positive', 'upside_vol', 'downside_vol', 'vol_ratio', 'return_skew', 'return_kurtosis', 'volume_change_rate', 'cat_volume_breakout', 'turnover_deviation', 'cat_turnover_spike', 'avg_volume_ratio', 'cat_volume_ratio_breakout', 'vol_spike', 'vol_std_5', 'atr_14', 'atr_6', 'obv', 'maobv_6', 'rsi_3', 'return_5', 'return_20', 'std_return_5', 'std_return_90', 'std_return_90_2', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4', 'rank_act_factor1', 'rank_act_factor2', 'rank_act_factor3', 'cov', 'delta_cov', 'alpha_22_improved', 'alpha_003', 'alpha_007', 'alpha_013', 'vol_break', 'weight_roc5', 'smallcap_concentration', 'cost_stability', 'high_cost_break_days', 'liquidity_risk', 'turnover_std', 'mv_volatility', 'volume_growth', 'mv_growth', 'momentum_factor', 'resonance_factor', 'log_close', 'cat_vol_spike', 'up', 'down', 'obv_maobv_6', 'std_return_5_over_std_return_90', 'std_return_90_minus_std_return_90_2', 'cat_af2', 'cat_af3', 'cat_af4', 'act_factor5', 'act_factor6', 'active_buy_volume_large', 'active_buy_volume_big', 'active_buy_volume_small', 'buy_lg_vol_minus_sell_lg_vol', 'buy_elg_vol_minus_sell_elg_vol', 'ctrl_strength', 'low_cost_dev', 'asymmetry', 'lock_factor', 'cat_vol_break', 'cost_atr_adj', 'cat_golden_resonance', 'mv_turnover_ratio', 'mv_adjusted_volume', 'mv_weighted_turnover', 'nonlinear_mv_volume', 'mv_volume_ratio', 'mv_momentum', 'lg_flow_mom_corr_20_60', 'lg_flow_accel', 'profit_pressure', 'underwater_resistance', 'cost_conc_std_20', 'profit_decay_20', 'vol_amp_loss_20', 'vol_drop_profit_cnt_5', 'lg_flow_vol_interact_20', 'cost_break_confirm_cnt_5', 'atr_norm_channel_pos_14', 'turnover_diff_skew_20', 'lg_sm_flow_diverge_20', 'pullback_strong_20_20', 'vol_wgt_hist_pos_20', 'vol_adj_roc_20', 'cs_rank_net_lg_flow_val', 'cs_rank_elg_buy_ratio', 'cs_rank_rel_profit_margin', 'cs_rank_cost_breadth', 'cs_rank_dist_to_upper_cost', 'cs_rank_winner_rate', 'cs_rank_intraday_range', 'cs_rank_close_pos_in_range', 'cs_rank_pos_in_hist_range', 'cs_rank_vol_x_profit_margin', 'cs_rank_lg_flow_price_concordance', 'cs_rank_turnover_per_winner', 'cs_rank_volume_ratio', 'cs_rank_elg_buy_sell_sm_ratio', 'cs_rank_cost_dist_vol_ratio', 'cs_rank_size', 'cat_up_limit', 'industry_obv', 'industry_return_5', 'industry_return_20', 'industry__ema_5', 'industry__ema_13', 'industry__ema_20', 'industry__ema_60', 'industry_act_factor1', 'industry_act_factor2', 'industry_act_factor3', 'industry_act_factor4', 'industry_act_factor5', 'industry_act_factor6', 'industry_rank_act_factor1', 'industry_rank_act_factor2', 'industry_rank_act_factor3', 'industry_return_5_percentile', 'industry_return_20_percentile']\n"
]
}
],
"source": [
"print(len(feature_columns))\n",
"print(feature_columns)"
]
},
{
"cell_type": "code",
"execution_count": 42,
"id": "bceabd1f",
"metadata": {},
"outputs": [],
"source": [
"# print(model.get_feature_importance())"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "new_trader",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue Copy Permalink