2025-02-23 23:47:19 +08:00
|
|
|
|
{
|
|
|
|
|
|
"cells": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "79a7758178bafdd3",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:15:11.540909Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:15:11.424065Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"%load_ext autoreload\n",
|
|
|
|
|
|
"%autoreload 2\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"import pandas as pd\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" \n",
|
|
|
|
|
|
"pd.set_option('display.max_columns', None)\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
|
|
|
|
|
"The autoreload extension is already loaded. To reload it, use:\n",
|
|
|
|
|
|
" %reload_ext autoreload\n"
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 81
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "a79cafb06a7e0e43",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:15:58.383324Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:15:11.540909Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
2025-04-28 11:02:52 +08:00
|
|
|
|
"from code.utils.utils import read_and_merge_h5_data\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
"\n",
|
|
|
|
|
|
"print('daily data')\n",
|
|
|
|
|
|
"df = read_and_merge_h5_data('../../data/daily_data.h5', key='daily_data',\n",
|
|
|
|
|
|
" columns=['ts_code', 'trade_date', 'open', 'close', 'high', 'low', 'vol'],\n",
|
|
|
|
|
|
" df=None)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"print('daily basic')\n",
|
|
|
|
|
|
"df = read_and_merge_h5_data('../../data/daily_basic.h5', key='daily_basic',\n",
|
|
|
|
|
|
" columns=['ts_code', 'trade_date', 'turnover_rate', 'pe_ttm', 'circ_mv', 'volume_ratio',\n",
|
|
|
|
|
|
" 'is_st'], df=df, join='inner')\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"print('stk limit')\n",
|
|
|
|
|
|
"df = read_and_merge_h5_data('../../data/stk_limit.h5', key='stk_limit',\n",
|
|
|
|
|
|
" columns=['ts_code', 'trade_date', 'pre_close', 'up_limit', 'down_limit'],\n",
|
|
|
|
|
|
" df=df)\n",
|
|
|
|
|
|
"print('money flow')\n",
|
|
|
|
|
|
"df = read_and_merge_h5_data('../../data/money_flow.h5', key='money_flow',\n",
|
|
|
|
|
|
" columns=['ts_code', 'trade_date', 'buy_sm_vol', 'sell_sm_vol', 'buy_lg_vol', 'sell_lg_vol',\n",
|
|
|
|
|
|
" 'buy_elg_vol', 'sell_elg_vol', 'net_mf_vol'],\n",
|
|
|
|
|
|
" df=df)\n",
|
|
|
|
|
|
"print(df.info())"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
|
|
|
|
|
"daily data\n",
|
|
|
|
|
|
"daily basic\n",
|
|
|
|
|
|
"inner merge on ['ts_code', 'trade_date']\n",
|
|
|
|
|
|
"stk limit\n",
|
|
|
|
|
|
"left merge on ['ts_code', 'trade_date']\n",
|
|
|
|
|
|
"money flow\n",
|
|
|
|
|
|
"left merge on ['ts_code', 'trade_date']\n",
|
|
|
|
|
|
"<class 'pandas.core.frame.DataFrame'>\n",
|
|
|
|
|
|
"RangeIndex: 8296325 entries, 0 to 8296324\n",
|
|
|
|
|
|
"Data columns (total 21 columns):\n",
|
|
|
|
|
|
" # Column Dtype \n",
|
|
|
|
|
|
"--- ------ ----- \n",
|
|
|
|
|
|
" 0 ts_code object \n",
|
|
|
|
|
|
" 1 trade_date datetime64[ns]\n",
|
|
|
|
|
|
" 2 open float64 \n",
|
|
|
|
|
|
" 3 close float64 \n",
|
|
|
|
|
|
" 4 high float64 \n",
|
|
|
|
|
|
" 5 low float64 \n",
|
|
|
|
|
|
" 6 vol float64 \n",
|
|
|
|
|
|
" 7 turnover_rate float64 \n",
|
|
|
|
|
|
" 8 pe_ttm float64 \n",
|
|
|
|
|
|
" 9 circ_mv float64 \n",
|
|
|
|
|
|
" 10 volume_ratio float64 \n",
|
|
|
|
|
|
" 11 is_st bool \n",
|
|
|
|
|
|
" 12 up_limit float64 \n",
|
|
|
|
|
|
" 13 down_limit float64 \n",
|
|
|
|
|
|
" 14 buy_sm_vol float64 \n",
|
|
|
|
|
|
" 15 sell_sm_vol float64 \n",
|
|
|
|
|
|
" 16 buy_lg_vol float64 \n",
|
|
|
|
|
|
" 17 sell_lg_vol float64 \n",
|
|
|
|
|
|
" 18 buy_elg_vol float64 \n",
|
|
|
|
|
|
" 19 sell_elg_vol float64 \n",
|
|
|
|
|
|
" 20 net_mf_vol float64 \n",
|
|
|
|
|
|
"dtypes: bool(1), datetime64[ns](1), float64(18), object(1)\n",
|
|
|
|
|
|
"memory usage: 1.2+ GB\n",
|
|
|
|
|
|
"None\n"
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 82
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "f7a55c19-b7dc-4d2f-a478-cffab11690df",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:16:00.292129Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:15:58.487459Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"print('industry')\n",
|
|
|
|
|
|
"df = read_and_merge_h5_data('../../data/industry_data.h5', key='industry_data',\n",
|
|
|
|
|
|
" columns=['ts_code', 'l2_code'],\n",
|
|
|
|
|
|
" df=df, on=['ts_code'], join='left')\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
|
|
|
|
|
"industry\n",
|
|
|
|
|
|
"left merge on ['ts_code']\n"
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 83
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "4077d4449d406c86",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:16:00.546130Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:16:00.386519Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"import pandas as pd\n",
|
|
|
|
|
|
"import numpy as np\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def calculate_indicators(df):\n",
|
|
|
|
|
|
" \"\"\"\n",
|
|
|
|
|
|
" 计算四个指标：当日涨跌幅、5日移动平均、RSI、MACD。\n",
|
|
|
|
|
|
" \"\"\"\n",
|
|
|
|
|
|
" df['daily_return'] = (df['close'] - df['pre_close']) / df['pre_close'] * 100\n",
|
|
|
|
|
|
" # df['5_day_ma'] = df['close'].rolling(window=5).mean()\n",
|
|
|
|
|
|
" delta = df['close'].diff()\n",
|
|
|
|
|
|
" gain = delta.where(delta > 0, 0)\n",
|
|
|
|
|
|
" loss = -delta.where(delta < 0, 0)\n",
|
|
|
|
|
|
" avg_gain = gain.rolling(window=14).mean()\n",
|
|
|
|
|
|
" avg_loss = loss.rolling(window=14).mean()\n",
|
|
|
|
|
|
" rs = avg_gain / avg_loss\n",
|
|
|
|
|
|
" df['RSI'] = 100 - (100 / (1 + rs))\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算MACD\n",
|
|
|
|
|
|
" ema12 = df['close'].ewm(span=12, adjust=False).mean()\n",
|
|
|
|
|
|
" ema26 = df['close'].ewm(span=26, adjust=False).mean()\n",
|
|
|
|
|
|
" df['MACD'] = ema12 - ema26\n",
|
|
|
|
|
|
" df['Signal_line'] = df['MACD'].ewm(span=9, adjust=False).mean()\n",
|
|
|
|
|
|
" df['MACD_hist'] = df['MACD'] - df['Signal_line']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" return df\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def generate_index_indicators(h5_filename):\n",
|
|
|
|
|
|
" df = pd.read_hdf(h5_filename, key='index_data')\n",
|
|
|
|
|
|
" df['trade_date'] = pd.to_datetime(df['trade_date'], format='%Y%m%d')\n",
|
|
|
|
|
|
" df = df.sort_values('trade_date')\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算每个ts_code的相关指标\n",
|
|
|
|
|
|
" df_indicators = []\n",
|
|
|
|
|
|
" for ts_code in df['ts_code'].unique():\n",
|
|
|
|
|
|
" df_index = df[df['ts_code'] == ts_code].copy()\n",
|
|
|
|
|
|
" df_index = calculate_indicators(df_index)\n",
|
|
|
|
|
|
" df_indicators.append(df_index)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 合并所有指数的结果\n",
|
|
|
|
|
|
" df_all_indicators = pd.concat(df_indicators, ignore_index=True)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 保留trade_date列，并将同一天的数据按ts_code合并成一行\n",
|
|
|
|
|
|
" df_final = df_all_indicators.pivot_table(\n",
|
|
|
|
|
|
" index='trade_date',\n",
|
|
|
|
|
|
" columns='ts_code',\n",
|
|
|
|
|
|
" values=['daily_return', 'RSI', 'MACD', 'Signal_line', 'MACD_hist'],\n",
|
|
|
|
|
|
" aggfunc='last'\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" df_final.columns = [f\"{col[1]}_{col[0]}\" for col in df_final.columns]\n",
|
|
|
|
|
|
" df_final = df_final.reset_index()\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" return df_final\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# 使用函数\n",
|
|
|
|
|
|
"h5_filename = '../../data/index_data.h5'\n",
|
|
|
|
|
|
"index_data = generate_index_indicators(h5_filename)\n",
|
|
|
|
|
|
"index_data = index_data.dropna()\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 84
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "c4e9e1d31da6dba6",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:16:00.737530Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:16:00.642938Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"import numpy as np\n",
|
|
|
|
|
|
"import talib\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def get_technical_factor(df):\n",
|
|
|
|
|
|
" # 按股票和日期排序\n",
|
|
|
|
|
|
" df = df.sort_values(by=['ts_code', 'trade_date'])\n",
|
|
|
|
|
|
" grouped = df.groupby('ts_code', group_keys=False)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算 up 和 down\n",
|
|
|
|
|
|
" df['log_close'] = np.log(df['close'])\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" df['up'] = (df['high'] - df[['close', 'open']].max(axis=1)) / df['close']\n",
|
|
|
|
|
|
" df['down'] = (df[['close', 'open']].min(axis=1) - df['low']) / df['close']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算 ATR\n",
|
|
|
|
|
|
" df['atr_14'] = grouped.apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.ATR(x['high'].values, x['low'].values, x['close'].values, timeperiod=14), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['atr_6'] = grouped.apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.ATR(x['high'].values, x['low'].values, x['close'].values, timeperiod=6), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算 OBV 及其均线\n",
|
|
|
|
|
|
" df['obv'] = grouped.apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.OBV(x['close'].values, x['vol'].values), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['maobv_6'] = grouped.apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.SMA(x['obv'].values, timeperiod=6), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['obv-maobv_6'] = df['obv'] - df['maobv_6']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算 RSI\n",
|
|
|
|
|
|
" df['rsi_3'] = grouped.apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.RSI(x['close'].values, timeperiod=3), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['rsi_6'] = grouped.apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.RSI(x['close'].values, timeperiod=6), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['rsi_9'] = grouped.apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.RSI(x['close'].values, timeperiod=9), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算 return_10 和 return_20\n",
|
|
|
|
|
|
" df['return_5'] = grouped['close'].apply(lambda x: x / x.shift(5) - 1)\n",
|
|
|
|
|
|
" df['return_10'] = grouped['close'].apply(lambda x: x / x.shift(10) - 1)\n",
|
|
|
|
|
|
" df['return_20'] = grouped['close'].apply(lambda x: x / x.shift(20) - 1)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算 avg_close_5\n",
|
|
|
|
|
|
" df['avg_close_5'] = grouped['close'].apply(lambda x: x.rolling(window=5).mean() / x)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算标准差指标\n",
|
|
|
|
|
|
" df['std_return_5'] = grouped['close'].apply(lambda x: x.pct_change().rolling(window=5).std())\n",
|
|
|
|
|
|
" df['std_return_15'] = grouped['close'].apply(lambda x: x.pct_change().rolling(window=15).std())\n",
|
|
|
|
|
|
" df['std_return_25'] = grouped['close'].apply(lambda x: x.pct_change().rolling(window=25).std())\n",
|
|
|
|
|
|
" df['std_return_90'] = grouped['close'].apply(lambda x: x.pct_change().rolling(window=90).std())\n",
|
|
|
|
|
|
" df['std_return_90_2'] = grouped['close'].apply(lambda x: x.shift(10).pct_change().rolling(window=90).std())\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算比值指标\n",
|
|
|
|
|
|
" df['std_return_5 / std_return_90'] = df['std_return_5'] / df['std_return_90']\n",
|
|
|
|
|
|
" df['std_return_5 / std_return_25'] = df['std_return_5'] / df['std_return_25']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算标准差差值\n",
|
|
|
|
|
|
" df['std_return_90 - std_return_90_2'] = df['std_return_90'] - df['std_return_90_2']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" return df\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def get_act_factor(df, cat=True):\n",
|
|
|
|
|
|
" # 按股票和日期排序\n",
|
|
|
|
|
|
" df = df.sort_values(by=['ts_code', 'trade_date'])\n",
|
|
|
|
|
|
" grouped = df.groupby('ts_code', group_keys=False)\n",
|
|
|
|
|
|
" # 计算 EMA 指标\n",
|
|
|
|
|
|
" df['ema_5'] = grouped['close'].apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.EMA(x.values, timeperiod=5), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['ema_13'] = grouped['close'].apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.EMA(x.values, timeperiod=13), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['ema_20'] = grouped['close'].apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.EMA(x.values, timeperiod=20), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['ema_60'] = grouped['close'].apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.EMA(x.values, timeperiod=60), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算 act_factor1, act_factor2, act_factor3, act_factor4\n",
|
|
|
|
|
|
" df['act_factor1'] = grouped['ema_5'].apply(\n",
|
|
|
|
|
|
" lambda x: np.arctan((x / x.shift(1) - 1) * 100) * 57.3 / 50\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['act_factor2'] = grouped['ema_13'].apply(\n",
|
|
|
|
|
|
" lambda x: np.arctan((x / x.shift(1) - 1) * 100) * 57.3 / 40\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['act_factor3'] = grouped['ema_20'].apply(\n",
|
|
|
|
|
|
" lambda x: np.arctan((x / x.shift(1) - 1) * 100) * 57.3 / 21\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" df['act_factor4'] = grouped['ema_60'].apply(\n",
|
|
|
|
|
|
" lambda x: np.arctan((x / x.shift(1) - 1) * 100) * 57.3 / 10\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" if cat:\n",
|
|
|
|
|
|
" df['cat_af1'] = df['act_factor1'] > 0\n",
|
|
|
|
|
|
" df['cat_af2'] = df['act_factor2'] > df['act_factor1']\n",
|
|
|
|
|
|
" df['cat_af3'] = df['act_factor3'] > df['act_factor2']\n",
|
|
|
|
|
|
" df['cat_af4'] = df['act_factor4'] > df['act_factor3']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算 act_factor5 和 act_factor6\n",
|
|
|
|
|
|
" df['act_factor5'] = df['act_factor1'] + df['act_factor2'] + df['act_factor3'] + df['act_factor4']\n",
|
|
|
|
|
|
" df['act_factor6'] = (df['act_factor1'] - df['act_factor2']) / np.sqrt(df['act_factor1']**2 + df['act_factor2']**2)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 根据 trade_date 截面计算排名\n",
|
|
|
|
|
|
" df['rank_act_factor1'] = df.groupby('trade_date', group_keys=False)['act_factor1'].rank(ascending=False, pct=True)\n",
|
|
|
|
|
|
" df['rank_act_factor2'] = df.groupby('trade_date', group_keys=False)['act_factor2'].rank(ascending=False, pct=True)\n",
|
|
|
|
|
|
" df['rank_act_factor3'] = df.groupby('trade_date', group_keys=False)['act_factor3'].rank(ascending=False, pct=True)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" return df\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def get_money_flow_factor(df):\n",
|
|
|
|
|
|
" # 计算资金流相关因子（字段名称见 tushare 数据说明）\n",
|
|
|
|
|
|
" df['active_buy_volume_large'] = df['buy_lg_vol'] / df['net_mf_vol']\n",
|
|
|
|
|
|
" df['active_buy_volume_big'] = df['buy_elg_vol'] / df['net_mf_vol']\n",
|
|
|
|
|
|
" df['active_buy_volume_small'] = df['buy_sm_vol'] / df['net_mf_vol']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" df['buy_lg_vol_minus_sell_lg_vol'] = (df['buy_lg_vol'] - df['sell_lg_vol']) / df['net_mf_vol']\n",
|
|
|
|
|
|
" df['buy_elg_vol_minus_sell_elg_vol'] = (df['buy_elg_vol'] - df['sell_elg_vol']) / df['net_mf_vol']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" df['log(circ_mv)'] = np.log(df['circ_mv'])\n",
|
|
|
|
|
|
" return df\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def get_alpha_factor(df):\n",
|
|
|
|
|
|
" df = df.sort_values(by=['ts_code', 'trade_date'])\n",
|
|
|
|
|
|
" grouped = df.groupby('ts_code')\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # alpha_022: 当前 close 与 5 日前 close 差值\n",
|
|
|
|
|
|
" df['alpha_022'] = grouped['close'].transform(lambda x: x - x.shift(5))\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # alpha_003: (close - open) / (high - low)\n",
|
|
|
|
|
|
" df['alpha_003'] = np.where(df['high'] != df['low'],\n",
|
|
|
|
|
|
" (df['close'] - df['open']) / (df['high'] - df['low']),\n",
|
|
|
|
|
|
" 0)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # alpha_007: 计算过去5日 close 与 vol 的相关性，并按 trade_date 排名\n",
|
|
|
|
|
|
" df['alpha_007'] = grouped.apply(lambda x: x['close'].rolling(5).corr(x['vol'])).reset_index(level=0, drop=True)\n",
|
|
|
|
|
|
" df['alpha_007'] = df.groupby('trade_date', group_keys=False)['alpha_007'].rank(ascending=True, pct=True)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # alpha_013: 计算过去5日 close 之和 - 20日 close 之和，并按 trade_date 排名\n",
|
|
|
|
|
|
" df['alpha_013'] = grouped['close'].transform(lambda x: x.rolling(5).sum() - x.rolling(20).sum())\n",
|
|
|
|
|
|
" df['alpha_013'] = df.groupby('trade_date', group_keys=False)['alpha_013'].rank(ascending=True, pct=True)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" return df\n",
|
|
|
|
|
|
"\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 85
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "a735bc02ceb4d872",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"jupyter": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"source_hidden": true
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:16:04.486357Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:16:00.946508Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"def read_industry_data(h5_filename):\n",
|
|
|
|
|
|
" # 读取 H5 文件中所有的行业数据\n",
|
|
|
|
|
|
" industry_data = pd.read_hdf(h5_filename, key='sw_daily', columns=[\n",
|
|
|
|
|
|
" 'ts_code', 'trade_date', 'open', 'close', 'high', 'low', 'pe', 'pb', 'vol'\n",
|
|
|
|
|
|
" ]) # 假设 H5 文件的键是 'industry_data'\n",
|
|
|
|
|
|
" industry_data = industry_data.reindex()\n",
|
|
|
|
|
|
" industry_data['trade_date'] = pd.to_datetime(df['trade_date'], format='%Y%m%d')\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" grouped = industry_data.groupby('ts_code', group_keys=False)\n",
|
|
|
|
|
|
" industry_data['obv'] = grouped.apply(\n",
|
|
|
|
|
|
" lambda x: pd.Series(talib.OBV(x['close'].values, x['vol'].values), index=x.index)\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
" industry_data['return_5'] = grouped['close'].apply(lambda x: x / x.shift(5) - 1)\n",
|
|
|
|
|
|
" # industry_data['return_20'] = grouped['close'].apply(lambda x: x / x.shift(20) - 1)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # industry_data = get_act_factor(industry_data, cat=False)\n",
|
|
|
|
|
|
" # industry_data = industry_data.sort_values(by=['trade_date', 'ts_code'])\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 计算每天每个 ts_code 的因子和当天所有 ts_code 的中位数的偏差\n",
|
|
|
|
|
|
" factor_columns = ['obv', 'return_5', 'return_20', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4'] # 因子列\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" for factor in factor_columns:\n",
|
|
|
|
|
|
" if factor in industry_data.columns:\n",
|
|
|
|
|
|
" # 计算每天每个 ts_code 的因子值与当天所有 ts_code 的中位数的偏差\n",
|
|
|
|
|
|
" industry_data[f'{factor}_deviation'] = industry_data.groupby('trade_date')[factor].transform(lambda x: x - x.median())\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" industry_data['return_5_percentile'] = industry_data.groupby('trade_date')['return_5'].transform(lambda x: x.rank(pct=True))\n",
|
|
|
|
|
|
" industry_data = industry_data.drop(columns=['open', 'close', 'high', 'low', 'pe', 'pb', 'vol'])\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" industry_data = industry_data.rename(\n",
|
|
|
|
|
|
" columns={col: f'industry_{col}' for col in industry_data.columns if col not in ['ts_code', 'trade_date']})\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" industry_data = industry_data.rename(columns={'ts_code': 'cat_l2_code'})\n",
|
|
|
|
|
|
" return industry_data\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"industry_df = read_industry_data('../../data/sw_daily.h5')\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 86
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "53f86ddc0677a6d7",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"scrolled": true,
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:16:04.581828Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:16:04.486357Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"origin_columns = df.columns.tolist()\n",
|
|
|
|
|
|
"origin_columns = [col for col in origin_columns if col not in ['turnover_rate', 'pe_ttm', 'volume_ratio', 'l2_code']]\n",
|
|
|
|
|
|
"origin_columns = [col for col in origin_columns if col not in index_data.columns]\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 87
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "dbe2fd8021b9417f",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"scrolled": true,
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:17:07.870387Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:16:04.675873Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"def filter_data(df):\n",
|
|
|
|
|
|
" # df = df.groupby('trade_date').apply(lambda x: x.nlargest(1000, 'act_factor1'))\n",
|
|
|
|
|
|
" df = df[~df['is_st']]\n",
|
|
|
|
|
|
" df = df[~df['ts_code'].str.endswith('BJ')]\n",
|
|
|
|
|
|
" df = df[~df['ts_code'].str.startswith('30')]\n",
|
|
|
|
|
|
" df = df[~df['ts_code'].str.startswith('68')]\n",
|
|
|
|
|
|
" df = df[~df['ts_code'].str.startswith('8')]\n",
|
|
|
|
|
|
" df = df.reset_index(drop=True)\n",
|
|
|
|
|
|
" return df\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"df = filter_data(df)\n",
|
|
|
|
|
|
"df = get_technical_factor(df)\n",
|
|
|
|
|
|
"df = get_act_factor(df)\n",
|
|
|
|
|
|
"df = get_money_flow_factor(df)\n",
|
|
|
|
|
|
"df = get_alpha_factor(df)\n",
|
|
|
|
|
|
"# df = df.merge(industry_df, on=['l2_code', 'trade_date'], how='left')\n",
|
|
|
|
|
|
"df = df.rename(columns={'l2_code': 'cat_l2_code'})\n",
|
|
|
|
|
|
"# df = df.merge(index_data, on='trade_date', how='left')\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"print(df.info())"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
|
|
|
|
|
"<class 'pandas.core.frame.DataFrame'>\n",
|
|
|
|
|
|
"Index: 5538535 entries, 1962 to 5538534\n",
|
|
|
|
|
|
"Data columns (total 72 columns):\n",
|
|
|
|
|
|
" # Column Dtype \n",
|
|
|
|
|
|
"--- ------ ----- \n",
|
|
|
|
|
|
" 0 ts_code object \n",
|
|
|
|
|
|
" 1 trade_date datetime64[ns]\n",
|
|
|
|
|
|
" 2 open float64 \n",
|
|
|
|
|
|
" 3 close float64 \n",
|
|
|
|
|
|
" 4 high float64 \n",
|
|
|
|
|
|
" 5 low float64 \n",
|
|
|
|
|
|
" 6 vol float64 \n",
|
|
|
|
|
|
" 7 turnover_rate float64 \n",
|
|
|
|
|
|
" 8 pe_ttm float64 \n",
|
|
|
|
|
|
" 9 circ_mv float64 \n",
|
|
|
|
|
|
" 10 volume_ratio float64 \n",
|
|
|
|
|
|
" 11 is_st bool \n",
|
|
|
|
|
|
" 12 up_limit float64 \n",
|
|
|
|
|
|
" 13 down_limit float64 \n",
|
|
|
|
|
|
" 14 buy_sm_vol float64 \n",
|
|
|
|
|
|
" 15 sell_sm_vol float64 \n",
|
|
|
|
|
|
" 16 buy_lg_vol float64 \n",
|
|
|
|
|
|
" 17 sell_lg_vol float64 \n",
|
|
|
|
|
|
" 18 buy_elg_vol float64 \n",
|
|
|
|
|
|
" 19 sell_elg_vol float64 \n",
|
|
|
|
|
|
" 20 net_mf_vol float64 \n",
|
|
|
|
|
|
" 21 cat_l2_code object \n",
|
|
|
|
|
|
" 22 log_close float64 \n",
|
|
|
|
|
|
" 23 up float64 \n",
|
|
|
|
|
|
" 24 down float64 \n",
|
|
|
|
|
|
" 25 atr_14 float64 \n",
|
|
|
|
|
|
" 26 atr_6 float64 \n",
|
|
|
|
|
|
" 27 obv float64 \n",
|
|
|
|
|
|
" 28 maobv_6 float64 \n",
|
|
|
|
|
|
" 29 obv-maobv_6 float64 \n",
|
|
|
|
|
|
" 30 rsi_3 float64 \n",
|
|
|
|
|
|
" 31 rsi_6 float64 \n",
|
|
|
|
|
|
" 32 rsi_9 float64 \n",
|
|
|
|
|
|
" 33 return_5 float64 \n",
|
|
|
|
|
|
" 34 return_10 float64 \n",
|
|
|
|
|
|
" 35 return_20 float64 \n",
|
|
|
|
|
|
" 36 avg_close_5 float64 \n",
|
|
|
|
|
|
" 37 std_return_5 float64 \n",
|
|
|
|
|
|
" 38 std_return_15 float64 \n",
|
|
|
|
|
|
" 39 std_return_25 float64 \n",
|
|
|
|
|
|
" 40 std_return_90 float64 \n",
|
|
|
|
|
|
" 41 std_return_90_2 float64 \n",
|
|
|
|
|
|
" 42 std_return_5 / std_return_90 float64 \n",
|
|
|
|
|
|
" 43 std_return_5 / std_return_25 float64 \n",
|
|
|
|
|
|
" 44 std_return_90 - std_return_90_2 float64 \n",
|
|
|
|
|
|
" 45 ema_5 float64 \n",
|
|
|
|
|
|
" 46 ema_13 float64 \n",
|
|
|
|
|
|
" 47 ema_20 float64 \n",
|
|
|
|
|
|
" 48 ema_60 float64 \n",
|
|
|
|
|
|
" 49 act_factor1 float64 \n",
|
|
|
|
|
|
" 50 act_factor2 float64 \n",
|
|
|
|
|
|
" 51 act_factor3 float64 \n",
|
|
|
|
|
|
" 52 act_factor4 float64 \n",
|
|
|
|
|
|
" 53 cat_af1 bool \n",
|
|
|
|
|
|
" 54 cat_af2 bool \n",
|
|
|
|
|
|
" 55 cat_af3 bool \n",
|
|
|
|
|
|
" 56 cat_af4 bool \n",
|
|
|
|
|
|
" 57 act_factor5 float64 \n",
|
|
|
|
|
|
" 58 act_factor6 float64 \n",
|
|
|
|
|
|
" 59 rank_act_factor1 float64 \n",
|
|
|
|
|
|
" 60 rank_act_factor2 float64 \n",
|
|
|
|
|
|
" 61 rank_act_factor3 float64 \n",
|
|
|
|
|
|
" 62 active_buy_volume_large float64 \n",
|
|
|
|
|
|
" 63 active_buy_volume_big float64 \n",
|
|
|
|
|
|
" 64 active_buy_volume_small float64 \n",
|
|
|
|
|
|
" 65 buy_lg_vol_minus_sell_lg_vol float64 \n",
|
|
|
|
|
|
" 66 buy_elg_vol_minus_sell_elg_vol float64 \n",
|
|
|
|
|
|
" 67 log(circ_mv) float64 \n",
|
|
|
|
|
|
" 68 alpha_022 float64 \n",
|
|
|
|
|
|
" 69 alpha_003 float64 \n",
|
|
|
|
|
|
" 70 alpha_007 float64 \n",
|
|
|
|
|
|
" 71 alpha_013 float64 \n",
|
|
|
|
|
|
"dtypes: bool(5), datetime64[ns](1), float64(64), object(2)\n",
|
|
|
|
|
|
"memory usage: 2.8+ GB\n",
|
|
|
|
|
|
"None\n"
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 88
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:17:08.379851Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:17:08.136804Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"def create_deviation_within_dates(df, feature_columns):\n",
|
|
|
|
|
|
" groupby_col = 'cat_l2_code' # 使用 trade_date 进行分组\n",
|
|
|
|
|
|
" new_columns = {}\n",
|
|
|
|
|
|
" ret_feature_columns = feature_columns[:]\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 自动选择所有数值型特征\n",
|
|
|
|
|
|
" num_features = [col for col in feature_columns if 'cat' not in col and 'index' not in col]\n",
|
|
|
|
|
|
" num_features = [col for col in feature_columns if 'cat' not in col and 'industry' not in col]\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 遍历所有数值型特征\n",
|
|
|
|
|
|
" for feature in num_features:\n",
|
|
|
|
|
|
" if feature == 'trade_date': # 不需要对 'trade_date' 计算偏差\n",
|
|
|
|
|
|
" continue\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" grouped_median = df.groupby(['trade_date', groupby_col])[feature].transform('median')\n",
|
|
|
|
|
|
" deviation_col_name = f'deviation_median_{feature}'\n",
|
|
|
|
|
|
" new_columns[deviation_col_name] = df[feature] - grouped_median\n",
|
|
|
|
|
|
" ret_feature_columns.append(deviation_col_name)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" grouped_mean = df.groupby(groupby_col)[feature].transform('mean')\n",
|
|
|
|
|
|
" deviation_col_name = f'deviation_mean_{feature}'\n",
|
|
|
|
|
|
" new_columns[deviation_col_name] = df[feature] - grouped_mean\n",
|
|
|
|
|
|
" ret_feature_columns.append(deviation_col_name)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # 将新计算的偏差特征与原始 DataFrame 合并\n",
|
|
|
|
|
|
" df = pd.concat([df, pd.DataFrame(new_columns)], axis=1)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # for feature in ['obv', 'return_20', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4']:\n",
|
|
|
|
|
|
" # df[f'deviation_industry_{feature}'] = df[feature] - df[f'industry_{feature}']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" return df, ret_feature_columns\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"id": "d345bcc43b15579e",
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 89
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "5f3d9aece75318cd",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:18:01.015229Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:17:08.523778Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"def get_qcuts(series, quantiles):\n",
|
|
|
|
|
|
" q = pd.qcut(series, q=quantiles, labels=False, duplicates='drop')\n",
|
|
|
|
|
|
" return q[-1] # 返回窗口最后一个元素的分位数标签\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"import pandas as pd\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def remove_outliers_label_percentile(label: pd.Series, lower_percentile: float = 0.01, upper_percentile: float = 0.99):\n",
|
|
|
|
|
|
" if not (0 <= lower_percentile < upper_percentile <= 1):\n",
|
|
|
|
|
|
" raise ValueError(\"Percentile values must satisfy 0 <= lower_percentile < upper_percentile <= 1.\")\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # Calculate lower and upper bounds based on percentiles\n",
|
|
|
|
|
|
" lower_bound = label.quantile(lower_percentile)\n",
|
|
|
|
|
|
" upper_bound = label.quantile(upper_percentile)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # Filter out values outside the bounds\n",
|
|
|
|
|
|
" filtered_label = label[(label >= lower_bound) & (label <= upper_bound)]\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # Print the number of removed outliers\n",
|
|
|
|
|
|
" print(f\"Removed {len(label) - len(filtered_label)} outliers.\")\n",
|
|
|
|
|
|
" return filtered_label\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def calculate_risk_adjusted_target(df, days=5):\n",
|
|
|
|
|
|
" df = df.sort_values(by=['ts_code', 'trade_date'])\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" df['future_close'] = df.groupby('ts_code')['close'].shift(-days)\n",
|
|
|
|
|
|
" df['future_return'] = (df['future_close'] - df['close']) / df['close']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" df['future_volatility'] = df.groupby('ts_code')['future_return'].rolling(days, min_periods=1).std().reset_index(\n",
|
|
|
|
|
|
" level=0, drop=True)\n",
|
|
|
|
|
|
" df['sharpe_ratio'] = df['future_return'] * df['future_volatility']\n",
|
|
|
|
|
|
" df['sharpe_ratio'].replace([np.inf, -np.inf], np.nan, inplace=True)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" return df['sharpe_ratio']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"future_close = df.groupby('ts_code')['close'].shift(-4)\n",
|
|
|
|
|
|
"future_return = (future_close - df['close']) / df['close']\n",
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"df['label'] = calculate_risk_adjusted_target(df)\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
"df['label'] = remove_outliers_label_percentile(df['label'])\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# df = df.apply(lambda x: x.astype('float32') if x.dtype in ['float64', 'float32'] else x)\n",
|
|
|
|
|
|
"df = df.sort_values(by=['trade_date', 'ts_code'])\n",
|
|
|
|
|
|
"train_data = df[(df['trade_date'] <= '2023-01-01') & (df['trade_date'] >= '2016-01-01')]\n",
|
|
|
|
|
|
"test_data = df[df['trade_date'] >= '2023-01-01']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"train_data = train_data.merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"train_data = train_data.merge(index_data, on='trade_date', how='left')\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
"test_data = test_data.merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"test_data = test_data.merge(index_data, on='trade_date', how='left')\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
"\n",
|
|
|
|
|
|
"train_data = train_data.groupby('trade_date', group_keys=False).apply(lambda x: x.nlargest(1000, 'return_20'))\n",
|
|
|
|
|
|
"test_data = test_data.groupby('trade_date', group_keys=False).apply(lambda x: x.nlargest(1000, 'return_20'))\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"train_data, test_data = train_data.replace([np.inf, -np.inf], np.nan), test_data.replace([np.inf, -np.inf], np.nan)\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"Removed 130394 outliers.\n"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 90
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:18:01.247742Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:18:01.062461Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"feature_columns = [col for col in train_data.columns if col not in ['trade_date',\n",
|
|
|
|
|
|
" 'ts_code',\n",
|
|
|
|
|
|
" 'label']]\n",
|
|
|
|
|
|
"feature_columns = [col for col in feature_columns if 'future' not in col]\n",
|
|
|
|
|
|
"feature_columns = [col for col in feature_columns if 'score' not in col]\n",
|
|
|
|
|
|
"feature_columns = [col for col in feature_columns if col not in origin_columns]\n",
|
|
|
|
|
|
"feature_columns = [col for col in feature_columns if not col.startswith('_')]\n",
|
|
|
|
|
|
"print(feature_columns)"
|
|
|
|
|
|
],
|
|
|
|
|
|
"id": "93d47ef451968346",
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
|
|
|
|
|
"['turnover_rate', 'pe_ttm', 'volume_ratio', 'cat_l2_code', 'log_close', 'up', 'down', 'atr_14', 'atr_6', 'obv', 'maobv_6', 'obv-maobv_6', 'rsi_3', 'rsi_6', 'rsi_9', 'return_5', 'return_10', 'return_20', 'avg_close_5', 'std_return_5', 'std_return_15', 'std_return_25', 'std_return_90', 'std_return_90_2', 'std_return_5 / std_return_90', 'std_return_5 / std_return_25', 'std_return_90 - std_return_90_2', 'ema_5', 'ema_13', 'ema_20', 'ema_60', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4', 'cat_af1', 'cat_af2', 'cat_af3', 'cat_af4', 'act_factor5', 'act_factor6', 'rank_act_factor1', 'rank_act_factor2', 'rank_act_factor3', 'active_buy_volume_large', 'active_buy_volume_big', 'active_buy_volume_small', 'buy_lg_vol_minus_sell_lg_vol', 'buy_elg_vol_minus_sell_elg_vol', 'log(circ_mv)', 'alpha_022', 'alpha_003', 'alpha_007', 'alpha_013', 'industry_obv', 'industry_return_5', 'industry_obv_deviation', 'industry_return_5_deviation', 'industry_return_5_percentile', '000852.SH_MACD', '000905.SH_MACD', '399006.SZ_MACD', '000852.SH_MACD_hist', '000905.SH_MACD_hist', '399006.SZ_MACD_hist', '000852.SH_RSI', '000905.SH_RSI', '399006.SZ_RSI', '000852.SH_Signal_line', '000905.SH_Signal_line', '399006.SZ_Signal_line', '000852.SH_daily_return', '000905.SH_daily_return', '399006.SZ_daily_return']\n"
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 91
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:18:23.307764Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:18:01.350526Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"# feature_columns_new = feature_columns[:]\n",
|
|
|
|
|
|
"train_data, feature_columns_new = create_deviation_within_dates(train_data, feature_columns)\n",
|
|
|
|
|
|
"print(f'feature_columns size: {len(feature_columns_new)}')\n",
|
|
|
|
|
|
"test_data, feature_columns_new = create_deviation_within_dates(test_data, feature_columns)\n",
|
|
|
|
|
|
"print(f'feature_columns size: {len(feature_columns_new)}')\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"train_data = train_data.dropna(subset=feature_columns_new)\n",
|
|
|
|
|
|
"train_data = train_data.dropna(subset=['label'])\n",
|
|
|
|
|
|
"train_data = train_data.reset_index(drop=True)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"test_data = test_data.dropna(subset=feature_columns_new)\n",
|
|
|
|
|
|
"test_data = test_data.dropna(subset=['label'])\n",
|
|
|
|
|
|
"test_data = test_data.reset_index(drop=True)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"print(len(train_data))\n",
|
|
|
|
|
|
"print(f\"最小日期: {train_data['trade_date'].min().strftime('%Y-%m-%d')}\")\n",
|
|
|
|
|
|
"print(f\"最大日期: {train_data['trade_date'].max().strftime('%Y-%m-%d')}\")\n",
|
|
|
|
|
|
"print(len(test_data))\n",
|
|
|
|
|
|
"print(f\"最小日期: {test_data['trade_date'].min().strftime('%Y-%m-%d')}\")\n",
|
|
|
|
|
|
"print(f\"最大日期: {test_data['trade_date'].max().strftime('%Y-%m-%d')}\")"
|
|
|
|
|
|
],
|
|
|
|
|
|
"id": "572576eea818c865",
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
|
|
|
|
|
"feature_columns size: 202\n",
|
|
|
|
|
|
"feature_columns size: 202\n",
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"803458\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
"最小日期: 2017-03-21\n",
|
|
|
|
|
|
"最大日期: 2022-12-30\n",
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"295396\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
"最小日期: 2023-01-03\n",
|
|
|
|
|
|
"最大日期: 2025-02-10\n"
|
|
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 92
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:18:23.562209Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:18:23.402196Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"cat_columns = [col for col in df.columns if col.startswith('cat')]\n",
|
|
|
|
|
|
"for col in cat_columns:\n",
|
|
|
|
|
|
" train_data[col] = train_data[col].astype('category')\n",
|
|
|
|
|
|
" test_data[col] = test_data[col].astype('category')"
|
|
|
|
|
|
],
|
|
|
|
|
|
"id": "2d7e37432f551aea",
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 93
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "8f134d435f71e9e2",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:18:23.750424Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:18:23.658374Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"import lightgbm as lgb\n",
|
|
|
|
|
|
"import numpy as np\n",
|
|
|
|
|
|
"import matplotlib.pyplot as plt\n",
|
|
|
|
|
|
"import optuna\n",
|
|
|
|
|
|
"from sklearn.model_selection import KFold\n",
|
|
|
|
|
|
"from sklearn.metrics import mean_absolute_error\n",
|
|
|
|
|
|
"import os\n",
|
|
|
|
|
|
"import json\n",
|
|
|
|
|
|
"import pickle\n",
|
|
|
|
|
|
"import hashlib\n",
|
|
|
|
|
|
"from catboost import Pool\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def train_light_model(train_data_df, test_data_df, params, feature_columns, callbacks, evals,\n",
|
|
|
|
|
|
" print_feature_importance=True, num_boost_round=100,\n",
|
|
|
|
|
|
" use_optuna=False):\n",
|
|
|
|
|
|
" train_data_df, test_data_df = train_data_df.dropna(subset=['label']), test_data_df.dropna(subset=['label'])\n",
|
|
|
|
|
|
" X_train = train_data_df[feature_columns]\n",
|
|
|
|
|
|
" y_train = train_data_df['label']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" X_val = test_data_df[feature_columns]\n",
|
|
|
|
|
|
" y_val = test_data_df['label']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" categorical_feature = [i for i, col in enumerate(feature_columns) if col.startswith('cat')]\n",
|
|
|
|
|
|
" print(f'categorical_feature: {categorical_feature}')\n",
|
|
|
|
|
|
" train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=categorical_feature)\n",
|
|
|
|
|
|
" val_data = lgb.Dataset(X_val, label=y_val, categorical_feature=categorical_feature)\n",
|
|
|
|
|
|
" model = lgb.train(\n",
|
|
|
|
|
|
" params, train_data, num_boost_round=num_boost_round,\n",
|
|
|
|
|
|
" valid_sets=[train_data, val_data], valid_names=['train', 'valid'],\n",
|
|
|
|
|
|
" callbacks=callbacks\n",
|
|
|
|
|
|
" )\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" if print_feature_importance:\n",
|
|
|
|
|
|
" lgb.plot_metric(evals)\n",
|
|
|
|
|
|
" # lgb.plot_tree(model, figsize=(20, 8))\n",
|
|
|
|
|
|
" lgb.plot_importance(model, importance_type='split', max_num_features=20)\n",
|
|
|
|
|
|
" plt.show()\n",
|
|
|
|
|
|
" return model\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"from catboost import CatBoostRegressor\n",
|
|
|
|
|
|
"import pandas as pd\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def train_catboost(train_data_df, test_data_df, feature_columns, params=None, plot=False):\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" train_data_df, test_data_df = train_data_df.dropna(subset=['label']), test_data_df.dropna(subset=['label'])\n",
|
|
|
|
|
|
" X_train = train_data_df[feature_columns]\n",
|
|
|
|
|
|
" y_train = train_data_df['label']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" X_val = test_data_df[feature_columns]\n",
|
|
|
|
|
|
" y_val = test_data_df['label']\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" cat_features = [i for i, col in enumerate(feature_columns) if col.startswith('cat')]\n",
|
|
|
|
|
|
" print(f'cat_features: {cat_features}')\n",
|
|
|
|
|
|
" train_pool = Pool(data=X_train, label=y_train, cat_features=cat_features)\n",
|
|
|
|
|
|
" val_pool = Pool(data=X_val, label=y_val, cat_features=cat_features)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" model = CatBoostRegressor(**params)\n",
|
|
|
|
|
|
" model.fit(train_pool,\n",
|
|
|
|
|
|
" eval_set=[train_pool, val_pool])\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" return model"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 94
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "4a4542e1ed6afe7d",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:18:23.926397Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:18:23.845350Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"light_params = {\n",
|
2025-02-24 00:23:19 +08:00
|
|
|
|
" # 'objective': 'regression',\n",
|
|
|
|
|
|
" # 'metric': 'l2',\n",
|
|
|
|
|
|
" 'objective': 'quantile', # 分位回归\n",
|
|
|
|
|
|
" 'metric': 'quantile', # 使用 quantile 作为评估指标\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
" 'alpha': 0.75, # 90% 分位数\n",
|
|
|
|
|
|
" 'learning_rate': 0.05,\n",
|
|
|
|
|
|
" 'is_unbalance': True,\n",
|
|
|
|
|
|
" 'num_leaves': 1024,\n",
|
|
|
|
|
|
" 'min_data_in_leaf': 128,\n",
|
|
|
|
|
|
" 'max_depth': 32,\n",
|
|
|
|
|
|
" 'max_bin': 1024,\n",
|
|
|
|
|
|
" 'feature_fraction': 0.7,\n",
|
|
|
|
|
|
" 'bagging_fraction': 0.7,\n",
|
|
|
|
|
|
" 'bagging_freq': 5,\n",
|
|
|
|
|
|
" 'lambda_l1': 1,\n",
|
|
|
|
|
|
" 'lambda_l2': 1,\n",
|
|
|
|
|
|
" # 'boosting_type': 'dart',\n",
|
|
|
|
|
|
" 'verbosity': -1\n",
|
|
|
|
|
|
"}"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 95
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "beeb098799ecfa6a",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:19:00.337345Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:18:24.039383Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"print('train data size: ', len(train_data))\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"evals = {}\n",
|
|
|
|
|
|
"model = train_light_model(train_data, test_data, light_params, feature_columns_new,\n",
|
|
|
|
|
|
" [lgb.log_evaluation(period=500),\n",
|
|
|
|
|
|
" lgb.callback.record_evaluation(evals),\n",
|
|
|
|
|
|
" lgb.early_stopping(50, first_metric_only=True)\n",
|
|
|
|
|
|
" ], evals,\n",
|
|
|
|
|
|
" num_boost_round=500, use_optuna=False,\n",
|
|
|
|
|
|
" print_feature_importance=True)"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"train data size: 803458\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
"categorical_feature: [3, 35, 36, 37, 38]\n",
|
|
|
|
|
|
"Training until validation scores don't improve for 50 rounds\n",
|
|
|
|
|
|
"Early stopping, best iteration is:\n",
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"[42]\ttrain's quantile: 0.00081298\tvalid's quantile: 0.000834099\n",
|
|
|
|
|
|
"Evaluated only: quantile\n"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
]
|
|
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"data": {
|
|
|
|
|
|
"text/plain": [
|
|
|
|
|
|
"<Figure size 640x480 with 1 Axes>"
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAloAAAHFCAYAAAAqg1fhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9QklEQVR4nO3dd1hT1/8H8HcSEsLeMgQZoiLiAly460Bx19aNtbX+aqdKW6tVv612qF1aW60dWmtrK22drThw4aIOBBw4EEEUGQaVqSTA/f0RiUZAAYkh8H49z30gJ+fe+7n3APlw7rnnigRBEEBEREREtU6s7wCIiIiI6ismWkREREQ6wkSLiIiISEeYaBERERHpCBMtIiIiIh1hokVERESkI0y0iIiIiHSEiRYRERGRjjDRIiIiItIRJlpEDdSaNWsgEokgEomwf//+cu8LggBvb2+IRCL06tWrRvtYsWIF1qxZU6119u/fX2lMtUVX+3gasVcmISEBH374IVJSUnSy/Q8//BAikahG6+rzvBDpGxMtogbOwsICq1atKlceFRWFpKQkWFhY1HjbNUm0/P39ER0dDX9//xrvV1/0GXtCQgLmz5+vs0Tr5ZdfRnR0dI3WNeQ2JXpSTLSIGrjRo0djw4YNyM3N1SpftWoVunTpgiZNmjyVOFQqFYqLi2FpaYnOnTvD0tLyqey3Nhhi7IWFhdWq7+rqis6dO9doX4Z0XohqGxMtogZu7NixAIA//vhDU5aTk4MNGzbgpZdeqnAdpVKJjz/+GD4+PjA2NoaDgwNefPFF3LhxQ1PHw8MDZ8+eRVRUlOYSpYeHB4D7l5J+/fVXvP3222jcuDGMjY1x6dKlSi8zHT16FEOGDIGdnR3kcjmaNm2K6dOnP/b4zp8/jwEDBsDU1BT29vaYOnUq8vLyytXz8PDApEmTypX36tVL69JpdWOfNGkSzM3NcenSJYSEhMDc3Bxubm54++23UVRUpLWva9eu4bnnnoOFhQWsra0xfvx4HD9+HCKR6JE9g2vWrMHzzz8PAOjdu7fmfJet06tXL/j5+eHAgQMICgqCqamppm3Dw8PRv39/ODs7w8TEBC1btsSsWbNQUFCgtY+KLh16eHhg8ODB2LFjB/z9/WFiYgIfHx+sXr1aq56+zgtRXcBEi6iBs7S0xHPPPaf14fjHH39ALBZj9OjR5eqXlpZi2LBhWLRoEcaNG4dt27Zh0aJFiIyMRK9evXDnzh0AwKZNm+Dl5YX27dsjOjoa0dHR2LRpk9a2Zs+ejdTUVKxcuRL//PMPGjVqVGGMO3fuRPfu3ZGamoqvvvoK27dvx9y5c5GZmfnIY8vMzETPnj1x5swZrFixAr/++ivy8/PxxhtvVPc0lVPV2AF1j9fQoUPRp08fbNmyBS+99BKWLFmCxYsXa+oUFBSgd+/e2LdvHxYvXow///wTjo6OFbbBwwYNGoRPP/0UALB8+XLN+R40aJCmTnp6OiZMmIBx48YhIiICr732GgAgMTERISEhWLVqFXbs2IHp06fjzz//xJAhQ6p0HuLj4/H2229jxowZ2LJlC9q0aYPJkyfjwIEDj11X1+eFqE4QiKhB+vnnnwUAwvHjx4V9+/YJAIQzZ84IgiAIHTp0ECZNmiQIgiC0atVK6Nmzp2a9P/74QwAgbNiwQWt7x48fFwAIK1as0JQ9vG6Zsv316NGj0vf27dunKWvatKnQtGlT4c6dO9U6xvfee08QiURCXFycVnm/fv3K7cPd3V144YUXym2jZ8+eWsdQ3dhfeOEFAYDw559/atUNCQkRWrRooXm9fPlyAYCwfft2rXqvvPKKAED4+eefH3msf/31V7l9P3gMAIQ9e/Y8chulpaWCSqUSoqKiBABCfHy85r0PPvhAePgjw93dXZDL5cKVK1c0ZXfu3BFsbW2FV155RVOmz/NCpG/s0SIi9OzZE02bNsXq1atx+vRpHD9+vNLLhv/++y+sra0xZMgQFBcXa5Z27drBycmpWneWjRw58rF1Ll68iKSkJEyePBlyubzK2waAffv2oVWrVmjbtq1W+bhx46q1nYpUJfYyIpGoXA9RmzZtcOXKFc3rqKgoWFhYYMCAAVr1yi7tPikbGxs888wz5covX76McePGwcnJCRKJBFKpFD179gQAnDt37rHbbdeundY4PrlcjubNm2sdW2Xqwnkh0jUjfQdARPonEonw4osvYtmyZbh79y6aN2+O7t27V1g3MzMTt2/fhkwmq/B9hUJR5f06Ozs/tk7ZuC9XV9cqb7dMdnY2PD09y5U7OTlVe1sPq0rsZUxNTcslicbGxrh7967mdXZ2NhwdHcutW1FZTVQUb35+Prp37w65XI6PP/4YzZs3h6mpKa5evYpnn31Wcxn4Uezs7MqVGRsbV2ndunBeiHSNiRYRAVAPTv7f//6HlStX4pNPPqm0nr29Pezs7LBjx44K36/OdBBVmZfJwcEBgHpAdHXZ2dkhIyOjXHlFZXK5vNwgbECdONrb25crr+mcUpWxs7PDsWPHypVXFGtNVBTv3r17cf36dezfv1/TiwUAt2/frpV91gZdnxciXeOlQyICADRu3BjvvvsuhgwZghdeeKHSeoMHD0Z2djZKSkoQGBhYbmnRooWmblV7Nh6lefPmmsuaFSVCj9K7d2+cPXsW8fHxWuW///57uboeHh44deqUVtnFixdx4cKF6gddAz179kReXh62b9+uVb5+/foqrW9sbAwA1TrfZclX2bplvv/++ypvQ9ee9LwQ6Rt7tIhIY9GiRY+tM2bMGKxbtw4hISGYNm0aOnbsCKlUimvXrmHfvn0YNmwYRowYAQBo3bo11q9fj/DwcHh5eUEul6N169bVjmv58uUYMmQIOnfujBkzZqBJkyZITU3Fzp07sW7dukrXmz59OlavXo1Bgwbh448/hqOjI9atW4fz58+XqxsaGooJEybgtddew8iRI3HlyhV89tlnmh41XXvhhRewZMkSTJgwAR9//DG8vb2xfft27Ny5EwAgFj/6/2I/Pz8AwA8//AALCwvI5XJ4enpWeGmvTFBQEGxsbDB16lR88MEHkEqlWLduXbnEVJ+e9LwQ6Rt/QomoWiQSCbZu3Yr3338fGzduxIgRIzB8+HAsWrSoXCI1f/589OzZE1OmTEHHjh2rPGXAw4KDg3HgwAE4OzvjrbfewoABA7BgwYLHjtNxcnJCVFQUfH198eqrr2LChAmQy+X49ttvy9UdN24cPvvsM+zcuRODBw/Gd999h++++w7NmzevUczVZWZmhr1796JXr16YOXMmRo4cidTUVKxYsQIAYG1t/cj1PT09sXTpUsTHx6NXr17o0KED/vnnn0euY2dnh23btsHU1BQTJkzASy+9BHNzc4SHh9fWYT2xJz0vRPomEgRB0HcQRERUsU8//RRz585FampqjW4IqK94XshQ8NIhEVEdUdbT5uPjA5VKhb1792LZsmWYMGFCg04meF7IkDHRIiKqI0xNTbFkyRKkpKSgqKgITZo0wXvvvYe5c+fqOzS94nkhQ8ZLh0REREQ6wsHwRERERDrCRIuIiIhIR5hoEREREekIB8PrWWlpKa5fvw4LC4taf6QHERER6YYgCMjLy4OLi8sjJ85loqVn169fh5ubm77DICIiohq4evXqI6cZYaKlZ2UP4E1OToatra2eo6HHUalU2LVrF/r37w+pVKrvcOgx2F6Gg21lWNheQG5uLtzc3DSf45VhoqVnZZcLLSwsYGlpqedo6HFUKhVMTU1haWnZYP+4GBK2l+FgWxkWttd9jxv2w8HwRERERDrCRIuIiIhIR5hoEREREekIx2gRERHVQyUlJVCpVDrZtkqlgpGREe7evYuSkhKd7EPfpFIpJBLJE2+HiRYREVE9IggCMjIycPv2bZ3uw8nJCVevXq3Xc0BaW1vDycnpiY6RiRYREVE9UpZkNWrUCKampjpJhEpLS5Gfnw9zc/NHTtZpqARBQGFhIbKysgAAzs7ONd4WEy0iIqJ6oqSkRJNk2dnZ6Ww/paWlUCqVkMvl9TLRAgATExMAQFZWFho1alTjy4j18+w
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
"output_type": "display_data"
|
|
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"data": {
|
|
|
|
|
|
"text/plain": [
|
|
|
|
|
|
"<Figure size 640x480 with 1 Axes>"
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAzUAAAHFCAYAAADL4dfcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3yO1//48dedvUN2oiFBYs9YoRUhEkTsUSP42KW1a6+q2LNao0bQauOrQWs0hFipETTRCNXaWokZCUHm9fvDL9fdW4JQpHg/H4/7ofd13te5znlH5T73Ode5NIqiKAghhBBCCCHEG0qvsBsghBBCCCGEEP+GDGqEEEIIIYQQbzQZ1AghhBBCCCHeaDKoEUIIIYQQQrzRZFAjhBBCCCGEeKPJoEYIIYQQQgjxRpNBjRBCCCGEEOKNJoMaIYQQQgghxBtNBjVCCCGEEEKIN5oMaoQQQohCtnr1ajQaTb6vESNGvJJrnjp1ismTJ3Px4sVXUv+/cfHiRTQaDatXry7spryw7du3M3ny5MJuhhDvDIPCboAQQgghHgkNDaVs2bI6x1xcXF7JtU6dOsVnn31GgwYNcHNzeyXXeFHOzs4cOnSIUqVKFXZTXtj27dv56quvZGAjxGsigxohhBDiP6JixYrUqFGjsJvxr2RmZqLRaDAwePGPGMbGxtSpU+cltur1uX//PmZmZoXdDCHeObL8TAghhHhDrF+/Hm9vb8zNzbGwsCAgIIDY2FidmGPHjvHhhx/i5uaGqakpbm5udOrUiUuXLqkxq1evpn379gD4+vqqS91yl3u5ubnRo0ePPNdv0KABDRo0UN/v3bsXjUbDN998w/DhwylWrBjGxsacPXsWgF27dtGoUSOsrKwwMzOjXr167N69+5n9zG/52eTJk9FoNPz222+0b98ea2trbGxsGDZsGFlZWZw5c4YmTZpgaWmJm5sbs2bN0qkzt63ffvstw4YNw8nJCVNTU3x8fPLkEOCnn37C29sbMzMzLC0tady4MYcOHdKJyW3Tr7/+Srt27ShatCilSpWiR48efPXVVwA6Swlzl/p99dVX1K9fHwcHB8zNzalUqRKzZs0iMzMzT74rVqzI0aNH+eCDDzAzM6NkyZLMmDGDnJwcndg7d+4wfPhwSpYsibGxMQ4ODjRr1ozff/9djcnIyGDq1KmULVsWY2Nj7O3t+d///seNGzee+TMR4r9OBjVCCCHEf0R2djZZWVk6r1zTpk2jU6dOlC9fnv/7v//jm2++4e7du3zwwQecOnVKjbt48SJlypRhwYIF7Nixg5kzZ5KYmEjNmjW5efMmAIGBgUybNg149AH70KFDHDp0iMDAwBdq95gxY7h8+TJLly5ly5YtODg48O233+Lv74+VlRVr1qzh//7v/7CxsSEgIKBAA5sn6dChA1WqVCE8PJw+ffowf/58hg4dSqtWrQgMDGTTpk00bNiQUaNGsXHjxjznjx07lvPnz7NixQpWrFjB1atXadCgAefPn1djvvvuO1q2bImVlRXff/89K1euJDk5mQYNGhAdHZ2nzjZt2lC6dGk2bNjA0qVLmTBhAu3atQNQc3vo0CGcnZ0BOHfuHJ07d+abb75h69at9OrVi9mzZ9OvX788dSclJdGlSxe6du3KTz/9RNOmTRkzZgzffvutGnP37l3ef/99li1bxv/+9z+2bNnC0qVL8fT0JDExEYCcnBxatmzJjBkz6Ny5M9u2bWPGjBlERkbSoEEDHjx48MI/EyH+ExQhhBBCFKrQ0FAFyPeVmZmpXL58WTEwMFA++eQTnfPu3r2rODk5KR06dHhi3VlZWcq9e/cUc3NzZeHCherxDRs2KICyZ8+ePOeUKFFC6d69e57jPj4+io+Pj/p+z549CqDUr19fJy4tLU2xsbFRgoKCdI5nZ2crVapUUWrVqvWUbCjKhQsXFEAJDQ1Vj02aNEkBlLlz5+rEVq1aVQGUjRs3qscyMzMVe3t7pU2bNnnaWr16dSUnJ0c9fvHiRcXQ0FDp3bu32kYXFxelUqVKSnZ2thp39+5dxcHBQalbt26eNk2cODFPHwYOHKgU5GNWdna2kpmZqaxdu1bR19dXbt++rZb5+PgogHLkyBGdc8qXL68EBASo76dMmaIASmRk5BOv8/333yuAEh4ernP86NGjCqAsXrz4mW0V4r9MZmqEEEKI/4i1a9dy9OhRnZeBgQE7duwgKyuLbt266czimJiY4OPjw969e9U67t27x6hRoyhdujQGBgYYGBhgYWFBWloap0+ffiXtbtu2rc77gwcPcvv2bbp3767T3pycHJo0acLRo0dJS0t7oWs1b95c5325cuXQaDQ0bdpUPWZgYEDp0qV1ltzl6ty5MxqNRn1fokQJ6taty549ewA4c+YMV69eJTg4GD097cckCwsL2rZty+HDh7l///5T+/8ssbGxtGjRAltbW/T19TE0NKRbt25kZ2fzxx9/6MQ6OTlRq1YtnWOVK1fW6dvPP/+Mp6cnfn5+T7zm1q1bKVKkCEFBQTo/k6pVq+Lk5KTzd0iIN5FsFCCEEEL8R5QrVy7fjQKuXbsGQM2aNfM9758fvjt37szu3buZMGECNWvWxMrKCo1GQ7NmzV7ZEqPcZVWPtzd3CVZ+bt++jbm5+XNfy8bGRue9kZERZmZmmJiY5Dmempqa53wnJ6d8j504cQKAW7duAXn7BI92osvJySE5OVlnM4D8Yp/k8uXLfPDBB5QpU4aFCxfi5uaGiYkJMTExDBw4MM/PyNbWNk8dxsbGOnE3btygePHiT73utWvXuHPnDkZGRvmW5y5NFOJNJYMaIYQQ4j/Ozs4OgB9++IESJUo8MS4lJYWtW7cyadIkRo8erR5PT0/n9u3bBb6eiYkJ6enpeY7fvHlTbcs//XPm45/tXbRo0RN3MXN0dCxwe16mpKSkfI/lDh5y/8y9F+Wfrl69ip6eHkWLFtU5/nj/n2bz5s2kpaWxceNGnZ9lXFxcget4nL29PX/99ddTY+zs7LC1tSUiIiLfcktLyxe+vhD/BTKoEUIIIf7jAgICMDAw4Ny5c09d6qTRaFAUBWNjY53jK1asIDs7W+dYbkx+szdubm789ttvOsf++OMPzpw5k++g5nH16tWjSJEinDp1io8//viZ8a/T999/z7Bhw9SByKVLlzh48CDdunUDoEyZMhQrVozvvvuOESNGqHFpaWmEh4erO6I9yz/za2pqqh7Pre+fPyNFUVi+fPkL96lp06ZMnDiRqKgoGjZsmG9M8+bNCQsLIzs7m9q1a7/wtYT4r5JBjRBCCPEf5+bmxpQpUxg3bhznz5+nSZMmFC1alGvXrhETE4O5uTmfffYZVlZW1K9fn9mzZ2NnZ4ebmxv79u1j5cqVFClSRKfOihUrAvD1119jaWmJiYkJ7u7u2NraEhwcTNeuXRkwYABt27bl0qVLzJo1C3t7+wK118LCgkWLFtG9e3du375Nu3btcHBw4MaNG5w4cYIbN26wZMmSl52mArl+/TqtW7emT58+pKSkMGnSJExMTBgzZgzwaCnfrFmz6NKlC82bN6dfv36kp6cze/Zs7ty5w4wZMwp0nUqVKgEwc+ZMmjZtir6+PpUrV6Zx48YYGRnRqVMnRo4cycOHD1myZAnJyckv3KchQ4awfv16WrZsyejRo6lVqxYPHjxg3759NG/eHF9fXz788EPWrVtHs2bNGDx4MLVq1cLQ0JC//vqLPXv20LJlS1q3bv3CbRCisMlGAUIIIcQbYMyYMfzwww/88ccfdO/enYCAAEaOHMmlS5eoX7++Gvfdd9/h6+vLyJEjadOmDceOHSMyMhJra2ud+tzd3VmwYAEnTpygQYMG1KxZky1btgCP7suZNWsWO3bsoHnz5ixZsoQlS5bg6elZ4PZ27dqVPXv2cO/ePfr164efnx+DBw/m119/pVGjRi8nKS9g2rRplChRgv/973/07NkTZ2dn9uzZQ6lSpdSYzp07s3nzZm7dukXHjh353//+h5WVFXv27OH9998v0HU6d+5M7969Wbx4Md7e3tSsWZOrV69StmxZwsPDSU5Opk2
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
"metadata": {},
|
|
|
|
|
|
"output_type": "display_data"
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 96
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "63235069-dc59-48fb-961a-e80373e41a61",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:19:00.519302Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:19:00.431726Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"print('train data size: ', len(train_data))\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"catboost_params = {\n",
|
|
|
|
|
|
" 'loss_function': 'MAE', # 90% 分位回归\n",
|
|
|
|
|
|
" 'iterations': 5000, # 训练轮数\n",
|
|
|
|
|
|
" 'learning_rate': 0.05, # 学习率，较低以防止过拟合\n",
|
|
|
|
|
|
" 'depth': 10, # 树的深度，防止过拟合\n",
|
|
|
|
|
|
" # 'l1_leaf_reg': 10.0, # l1 正则化，提高泛化能力\n",
|
|
|
|
|
|
" # 'bagging_temperature': 1, # 降低过拟合\n",
|
|
|
|
|
|
" # 'subsample': 0.8, # 每轮随机 80% 的样本，减少过拟合\n",
|
|
|
|
|
|
" 'colsample_bylevel': 0.8, # 每层 80% 特征子集，防止过拟合\n",
|
|
|
|
|
|
" 'random_seed': 42, # 固定随机种子，保证可复现\n",
|
|
|
|
|
|
" 'verbose': 500, # 每 100 轮打印一次信息\n",
|
|
|
|
|
|
" 'early_stopping_rounds': 100, # 早停，防止过拟合\n",
|
|
|
|
|
|
" # 'task_type': 'GPU'\n",
|
|
|
|
|
|
"}\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"# model = train_catboost(train_data, test_data, feature_columns_new, catboost_params)"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stdout",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"train data size: 803458\n"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 97
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:19:00.712334Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:19:00.629410Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"source": [
|
|
|
|
|
|
"from tqdm import tqdm\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
"def incremental_training(test_data: pd.DataFrame,\n",
|
|
|
|
|
|
" model,\n",
|
|
|
|
|
|
" days: int,\n",
|
|
|
|
|
|
" back_days: int,\n",
|
|
|
|
|
|
" feature_columns: list,\n",
|
|
|
|
|
|
" params: dict,\n",
|
|
|
|
|
|
" model_type: str = 'lightgbm'):\n",
|
|
|
|
|
|
" if model_type not in ['lightgbm', 'catboost']:\n",
|
|
|
|
|
|
" raise ValueError(\"model_type must be either 'lightgbm' or 'catboost'\")\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" test_data = test_data.sort_values(by='trade_date')\n",
|
|
|
|
|
|
" scores = []\n",
|
|
|
|
|
|
" unique_trade_dates = sorted(test_data['trade_date'].unique())\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" new_model = None\n",
|
|
|
|
|
|
" for i in tqdm(range(0, len(unique_trade_dates), days)):\n",
|
|
|
|
|
|
" # Get the current window of trade dates\n",
|
|
|
|
|
|
" current_dates = unique_trade_dates[i:i + days]\n",
|
|
|
|
|
|
" window_data = test_data[test_data['trade_date'].isin(current_dates)]\n",
|
|
|
|
|
|
" X = window_data[feature_columns]\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" if new_model is not None:\n",
|
|
|
|
|
|
" window_scores = new_model.predict(X, prediction_type='RawFormulaVal')\n",
|
|
|
|
|
|
" else:\n",
|
|
|
|
|
|
" window_scores = model.predict(X, prediction_type='RawFormulaVal')\n",
|
|
|
|
|
|
" scores.extend(window_scores)\n",
|
|
|
|
|
|
"\n",
|
2025-02-24 00:23:19 +08:00
|
|
|
|
" # # Prepare data for incremental training\n",
|
|
|
|
|
|
" # current_dates = unique_trade_dates[max(0, i - back_days):i + days]\n",
|
|
|
|
|
|
" # window_data = test_data[test_data['trade_date'].isin(current_dates)]\n",
|
|
|
|
|
|
" # X_train = window_data[feature_columns]\n",
|
|
|
|
|
|
" X_train = X\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
" y_train = window_data['label'] # Assuming 'label' is what you're predicting\n",
|
|
|
|
|
|
" # Incrementally train the model\n",
|
|
|
|
|
|
" if len(y_train.unique()) > 1:\n",
|
|
|
|
|
|
" if model_type == 'lightgbm':\n",
|
|
|
|
|
|
" categorical_feature = [i for i, col in enumerate(feature_columns) if col.startswith('cat')]\n",
|
|
|
|
|
|
" train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=categorical_feature)\n",
|
|
|
|
|
|
" new_model = lgb.train(params,\n",
|
|
|
|
|
|
" train_set=train_data,\n",
|
|
|
|
|
|
" num_boost_round=100,\n",
|
|
|
|
|
|
" init_model=model,\n",
|
|
|
|
|
|
" keep_training_booster=True)\n",
|
|
|
|
|
|
" elif model_type == 'catboost':\n",
|
|
|
|
|
|
" from catboost import Pool\n",
|
|
|
|
|
|
" train_data = Pool(data=X_train, label=y_train, cat_features=[col for col in feature_columns if col.startswith('cat')])\n",
|
|
|
|
|
|
" # model.set_params(**params)\n",
|
|
|
|
|
|
" model.fit(train_data, init_model=model)\n",
|
|
|
|
|
|
" else:\n",
|
|
|
|
|
|
" print(current_dates)\n",
|
|
|
|
|
|
"\n",
|
|
|
|
|
|
" # Add the scores as a new 'score' column to the test_data\n",
|
|
|
|
|
|
" test_data['score'] = scores\n",
|
|
|
|
|
|
" return test_data"
|
|
|
|
|
|
],
|
|
|
|
|
|
"id": "465944b1d463e4b1",
|
|
|
|
|
|
"outputs": [],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 98
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "7359f89064a124d2",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:19:43.822814Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:19:00.806697Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"# future_close = test_data.groupby('ts_code')['close'].shift(-3)\n",
|
|
|
|
|
|
"# future_return = (future_close - df['close']) / df['close']\n",
|
|
|
|
|
|
"# test_data['label'] = future_return\n",
|
|
|
|
|
|
"# test_data['label'] = remove_outliers_label_percentile(df['label'])\n",
|
2025-02-23 23:47:19 +08:00
|
|
|
|
"\n",
|
|
|
|
|
|
"predictions_test = incremental_training(test_data, model, 5, 0, feature_columns_new, light_params, model_type='lightgbm')\n",
|
|
|
|
|
|
"predictions_test = predictions_test.loc[predictions_test.groupby('trade_date')['score'].idxmax()]\n",
|
|
|
|
|
|
"predictions_test[['trade_date', 'score', 'ts_code']].to_csv('predictions_test.tsv', index=False)\n"
|
|
|
|
|
|
],
|
|
|
|
|
|
"outputs": [
|
|
|
|
|
|
{
|
|
|
|
|
|
"name": "stderr",
|
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
|
"text": [
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"100%|██████████| 101/101 [00:42<00:00, 2.37it/s]\n"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
]
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"execution_count": 99
|
2025-02-23 23:47:19 +08:00
|
|
|
|
},
|
|
|
|
|
|
{
|
|
|
|
|
|
"cell_type": "code",
|
|
|
|
|
|
"id": "b427ce41-9739-4e9e-bea8-5f2551fec5d7",
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"ExecuteTime": {
|
2025-02-24 00:23:19 +08:00
|
|
|
|
"end_time": "2025-02-23T16:19:43.950667Z",
|
|
|
|
|
|
"start_time": "2025-02-23T16:19:43.934996Z"
|
2025-02-23 23:47:19 +08:00
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"source": [],
|
|
|
|
|
|
"outputs": [],
|
|
|
|
|
|
"execution_count": null
|
|
|
|
|
|
}
|
|
|
|
|
|
],
|
|
|
|
|
|
"metadata": {
|
|
|
|
|
|
"kernelspec": {
|
|
|
|
|
|
"display_name": "Python 3 (ipykernel)",
|
|
|
|
|
|
"language": "python",
|
|
|
|
|
|
"name": "python3"
|
|
|
|
|
|
},
|
|
|
|
|
|
"language_info": {
|
|
|
|
|
|
"codemirror_mode": {
|
|
|
|
|
|
"name": "ipython",
|
|
|
|
|
|
"version": 3
|
|
|
|
|
|
},
|
|
|
|
|
|
"file_extension": ".py",
|
|
|
|
|
|
"mimetype": "text/x-python",
|
|
|
|
|
|
"name": "python",
|
|
|
|
|
|
"nbconvert_exporter": "python",
|
|
|
|
|
|
"pygments_lexer": "ipython3",
|
|
|
|
|
|
"version": "3.8.19"
|
|
|
|
|
|
}
|
|
|
|
|
|
},
|
|
|
|
|
|
"nbformat": 4,
|
|
|
|
|
|
"nbformat_minor": 5
|
|
|
|
|
|
}
|
