NewStock/code/train/Rank.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "79a7758178bafdd3",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "ExecuteTime": {
     "end_time": "2025-04-03T12:46:06.987506Z",
     "start_time": "2025-04-03T12:46:06.259551Z"
    }
   },
   "source": [
    "# %load_ext autoreload\n",
    "# %autoreload 2\n",
    "\n",
    "import pandas as pd\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "pd.set_option('display.max_columns', None)\n"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "id": "a79cafb06a7e0e43",
   "metadata": {
    "scrolled": true,
    "ExecuteTime": {
     "end_time": "2025-04-03T12:47:00.212859Z",
     "start_time": "2025-04-03T12:46:06.998047Z"
    }
   },
   "source": [
    "from utils.utils import read_and_merge_h5_data\n",
    "\n",
    "print('daily data')\n",
    "df = read_and_merge_h5_data('../../data/daily_data.h5', key='daily_data',\n",
    "                            columns=['ts_code', 'trade_date', 'open', 'close', 'high', 'low', 'vol', 'pct_chg'],\n",
    "                            df=None)\n",
    "\n",
    "print('daily basic')\n",
    "df = read_and_merge_h5_data('../../data/daily_basic.h5', key='daily_basic',\n",
    "                            columns=['ts_code', 'trade_date', 'turnover_rate', 'pe_ttm', 'circ_mv', 'volume_ratio',\n",
    "                                     'is_st'], df=df, join='inner')\n",
    "\n",
    "print('stk limit')\n",
    "df = read_and_merge_h5_data('../../data/stk_limit.h5', key='stk_limit',\n",
    "                            columns=['ts_code', 'trade_date', 'pre_close', 'up_limit', 'down_limit'],\n",
    "                            df=df)\n",
    "print('money flow')\n",
    "df = read_and_merge_h5_data('../../data/money_flow.h5', key='money_flow',\n",
    "                            columns=['ts_code', 'trade_date', 'buy_sm_vol', 'sell_sm_vol', 'buy_lg_vol', 'sell_lg_vol',\n",
    "                                     'buy_elg_vol', 'sell_elg_vol', 'net_mf_vol'],\n",
    "                            df=df)\n",
    "print('cyq perf')\n",
    "df = read_and_merge_h5_data('../../data/cyq_perf.h5', key='cyq_perf',\n",
    "                            columns=['ts_code', 'trade_date', 'his_low', 'his_high', 'cost_5pct', 'cost_15pct',\n",
    "                                     'cost_50pct',\n",
    "                                     'cost_85pct', 'cost_95pct', 'weight_avg', 'winner_rate'],\n",
    "                            df=df)\n",
    "print(df.info())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "daily data\n",
      "daily basic\n",
      "inner merge on ['ts_code', 'trade_date']\n",
      "stk limit\n",
      "left merge on ['ts_code', 'trade_date']\n",
      "money flow\n",
      "left merge on ['ts_code', 'trade_date']\n",
      "cyq perf\n",
      "left merge on ['ts_code', 'trade_date']\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8477357 entries, 0 to 8477356\n",
      "Data columns (total 31 columns):\n",
      " #   Column         Dtype         \n",
      "---  ------         -----         \n",
      " 0   ts_code        object        \n",
      " 1   trade_date     datetime64[ns]\n",
      " 2   open           float64       \n",
      " 3   close          float64       \n",
      " 4   high           float64       \n",
      " 5   low            float64       \n",
      " 6   vol            float64       \n",
      " 7   pct_chg        float64       \n",
      " 8   turnover_rate  float64       \n",
      " 9   pe_ttm         float64       \n",
      " 10  circ_mv        float64       \n",
      " 11  volume_ratio   float64       \n",
      " 12  is_st          bool          \n",
      " 13  up_limit       float64       \n",
      " 14  down_limit     float64       \n",
      " 15  buy_sm_vol     float64       \n",
      " 16  sell_sm_vol    float64       \n",
      " 17  buy_lg_vol     float64       \n",
      " 18  sell_lg_vol    float64       \n",
      " 19  buy_elg_vol    float64       \n",
      " 20  sell_elg_vol   float64       \n",
      " 21  net_mf_vol     float64       \n",
      " 22  his_low        float64       \n",
      " 23  his_high       float64       \n",
      " 24  cost_5pct      float64       \n",
      " 25  cost_15pct     float64       \n",
      " 26  cost_50pct     float64       \n",
      " 27  cost_85pct     float64       \n",
      " 28  cost_95pct     float64       \n",
      " 29  weight_avg     float64       \n",
      " 30  winner_rate    float64       \n",
      "dtypes: bool(1), datetime64[ns](1), float64(28), object(1)\n",
      "memory usage: 1.9+ GB\n",
      "None\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "cac01788dac10678",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T12:47:10.527104Z",
     "start_time": "2025-04-03T12:47:00.488715Z"
    }
   },
   "source": [
    "print('industry')\n",
    "industry_df = read_and_merge_h5_data('../../data/industry_data.h5', key='industry_data',\n",
    "                                     columns=['ts_code', 'l2_code', 'in_date'],\n",
    "                                     df=None, on=['ts_code'], join='left')\n",
    "\n",
    "\n",
    "def merge_with_industry_data(df, industry_df):\n",
    "    # 确保日期字段是 datetime 类型\n",
    "    df['trade_date'] = pd.to_datetime(df['trade_date'])\n",
    "    industry_df['in_date'] = pd.to_datetime(industry_df['in_date'])\n",
    "\n",
    "    # 对 industry_df 按 ts_code 和 in_date 排序\n",
    "    industry_df_sorted = industry_df.sort_values(['in_date', 'ts_code'])\n",
    "\n",
    "    # 对原始 df 按 ts_code 和 trade_date 排序\n",
    "    df_sorted = df.sort_values(['trade_date', 'ts_code'])\n",
    "\n",
    "    # 使用 merge_asof 进行向后合并\n",
    "    merged = pd.merge_asof(\n",
    "        df_sorted,\n",
    "        industry_df_sorted,\n",
    "        by='ts_code',  # 按 ts_code 分组\n",
    "        left_on='trade_date',\n",
    "        right_on='in_date',\n",
    "        direction='backward'\n",
    "    )\n",
    "\n",
    "    # 获取每个 ts_code 的最早 in_date 记录\n",
    "    min_in_date_per_ts = (industry_df_sorted\n",
    "    .groupby('ts_code')\n",
    "    .first()\n",
    "    .reset_index()[['ts_code', 'l2_code']])\n",
    "\n",
    "    # 填充未匹配到的记录（trade_date 早于所有 in_date 的情况）\n",
    "    merged['l2_code'] = merged['l2_code'].fillna(\n",
    "        merged['ts_code'].map(min_in_date_per_ts.set_index('ts_code')['l2_code'])\n",
    "    )\n",
    "\n",
    "    # 保留需要的列并重置索引\n",
    "    result = merged.reset_index(drop=True)\n",
    "    return result\n",
    "\n",
    "\n",
    "# 使用示例\n",
    "df = merge_with_industry_data(df, industry_df)\n",
    "# print(mdf[mdf['ts_code'] == '600751.SH'][['ts_code', 'trade_date', 'l2_code']])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "industry\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "id": "c4e9e1d31da6dba6",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "ExecuteTime": {
     "end_time": "2025-04-03T12:47:10.719252Z",
     "start_time": "2025-04-03T12:47:10.541247Z"
    }
   },
   "source": [
    "def calculate_indicators(df):\n",
    "    \"\"\"\n",
    "    计算四个指标：当日涨跌幅、5日移动平均、RSI、MACD。\n",
    "    \"\"\"\n",
    "    df = df.sort_values('trade_date')\n",
    "    df['daily_return'] = (df['close'] - df['pre_close']) / df['pre_close'] * 100\n",
    "    # df['5_day_ma'] = df['close'].rolling(window=5).mean()\n",
    "    delta = df['close'].diff()\n",
    "    gain = delta.where(delta > 0, 0)\n",
    "    loss = -delta.where(delta < 0, 0)\n",
    "    avg_gain = gain.rolling(window=14).mean()\n",
    "    avg_loss = loss.rolling(window=14).mean()\n",
    "    rs = avg_gain / avg_loss\n",
    "    df['RSI'] = 100 - (100 / (1 + rs))\n",
    "\n",
    "    # 计算MACD\n",
    "    ema12 = df['close'].ewm(span=12, adjust=False).mean()\n",
    "    ema26 = df['close'].ewm(span=26, adjust=False).mean()\n",
    "    df['MACD'] = ema12 - ema26\n",
    "    df['Signal_line'] = df['MACD'].ewm(span=9, adjust=False).mean()\n",
    "    df['MACD_hist'] = df['MACD'] - df['Signal_line']\n",
    "\n",
    "    # 4. 情绪因子1：市场上涨比例（Up Ratio）\n",
    "    df['up_ratio'] = df['daily_return'].apply(lambda x: 1 if x > 0 else 0)\n",
    "    df['up_ratio_20d'] = df['up_ratio'].rolling(window=20).mean()  # 过去20天上涨比例\n",
    "\n",
    "    # 5. 情绪因子2：成交量变化率（Volume Change Rate）\n",
    "    df['volume_mean'] = df['vol'].rolling(window=20).mean()  # 过去20天的平均成交量\n",
    "    df['volume_change_rate'] = (df['vol'] - df['volume_mean']) / df['volume_mean'] * 100  # 成交量变化率\n",
    "\n",
    "    # 6. 情绪因子3：波动率（Volatility）\n",
    "    df['volatility'] = df['daily_return'].rolling(window=20).std()  # 过去20天的日收益率标准差\n",
    "\n",
    "    # 7. 情绪因子4：成交额变化率（Amount Change Rate）\n",
    "    df['amount_mean'] = df['amount'].rolling(window=20).mean()  # 过去20天的平均成交额\n",
    "    df['amount_change_rate'] = (df['amount'] - df['amount_mean']) / df['amount_mean'] * 100  # 成交额变化率\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def generate_index_indicators(h5_filename):\n",
    "    df = pd.read_hdf(h5_filename, key='index_data')\n",
    "    df['trade_date'] = pd.to_datetime(df['trade_date'], format='%Y%m%d')\n",
    "    df = df.sort_values('trade_date')\n",
    "\n",
    "    # 计算每个ts_code的相关指标\n",
    "    df_indicators = []\n",
    "    for ts_code in df['ts_code'].unique():\n",
    "        df_index = df[df['ts_code'] == ts_code].copy()\n",
    "        df_index = calculate_indicators(df_index)\n",
    "        df_indicators.append(df_index)\n",
    "\n",
    "    # 合并所有指数的结果\n",
    "    df_all_indicators = pd.concat(df_indicators, ignore_index=True)\n",
    "\n",
    "    # 保留trade_date列，并将同一天的数据按ts_code合并成一行\n",
    "    df_final = df_all_indicators.pivot_table(\n",
    "        index='trade_date',\n",
    "        columns='ts_code',\n",
    "        values=['daily_return', 'RSI', 'MACD', 'Signal_line',\n",
    "                'MACD_hist', 'up_ratio_20d', 'volume_change_rate', 'volatility',\n",
    "                'amount_change_rate', 'amount_mean'],\n",
    "        aggfunc='last'\n",
    "    )\n",
    "\n",
    "    df_final.columns = [f\"{col[1]}_{col[0]}\" for col in df_final.columns]\n",
    "    df_final = df_final.reset_index()\n",
    "\n",
    "    return df_final\n",
    "\n",
    "\n",
    "# 使用函数\n",
    "h5_filename = '../../data/index_data.h5'\n",
    "index_data = generate_index_indicators(h5_filename)\n",
    "index_data = index_data.dropna()\n"
   ],
   "outputs": [],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "id": "a735bc02ceb4d872",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T12:47:10.821169Z",
     "start_time": "2025-04-03T12:47:10.751831Z"
    }
   },
   "source": [
    "import numpy as np\n",
    "import talib\n",
    "\n",
    "def get_rolling_factor(df):\n",
    "    old_columns = df.columns.tolist()[:]\n",
    "\n",
    "\n",
    "    # 按股票和日期排序（如果尚未排序）\n",
    "    df = df.sort_values(by=['ts_code', 'trade_date'])\n",
    "\n",
    "    grouped = df.groupby('ts_code', group_keys=False)\n",
    "\n",
    "    # 提前计算布尔掩码\n",
    "    window = 5\n",
    "    return_threshold = 0.0\n",
    "\n",
    "    df['_is_upside'] = df['pct_chg'] > return_threshold\n",
    "    df['_is_downside'] = df['pct_chg'] < -return_threshold\n",
    "        \n",
    "    # 1. 上行波动率 (20日)\n",
    "    def rolling_upside_volatility(series, _is_upside, window):\n",
    "        # 提取正收益\n",
    "        positive_returns = series.where(_is_upside, np.nan)\n",
    "        # 计算滚动窗口标准差\n",
    "        return positive_returns.rolling(window=window, min_periods=2).std()\n",
    "    \n",
    "    df[f'upside_volatility_{window}'] = grouped.apply(\n",
    "        lambda x: rolling_upside_volatility(x['pct_chg'], x['_is_upside'], window)\n",
    "    ).reset_index(level=0, drop=True)\n",
    "    \n",
    "    # 2. 下行波动率 (20日)\n",
    "    def rolling_downside_volatility(series, _is_downside, window):\n",
    "        # 提取负收益\n",
    "        negative_returns = series.where(_is_downside, np.nan)\n",
    "        # 计算滚动窗口标准差\n",
    "        return negative_returns.rolling(window=window, min_periods=2).std()\n",
    "    \n",
    "    df[f'downside_volatility_{window}'] = grouped.apply(\n",
    "        lambda x: rolling_downside_volatility(x['pct_chg'], x['_is_downside'], window)\n",
    "    ).reset_index(level=0, drop=True)\n",
    "    \n",
    "    # 3. 上行/下行波动率比率 (20日)\n",
    "    df[f'volatility_ratio_{window}'] = df[f'upside_volatility_{window}'] / (df[f'downside_volatility_{window}'] + 1e-8)\n",
    "    \n",
    "    # 4. 上行半方差 (20日)\n",
    "    def rolling_upside_semi_variance(series, _is_upside, window, threshold):\n",
    "        # 提取正收益\n",
    "        positive_returns = series.where(_is_upside, np.nan)\n",
    "        # 计算平方偏差\n",
    "        squared_deviation = (positive_returns - threshold)**2\n",
    "        # 计算滚动窗口均值\n",
    "        return squared_deviation.rolling(window=window, min_periods=2).mean()\n",
    "    \n",
    "    df[f'upside_semi_variance_{window}'] = grouped.apply(\n",
    "        lambda x: rolling_upside_semi_variance(x['pct_chg'], x['_is_upside'], window, return_threshold)\n",
    "    ).reset_index(level=0, drop=True)\n",
    "    \n",
    "    # 5. 下行半方差 (20日)\n",
    "    def rolling_downside_semi_variance(series, _is_downside, window, threshold):\n",
    "        # 提取负收益\n",
    "        negative_returns = series.where(_is_downside, np.nan)\n",
    "        # 计算平方偏差\n",
    "        squared_deviation = (negative_returns - (-threshold))**2\n",
    "        # 计算滚动窗口均值\n",
    "        return squared_deviation.rolling(window=window, min_periods=2).mean()\n",
    "    \n",
    "    df[f'downside_semi_variance_{window}'] = grouped.apply(\n",
    "        lambda x: rolling_downside_semi_variance(x['pct_chg'], x['_is_downside'], window, return_threshold)\n",
    "    ).reset_index(level=0, drop=True)\n",
    "\n",
    "    # 8. 正负收益天数比率 (20日)\n",
    "    df[f'positive_negative_days_ratio_{window}'] = grouped['pct_chg'].rolling(window=window, min_periods=2).apply(lambda x: np.sum(x > 0) / (np.sum(x < 0) + 1e-8)).reset_index(level=0, drop=True)\n",
    "\n",
    "    # 9. 正收益幅度均值 (20日)\n",
    "    def average_positive_return_magnitude(series):\n",
    "        positive_returns = series[series > return_threshold]\n",
    "        if positive_returns.empty:\n",
    "            return 0\n",
    "        return positive_returns.mean()\n",
    "    df[f'avg_positive_return_magnitude_{window}'] = grouped['pct_chg'].rolling(window=window, min_periods=2).apply(average_positive_return_magnitude).reset_index(level=0, drop=True)\n",
    "\n",
    "    # 10. 负收益幅度均值 (20日)\n",
    "    def average_negative_return_magnitude(series):\n",
    "        negative_returns = series[series < -return_threshold]\n",
    "        if negative_returns.empty:\n",
    "            return 0\n",
    "        return np.abs(negative_returns.mean())\n",
    "    df[f'avg_negative_return_magnitude_{window}'] = grouped['pct_chg'].rolling(window=window, min_periods=2).apply(average_negative_return_magnitude).reset_index(level=0, drop=True)\n",
    "\n",
    "    # df[\"gap_next_open\"] = (df[\"open\"].shift(-1) - df[\"close\"]) / df[\"close\"]\n",
    "\n",
    "    df['return_skew'] = grouped['pct_chg'].rolling(window=5).skew().reset_index(0, drop=True)\n",
    "    df['return_kurtosis'] = grouped['pct_chg'].rolling(window=5).kurt().reset_index(0, drop=True)\n",
    "\n",
    "    # 因子 1：短期成交量变化率\n",
    "    df['volume_change_rate'] = (\n",
    "            grouped['vol'].rolling(window=2).mean() /\n",
    "            grouped['vol'].rolling(window=10).mean() - 1\n",
    "    ).reset_index(level=0, drop=True)  # 确保索引对齐\n",
    "\n",
    "    # 因子 2：成交量突破信号\n",
    "    max_volume = grouped['vol'].rolling(window=5).max().reset_index(level=0, drop=True)  # 确保索引对齐\n",
    "    df['cat_volume_breakout'] = (df['vol'] > max_volume)\n",
    "\n",
    "    # 因子 3：换手率均线偏离度\n",
    "    mean_turnover = grouped['turnover_rate'].rolling(window=3).mean().reset_index(level=0, drop=True)\n",
    "    std_turnover = grouped['turnover_rate'].rolling(window=3).std().reset_index(level=0, drop=True)\n",
    "    df['turnover_deviation'] = (df['turnover_rate'] - mean_turnover) / std_turnover\n",
    "\n",
    "    # 因子 4：换手率激增信号\n",
    "    df['cat_turnover_spike'] = (df['turnover_rate'] > mean_turnover + 2 * std_turnover)\n",
    "\n",
    "    # 因子 5：量比均值\n",
    "    df['avg_volume_ratio'] = grouped['volume_ratio'].rolling(window=3).mean().reset_index(level=0, drop=True)\n",
    "\n",
    "    # 因子 6：量比突破信号\n",
    "    max_volume_ratio = grouped['volume_ratio'].rolling(window=5).max().reset_index(level=0, drop=True)\n",
    "    df['cat_volume_ratio_breakout'] = (df['volume_ratio'] > max_volume_ratio)\n",
    "\n",
    "    df['vol_spike'] = grouped.apply(\n",
    "        lambda x: pd.Series(x['vol'].rolling(20).mean(), index=x.index)\n",
    "    )\n",
    "    df['vol_std_5'] = df['vol'].pct_change().rolling(5).std()\n",
    "\n",
    "    # 计算 ATR\n",
    "    df['atr_14'] = grouped.apply(\n",
    "        lambda x: pd.Series(talib.ATR(x['high'].values, x['low'].values, x['close'].values, timeperiod=14),\n",
    "                            index=x.index)\n",
    "    )\n",
    "    df['atr_6'] = grouped.apply(\n",
    "        lambda x: pd.Series(talib.ATR(x['high'].values, x['low'].values, x['close'].values, timeperiod=6),\n",
    "                            index=x.index)\n",
    "    )\n",
    "\n",
    "    # 计算 OBV 及其均线\n",
    "    df['obv'] = grouped.apply(\n",
    "        lambda x: pd.Series(talib.OBV(x['close'].values, x['vol'].values), index=x.index)\n",
    "    )\n",
    "    df['maobv_6'] = grouped.apply(\n",
    "        lambda x: pd.Series(talib.SMA(x['obv'].values, timeperiod=6), index=x.index)\n",
    "    )\n",
    "\n",
    "    df['rsi_3'] = grouped.apply(\n",
    "        lambda x: pd.Series(talib.RSI(x['close'].values, timeperiod=3), index=x.index)\n",
    "    )\n",
    "    # df['rsi_6'] = grouped.apply(\n",
    "    #     lambda x: pd.Series(talib.RSI(x['close'].values, timeperiod=6), index=x.index)\n",
    "    # )\n",
    "    # df['rsi_9'] = grouped.apply(\n",
    "    #     lambda x: pd.Series(talib.RSI(x['close'].values, timeperiod=9), index=x.index)\n",
    "    # )\n",
    "\n",
    "    # 计算 return_10 和 return_20\n",
    "    df['return_5'] = grouped['close'].apply(lambda x: x / x.shift(5) - 1)\n",
    "    # df['return_10'] = grouped['close'].apply(lambda x: x / x.shift(10) - 1)\n",
    "    df['return_20'] = grouped['close'].apply(lambda x: x / x.shift(20) - 1)\n",
    "\n",
    "    # df['avg_close_5'] = grouped['close'].apply(lambda x: x.rolling(window=5).mean() / x)\n",
    "\n",
    "    # 计算标准差指标\n",
    "    df['std_return_5'] = grouped['close'].apply(lambda x: x.pct_change().rolling(window=5).std())\n",
    "    # df['std_return_15'] = grouped['close'].apply(lambda x: x.pct_change().rolling(window=15).std())\n",
    "    # df['std_return_25'] = grouped['close'].apply(lambda x: x.pct_change().rolling(window=25).std())\n",
    "    df['std_return_90'] = grouped['close'].apply(lambda x: x.pct_change().rolling(window=90).std())\n",
    "    df['std_return_90_2'] = grouped['close'].apply(lambda x: x.shift(10).pct_change().rolling(window=90).std())\n",
    "\n",
    "    # 计算 EMA 指标\n",
    "    df['_ema_5'] = grouped['close'].apply(\n",
    "        lambda x: pd.Series(talib.EMA(x.values, timeperiod=5), index=x.index)\n",
    "    )\n",
    "    df['_ema_13'] = grouped['close'].apply(\n",
    "        lambda x: pd.Series(talib.EMA(x.values, timeperiod=13), index=x.index)\n",
    "    )\n",
    "    df['_ema_20'] = grouped['close'].apply(\n",
    "        lambda x: pd.Series(talib.EMA(x.values, timeperiod=20), index=x.index)\n",
    "    )\n",
    "    df['_ema_60'] = grouped['close'].apply(\n",
    "        lambda x: pd.Series(talib.EMA(x.values, timeperiod=60), index=x.index)\n",
    "    )\n",
    "\n",
    "    # 计算 act_factor1, act_factor2, act_factor3, act_factor4\n",
    "    df['act_factor1'] = grouped['_ema_5'].apply(\n",
    "        lambda x: np.arctan((x / x.shift(1) - 1) * 100) * 57.3 / 50\n",
    "    )\n",
    "    df['act_factor2'] = grouped['_ema_13'].apply(\n",
    "        lambda x: np.arctan((x / x.shift(1) - 1) * 100) * 57.3 / 40\n",
    "    )\n",
    "    df['act_factor3'] = grouped['_ema_20'].apply(\n",
    "        lambda x: np.arctan((x / x.shift(1) - 1) * 100) * 57.3 / 21\n",
    "    )\n",
    "    df['act_factor4'] = grouped['_ema_60'].apply(\n",
    "        lambda x: np.arctan((x / x.shift(1) - 1) * 100) * 57.3 / 10\n",
    "    )\n",
    "\n",
    "    # 根据 trade_date 截面计算排名\n",
    "    df['rank_act_factor1'] = df.groupby('trade_date', group_keys=False)['act_factor1'].rank(ascending=False, pct=True)\n",
    "    df['rank_act_factor2'] = df.groupby('trade_date', group_keys=False)['act_factor2'].rank(ascending=False, pct=True)\n",
    "    df['rank_act_factor3'] = df.groupby('trade_date', group_keys=False)['act_factor3'].rank(ascending=False, pct=True)\n",
    "\n",
    "    df['log(circ_mv)'] = np.log(df['circ_mv'])\n",
    "\n",
    "    def rolling_covariance(x, y, window):\n",
    "        return x.rolling(window).cov(y)\n",
    "\n",
    "    def delta(series, period):\n",
    "        return series.diff(period)\n",
    "\n",
    "    def rank(series):\n",
    "        return series.rank(pct=True)\n",
    "\n",
    "    def stddev(series, window):\n",
    "        return series.rolling(window).std()\n",
    "\n",
    "    window_high_volume = 5\n",
    "    window_close_stddev = 20\n",
    "    period_delta = 5\n",
    "    df['cov'] = rolling_covariance(df['high'], df['vol'], window_high_volume)\n",
    "    df['delta_cov'] = delta(df['cov'], period_delta)\n",
    "    df['_rank_stddev'] = rank(stddev(df['close'], window_close_stddev))\n",
    "    df['alpha_22_improved'] = -1 * df['delta_cov'] * df['_rank_stddev']\n",
    "\n",
    "    df['alpha_003'] = np.where(df['high'] != df['low'],\n",
    "                               (df['close'] - df['open']) / (df['high'] - df['low']),\n",
    "                               0)\n",
    "\n",
    "    df['alpha_007'] = grouped.apply(lambda x: x['close'].rolling(5).corr(x['vol'])).reset_index(level=0, drop=True)\n",
    "    df['alpha_007'] = df.groupby('trade_date', group_keys=False)['alpha_007'].rank(ascending=True, pct=True)\n",
    "\n",
    "    df['alpha_013'] = grouped['close'].transform(lambda x: x.rolling(5).sum() - x.rolling(20).sum())\n",
    "    df['alpha_013'] = df.groupby('trade_date', group_keys=False)['alpha_013'].rank(ascending=True, pct=True)\n",
    "\n",
    "    df['cat_up_limit'] = (df['close'] == df['up_limit'])  # 是否涨停（1表示涨停，0表示未涨停）\n",
    "    df['cat_down_limit'] = (df['close'] == df['down_limit'])  # 是否跌停（1表示跌停，0表示未跌停）\n",
    "    df['up_limit_count_10d'] = grouped['cat_up_limit'].rolling(window=10, min_periods=1).sum().reset_index(level=0,\n",
    "                                                                                                           drop=True)\n",
    "    df['down_limit_count_10d'] = grouped['cat_down_limit'].rolling(window=10, min_periods=1).sum().reset_index(level=0,\n",
    "                                                                                                               drop=True)\n",
    "\n",
    "    # 3. 最近连续涨跌停天数\n",
    "    def calculate_consecutive_limits(series):\n",
    "        \"\"\"\n",
    "        计算连续涨停/跌停天数。\n",
    "        \"\"\"\n",
    "        consecutive_up = series * (series.groupby((series != series.shift()).cumsum()).cumcount() + 1)\n",
    "        consecutive_down = series * (series.groupby((series != series.shift()).cumsum()).cumcount() + 1)\n",
    "        return consecutive_up, consecutive_down\n",
    "\n",
    "    # 连续涨停天数\n",
    "    df['consecutive_up_limit'] = grouped['cat_up_limit'].apply(\n",
    "        lambda x: calculate_consecutive_limits(x)[0]\n",
    "    ).reset_index(level=0, drop=True)\n",
    "\n",
    "    df['vol_break'] = np.where((df['close'] > df['cost_85pct']) & (df['volume_ratio'] > 2), 1, 0)\n",
    "\n",
    "    df['weight_roc5'] = grouped['weight_avg'].apply(lambda x: x.pct_change(5))\n",
    "\n",
    "    def rolling_corr(group):\n",
    "        roc_close = group['close'].pct_change()\n",
    "        roc_weight = group['weight_avg'].pct_change()\n",
    "        return roc_close.rolling(10).corr(roc_weight)\n",
    "\n",
    "    df['price_cost_divergence'] = grouped.apply(rolling_corr)\n",
    "\n",
    "    df['smallcap_concentration'] = (1 / df['log(circ_mv)']) * (df['cost_85pct'] - df['cost_15pct'])\n",
    "\n",
    "    # 16. 筹码稳定性指数 (20日波动率)\n",
    "    df['weight_std20'] = grouped['weight_avg'].apply(lambda x: x.rolling(20).std())\n",
    "    df['cost_stability'] = df['weight_std20'] / grouped['weight_avg'].transform(lambda x: x.rolling(20).mean())\n",
    "\n",
    "    # 17. 成本区间突破标记\n",
    "    df['high_cost_break_days'] = grouped.apply(lambda g: g['close'].gt(g['cost_95pct']).rolling(5).sum())\n",
    "\n",
    "    # 20. 筹码-流动性风险\n",
    "    df['liquidity_risk'] = (df['cost_95pct'] - df['cost_5pct']) * (\n",
    "            1 / grouped['vol'].transform(lambda x: x.rolling(10).mean()))\n",
    "\n",
    "    # 7. 市值波动率因子\n",
    "    df['turnover_std'] = grouped['turnover_rate'].rolling(window=20).std().reset_index(level=0, drop=True)\n",
    "    df['mv_volatility'] = grouped.apply(lambda x: x['turnover_std'] / x['log(circ_mv)']).reset_index(level=0, drop=True)\n",
    "\n",
    "    # 8. 市值成长性因子\n",
    "    df['volume_growth'] = grouped['vol'].pct_change(periods=20).reset_index(level=0, drop=True)\n",
    "    df['mv_growth'] = grouped.apply(lambda x: x['volume_growth'] / x['log(circ_mv)']).reset_index(level=0, drop=True)\n",
    "\n",
    "    df[\"ar\"] = df[\"high\"].div(df[\"open\"]).rolling(3).sum() / df[\"open\"].div(df[\"low\"]).rolling(3).sum() * 100\n",
    "    # 计算 BR 指标\n",
    "    df[\"pre_close\"] = df[\"close\"].shift(1)\n",
    "    df[\"br_up\"] = (df[\"high\"] - df[\"pre_close\"]).clip(lower=0)\n",
    "    df[\"br_down\"] = (df[\"pre_close\"] - df[\"low\"]).clip(lower=0)\n",
    "    df[\"br\"] = df[\"br_up\"].rolling(3).sum() / df[\"br_down\"].rolling(3).sum() * 100\n",
    "    df['arbr'] = df['ar'] - df['br']\n",
    "    df.drop(columns=[\"pre_close\", \"br_up\", \"br_down\", 'ar', 'br'], inplace=True)\n",
    "\n",
    "    df.drop(columns=['weight_std20'], inplace=True, errors='ignore')\n",
    "    new_columns = [col for col in df.columns.tolist()[:] if col not in old_columns]\n",
    "\n",
    "    return df, new_columns\n",
    "\n",
    "\n",
    "def get_simple_factor(df):\n",
    "    old_columns = df.columns.tolist()[:]\n",
    "    df = df.sort_values(by=['ts_code', 'trade_date'])\n",
    "\n",
    "    alpha = 0.5\n",
    "    df['momentum_factor'] = df['volume_change_rate'] + alpha * df['turnover_deviation']\n",
    "    df['resonance_factor'] = df['volume_ratio'] * df['pct_chg']\n",
    "    df['log_close'] = np.log(df['close'])\n",
    "\n",
    "    df['cat_vol_spike'] = df['vol'] > 2 * df['vol_spike']\n",
    "\n",
    "    df['up'] = (df['high'] - df[['close', 'open']].max(axis=1)) / df['close']\n",
    "    df['down'] = (df[['close', 'open']].min(axis=1) - df['low']) / df['close']\n",
    "\n",
    "    df['obv-maobv_6'] = df['obv'] - df['maobv_6']\n",
    "\n",
    "    # 计算比值指标\n",
    "    df['std_return_5 / std_return_90'] = df['std_return_5'] / df['std_return_90']\n",
    "    # df['std_return_5 / std_return_25'] = df['std_return_5'] / df['std_return_25']\n",
    "\n",
    "    # 计算标准差差值\n",
    "    df['std_return_90 - std_return_90_2'] = df['std_return_90'] - df['std_return_90_2']\n",
    "\n",
    "    # df['cat_af1'] = df['act_factor1'] > 0\n",
    "    df['cat_af2'] = df['act_factor2'] > df['act_factor1']\n",
    "    df['cat_af3'] = df['act_factor3'] > df['act_factor2']\n",
    "    df['cat_af4'] = df['act_factor4'] > df['act_factor3']\n",
    "\n",
    "    # 计算 act_factor5 和 act_factor6\n",
    "    df['act_factor5'] = df['act_factor1'] + df['act_factor2'] + df['act_factor3'] + df['act_factor4']\n",
    "    df['act_factor6'] = (df['act_factor1'] - df['act_factor2']) / np.sqrt(\n",
    "        df['act_factor1'] ** 2 + df['act_factor2'] ** 2)\n",
    "\n",
    "    df['active_buy_volume_large'] = df['buy_lg_vol'] / df['net_mf_vol']\n",
    "    df['active_buy_volume_big'] = df['buy_elg_vol'] / df['net_mf_vol']\n",
    "    df['active_buy_volume_small'] = df['buy_sm_vol'] / df['net_mf_vol']\n",
    "\n",
    "    df['buy_lg_vol_minus_sell_lg_vol'] = (df['buy_lg_vol'] - df['sell_lg_vol']) / df['net_mf_vol']\n",
    "    df['buy_elg_vol_minus_sell_elg_vol'] = (df['buy_elg_vol'] - df['sell_elg_vol']) / df['net_mf_vol']\n",
    "\n",
    "    df['log(circ_mv)'] = np.log(df['circ_mv'])\n",
    "\n",
    "    df['ctrl_strength'] = (df['cost_85pct'] - df['cost_15pct']) / (df['his_high'] - df['his_low'])\n",
    "\n",
    "    df['low_cost_dev'] = (df['close'] - df['cost_5pct']) / (df['cost_50pct'] - df['cost_5pct'])\n",
    "\n",
    "    df['asymmetry'] = (df['cost_95pct'] - df['cost_50pct']) / (df['cost_50pct'] - df['cost_5pct'])\n",
    "\n",
    "    df['lock_factor'] = df['turnover_rate'] * (\n",
    "            1 - (df['cost_95pct'] - df['cost_5pct']) / (df['his_high'] - df['his_low']))\n",
    "\n",
    "    df['cat_vol_break'] = (df['close'] > df['cost_85pct']) & (df['volume_ratio'] > 2)\n",
    "\n",
    "    df['cost_atr_adj'] = (df['cost_95pct'] - df['cost_5pct']) / df['atr_14']\n",
    "\n",
    "    # 12. 小盘股筹码集中度\n",
    "    df['smallcap_concentration'] = (1 / df['log(circ_mv)']) * (df['cost_85pct'] - df['cost_15pct'])\n",
    "\n",
    "    df['cat_golden_resonance'] = ((df['close'] > df['weight_avg']) &\n",
    "                                  (df['volume_ratio'] > 1.5) &\n",
    "                                  (df['winner_rate'] > 0.7))\n",
    "\n",
    "    df['mv_turnover_ratio'] = df['turnover_rate'] / df['log(circ_mv)']\n",
    "\n",
    "    df['mv_adjusted_volume'] = df['vol'] / df['log(circ_mv)']\n",
    "\n",
    "    df['mv_weighted_turnover'] = df['turnover_rate'] * (1 / df['log(circ_mv)'])\n",
    "\n",
    "    df['nonlinear_mv_volume'] = df['vol'] / df['log(circ_mv)']\n",
    "\n",
    "    df['mv_volume_ratio'] = df['volume_ratio'] / df['log(circ_mv)']\n",
    "\n",
    "    df['mv_momentum'] = df['turnover_rate'] * df['volume_ratio'] / df['log(circ_mv)']\n",
    "\n",
    "    drop_columns = [col for col in df.columns if col.startswith('_')]\n",
    "    df.drop(columns=drop_columns, inplace=True, errors='ignore')\n",
    "\n",
    "    new_columns = [col for col in df.columns.tolist()[:] if col not in old_columns]\n",
    "    return df, new_columns\n"
   ],
   "outputs": [],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "id": "53f86ddc0677a6d7",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "scrolled": true,
    "ExecuteTime": {
     "end_time": "2025-04-03T12:47:15.944254Z",
     "start_time": "2025-04-03T12:47:10.826179Z"
    }
   },
   "source": [
    "from utils.factor import get_act_factor\n",
    "\n",
    "\n",
    "def read_industry_data(h5_filename):\n",
    "    # 读取 H5 文件中所有的行业数据\n",
    "    industry_data = pd.read_hdf(h5_filename, key='sw_daily', columns=[\n",
    "        'ts_code', 'trade_date', 'open', 'close', 'high', 'low', 'pe', 'pb', 'vol'\n",
    "    ])  # 假设 H5 文件的键是 'industry_data'\n",
    "    industry_data = industry_data.sort_values(by=['ts_code', 'trade_date'])\n",
    "    industry_data = industry_data.reindex()\n",
    "    industry_data['trade_date'] = pd.to_datetime(industry_data['trade_date'], format='%Y%m%d')\n",
    "\n",
    "    grouped = industry_data.groupby('ts_code', group_keys=False)\n",
    "    industry_data['obv'] = grouped.apply(\n",
    "        lambda x: pd.Series(talib.OBV(x['close'].values, x['vol'].values), index=x.index)\n",
    "    )\n",
    "    industry_data['return_5'] = grouped['close'].apply(lambda x: x / x.shift(5) - 1)\n",
    "    industry_data['return_20'] = grouped['close'].apply(lambda x: x / x.shift(20) - 1)\n",
    "\n",
    "    industry_data = get_act_factor(industry_data, cat=False)\n",
    "    industry_data = industry_data.sort_values(by=['trade_date', 'ts_code'])\n",
    "\n",
    "    # # 计算每天每个 ts_code 的因子和当天所有 ts_code 的中位数的偏差\n",
    "    # factor_columns = ['obv', 'return_5', 'return_20', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4']  # 因子列\n",
    "    # \n",
    "    # for factor in factor_columns:\n",
    "    #     if factor in industry_data.columns:\n",
    "    #         # 计算每天每个 ts_code 的因子值与当天所有 ts_code 的中位数的偏差\n",
    "    #         industry_data[f'{factor}_deviation'] = industry_data.groupby('trade_date')[factor].transform(\n",
    "    #             lambda x: x - x.mean())\n",
    "\n",
    "    industry_data['return_5_percentile'] = industry_data.groupby('trade_date')['return_5'].transform(\n",
    "        lambda x: x.rank(pct=True))\n",
    "    industry_data['return_20_percentile'] = industry_data.groupby('trade_date')['return_20'].transform(\n",
    "        lambda x: x.rank(pct=True))\n",
    "    industry_data = industry_data.drop(columns=['open', 'close', 'high', 'low', 'pe', 'pb', 'vol'])\n",
    "\n",
    "    industry_data = industry_data.rename(\n",
    "        columns={col: f'industry_{col}' for col in industry_data.columns if col not in ['ts_code', 'trade_date']})\n",
    "\n",
    "    industry_data = industry_data.rename(columns={'ts_code': 'cat_l2_code'})\n",
    "    return industry_data\n",
    "\n",
    "\n",
    "industry_df = read_industry_data('../../data/sw_daily.h5')\n"
   ],
   "outputs": [],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "id": "dbe2fd8021b9417f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T12:47:15.969344Z",
     "start_time": "2025-04-03T12:47:15.963327Z"
    }
   },
   "source": [
    "origin_columns = df.columns.tolist()\n",
    "origin_columns = [col for col in origin_columns if\n",
    "                  col not in ['turnover_rate', 'pe_ttm', 'volume_ratio', 'vol', 'pct_chg', 'l2_code', 'winner_rate']]\n",
    "origin_columns = [col for col in origin_columns if col not in index_data.columns]\n",
    "origin_columns = [col for col in origin_columns if 'cyq' not in col]\n",
    "print(origin_columns)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ts_code', 'open', 'close', 'high', 'low', 'circ_mv', 'is_st', 'up_limit', 'down_limit', 'buy_sm_vol', 'sell_sm_vol', 'buy_lg_vol', 'sell_lg_vol', 'buy_elg_vol', 'sell_elg_vol', 'net_mf_vol', 'his_low', 'his_high', 'cost_5pct', 'cost_15pct', 'cost_50pct', 'cost_85pct', 'cost_95pct', 'weight_avg', 'in_date']\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "id": "85c3e3d0235ffffa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T12:47:16.089879Z",
     "start_time": "2025-04-03T12:47:15.990101Z"
    }
   },
   "source": [
    "print(df[df['is_st']][['ts_code', 'trade_date', 'is_st']])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           ts_code trade_date  is_st\n",
      "29       000037.SZ 2017-01-03   True\n",
      "72       000408.SZ 2017-01-03   True\n",
      "95       000504.SZ 2017-01-03   True\n",
      "96       000505.SZ 2017-01-03   True\n",
      "101      000511.SZ 2017-01-03   True\n",
      "...            ...        ...    ...\n",
      "8476334  603869.SH 2025-03-28   True\n",
      "8476339  603879.SH 2025-03-28   True\n",
      "8476386  603959.SH 2025-03-28   True\n",
      "8476769  688282.SH 2025-03-28   True\n",
      "8476773  688287.SH 2025-03-28   True\n",
      "\n",
      "[192712 rows x 3 columns]\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "id": "92d84ce15a562ec6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T13:08:01.612695Z",
     "start_time": "2025-04-03T12:47:16.121802Z"
    }
   },
   "source": [
    "def filter_data(df):\n",
    "    # df = df.groupby('trade_date').apply(lambda x: x.nlargest(1000, 'act_factor1'))\n",
    "    df = df[~df['is_st']]\n",
    "    df = df[~df['ts_code'].str.endswith('BJ')]\n",
    "    df = df[~df['ts_code'].str.startswith('30')]\n",
    "    df = df[~df['ts_code'].str.startswith('68')]\n",
    "    df = df[~df['ts_code'].str.startswith('8')]\n",
    "    df = df[df['trade_date'] >= '20180101']\n",
    "    if 'in_date' in df.columns:\n",
    "        df = df.drop(columns=['in_date'])\n",
    "    df = df.reset_index(drop=True)\n",
    "    return df\n",
    "\n",
    "\n",
    "df = filter_data(df)\n",
    "# df = get_technical_factor(df)\n",
    "# df = get_act_factor(df)\n",
    "# df = get_money_flow_factor(df)\n",
    "# df = get_alpha_factor(df)\n",
    "# df = get_limit_factor(df)\n",
    "# df = get_cyp_perf_factor(df)\n",
    "# df = get_mv_factors(df)\n",
    "df, _ = get_rolling_factor(df)\n",
    "df, _ = get_simple_factor(df)\n",
    "# df = df.merge(industry_df, on=['l2_code', 'trade_date'], how='left')\n",
    "df = df.rename(columns={'l2_code': 'cat_l2_code'})\n",
    "# df = df.merge(index_data, on='trade_date', how='left')\n",
    "\n",
    "print(df.info())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 5118212 entries, 0 to 5118211\n",
      "Columns: 123 entries, ts_code to mv_momentum\n",
      "dtypes: bool(12), datetime64[ns](1), float64(106), int32(1), int64(1), object(2)\n",
      "memory usage: 4.3+ GB\n",
      "None\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "id": "b87b938028afa206",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T13:08:03.658725Z",
     "start_time": "2025-04-03T13:08:02.469611Z"
    }
   },
   "source": [
    "from scipy.stats import ks_2samp, wasserstein_distance\n",
    "\n",
    "\n",
    "def remove_shifted_features(train_data, test_data, feature_columns, ks_threshold=0.05, wasserstein_threshold=0.1,\n",
    "                            importance_threshold=0.05):\n",
    "    dropped_features = []\n",
    "\n",
    "    # **统计数据漂移**\n",
    "    numeric_columns = train_data.select_dtypes(include=['float64', 'int64']).columns\n",
    "    numeric_columns = [col for col in numeric_columns if col in feature_columns]\n",
    "    for feature in numeric_columns:\n",
    "        ks_stat, p_value = ks_2samp(train_data[feature], test_data[feature])\n",
    "        wasserstein_dist = wasserstein_distance(train_data[feature], test_data[feature])\n",
    "\n",
    "        if p_value < ks_threshold or wasserstein_dist > wasserstein_threshold:\n",
    "            dropped_features.append(feature)\n",
    "\n",
    "    print(f\"检测到 {len(dropped_features)} 个可能漂移的特征: {dropped_features}\")\n",
    "\n",
    "    # **应用阈值进行最终筛选**\n",
    "    filtered_features = [f for f in feature_columns if f not in dropped_features]\n",
    "\n",
    "    return filtered_features, dropped_features\n",
    "\n"
   ],
   "outputs": [],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "id": "f4f16d63ad18d1bc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T13:08:03.670700Z",
     "start_time": "2025-04-03T13:08:03.665739Z"
    }
   },
   "source": [
    "def create_deviation_within_dates(df, feature_columns):\n",
    "    groupby_col = 'cat_l2_code'  # 使用 trade_date 进行分组\n",
    "    new_columns = {}\n",
    "    ret_feature_columns = feature_columns[:]\n",
    "\n",
    "    # 自动选择所有数值型特征\n",
    "    num_features = [col for col in feature_columns if 'cat' not in col and 'index' not in col]\n",
    "\n",
    "    # num_features = ['vol', 'pct_chg', 'turnover_rate', 'volume_ratio', 'cat_vol_spike', 'obv', 'maobv_6', 'return_5', 'return_10', 'return_20', 'std_return_5', 'std_return_15', 'std_return_90', 'std_return_90_2', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4', 'act_factor5', 'act_factor6', 'rank_act_factor1', 'rank_act_factor2', 'rank_act_factor3', 'active_buy_volume_large', 'active_buy_volume_big', 'active_buy_volume_small', 'alpha_022', 'alpha_003', 'alpha_007', 'alpha_013']\n",
    "    num_features = [col for col in num_features if 'cat' not in col and 'industry' not in col]\n",
    "    num_features = [col for col in num_features if 'limit' not in col]\n",
    "    num_features = [col for col in num_features if 'cyq' not in col]\n",
    "\n",
    "    # 遍历所有数值型特征\n",
    "    for feature in num_features:\n",
    "        if feature == 'trade_date':  # 不需要对 'trade_date' 计算偏差\n",
    "            continue\n",
    "\n",
    "        # grouped_mean = df.groupby(['trade_date'])[feature].transform('mean')\n",
    "        # deviation_col_name = f'deviation_mean_{feature}'\n",
    "        # new_columns[deviation_col_name] = df[feature] - grouped_mean\n",
    "        # ret_feature_columns.append(deviation_col_name)\n",
    "\n",
    "        grouped_mean = df.groupby(['trade_date', groupby_col])[feature].transform('mean')\n",
    "        deviation_col_name = f'deviation_mean_{feature}'\n",
    "        new_columns[deviation_col_name] = df[feature] - grouped_mean\n",
    "        ret_feature_columns.append(deviation_col_name)\n",
    "\n",
    "    # 将新计算的偏差特征与原始 DataFrame 合并\n",
    "    df = pd.concat([df, pd.DataFrame(new_columns)], axis=1)\n",
    "\n",
    "    # for feature in ['obv', 'return_20', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4']:\n",
    "    #     df[f'deviation_industry_{feature}'] = df[feature] - df[f'industry_{feature}']\n",
    "\n",
    "    return df, ret_feature_columns\n"
   ],
   "outputs": [],
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "id": "40e6b68a91b30c79",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T13:08:04.694262Z",
     "start_time": "2025-04-03T13:08:03.694904Z"
    }
   },
   "source": [
    "import pandas as pd\n",
    "\n",
    "\n",
    "def remove_outliers_label_percentile(label: pd.Series, lower_percentile: float = 0.01, upper_percentile: float = 0.99,\n",
    "                                     log=True):\n",
    "    if not (0 <= lower_percentile < upper_percentile <= 1):\n",
    "        raise ValueError(\"Percentile values must satisfy 0 <= lower_percentile < upper_percentile <= 1.\")\n",
    "\n",
    "    # Calculate lower and upper bounds based on percentiles\n",
    "    lower_bound = label.quantile(lower_percentile)\n",
    "    upper_bound = label.quantile(upper_percentile)\n",
    "\n",
    "    # Filter out values outside the bounds\n",
    "    filtered_label = label[(label >= lower_bound) & (label <= upper_bound)]\n",
    "\n",
    "    # Print the number of removed outliers\n",
    "    if log:\n",
    "        print(f\"Removed {len(label) - len(filtered_label)} outliers.\")\n",
    "    return filtered_label\n",
    "\n",
    "\n",
    "def calculate_risk_adjusted_target(df, days=5):\n",
    "    df = df.sort_values(by=['ts_code', 'trade_date'])\n",
    "\n",
    "    df['future_close'] = df.groupby('ts_code')['close'].shift(-days)\n",
    "    df['future_open'] = df.groupby('ts_code')['open'].shift(-1)\n",
    "    df['future_return'] = (df['future_close'] - df['future_open']) / df['future_open']\n",
    "\n",
    "    df['future_volatility'] = df.groupby('ts_code')['future_return'].rolling(days, min_periods=1).std().reset_index(\n",
    "        level=0, drop=True)\n",
    "    sharpe_ratio = df['future_return'] * df['future_volatility']\n",
    "    sharpe_ratio.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "\n",
    "    return sharpe_ratio\n",
    "\n",
    "\n",
    "def calculate_score(df, days=5, lambda_param=1.0):\n",
    "    def calculate_max_drawdown(prices):\n",
    "        peak = prices.iloc[0]  # 初始化峰值\n",
    "        max_drawdown = 0  # 初始化最大回撤\n",
    "\n",
    "        for price in prices:\n",
    "            if price > peak:\n",
    "                peak = price  # 更新峰值\n",
    "            else:\n",
    "                drawdown = (peak - price) / peak  # 计算当前回撤\n",
    "                max_drawdown = max(max_drawdown, drawdown)  # 更新最大回撤\n",
    "\n",
    "        return max_drawdown\n",
    "\n",
    "    def compute_stock_score(stock_df):\n",
    "        stock_df = stock_df.sort_values(by=['trade_date'])\n",
    "        future_return = stock_df['future_return']\n",
    "        # 使用已有的 pct_chg 字段计算波动率\n",
    "        volatility = stock_df['pct_chg'].rolling(days).std().shift(-days)\n",
    "        max_drawdown = stock_df['close'].rolling(days).apply(calculate_max_drawdown, raw=False).shift(-days)\n",
    "        score = future_return - lambda_param * max_drawdown\n",
    "        return score\n",
    "\n",
    "    # # 确保 DataFrame 按照股票代码和交易日期排序\n",
    "    # df = df.sort_values(by=['ts_code', 'trade_date'])\n",
    "\n",
    "    # 对每个股票分别计算 score\n",
    "    df['score'] = df.groupby('ts_code').apply(compute_stock_score).reset_index(level=0, drop=True)\n",
    "\n",
    "    return df['score']\n",
    "\n",
    "\n",
    "def remove_highly_correlated_features(df, feature_columns, threshold=0.9):\n",
    "    numeric_features = df[feature_columns].select_dtypes(include=[np.number]).columns.tolist()\n",
    "    if not numeric_features:\n",
    "        raise ValueError(\"No numeric features found in the provided data.\")\n",
    "\n",
    "    corr_matrix = df[numeric_features].corr().abs()\n",
    "    upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n",
    "    to_drop = [column for column in upper.columns if any(upper[column] > threshold)]\n",
    "    remaining_features = [col for col in feature_columns if col not in to_drop\n",
    "                          or 'act' in col or 'af' in col]\n",
    "    return remaining_features\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "\n",
    "def cross_sectional_standardization(df, features):\n",
    "    df_sorted = df.sort_values(by='trade_date')  # 按时间排序\n",
    "    df_standardized = df_sorted.copy()\n",
    "\n",
    "    for date in df_sorted['trade_date'].unique():\n",
    "        # 获取当前时间点的数据\n",
    "        current_data = df_standardized[df_standardized['trade_date'] == date]\n",
    "\n",
    "        # 只对指定特征进行标准化\n",
    "        scaler = StandardScaler()\n",
    "        standardized_values = scaler.fit_transform(current_data[features])\n",
    "\n",
    "        # 将标准化结果重新赋值回去\n",
    "        df_standardized.loc[df_standardized['trade_date'] == date, features] = standardized_values\n",
    "\n",
    "    return df_standardized\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import statsmodels.api as sm\n",
    "\n",
    "from concurrent.futures import ProcessPoolExecutor\n",
    "\n",
    "\n",
    "def neutralize_manual(df, features, industry_col, mkt_cap_col):\n",
    "    \"\"\" 手动实现简单回归以提升速度 \"\"\"\n",
    "\n",
    "    for col in features:\n",
    "        residuals = []\n",
    "        for _, group in df.groupby(industry_col):\n",
    "            if len(group) > 1:\n",
    "                x = np.log(group[mkt_cap_col])  # 市值对数\n",
    "                y = group[col]  # 因子值\n",
    "                beta = np.cov(y, x)[0, 1] / np.var(x)  # 计算斜率\n",
    "                alpha = np.mean(y) - beta * np.mean(x)  # 计算截距\n",
    "                resid = y - (alpha + beta * x)  # 计算残差\n",
    "                residuals.extend(resid)\n",
    "            else:\n",
    "                residuals.extend(group[col])  # 样本不足时保留原值\n",
    "\n",
    "        df[col] = residuals\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "import gc\n",
    "\n",
    "gc.collect()\n",
    "\n",
    "\n",
    "def mad_filter(df, features, n=3):\n",
    "    for col in features:\n",
    "        median = df[col].median()\n",
    "        mad = np.median(np.abs(df[col] - median))\n",
    "        upper = median + n * mad\n",
    "        lower = median - n * mad\n",
    "        df[col] = np.clip(df[col], lower, upper)  # 截断极值\n",
    "    return df\n",
    "\n",
    "\n",
    "def percentile_filter(df, features, lower_percentile=0.01, upper_percentile=0.99):\n",
    "    for col in features:\n",
    "        # 按日期分组计算上下百分位数\n",
    "        lower_bound = df.groupby('trade_date')[col].transform(\n",
    "            lambda x: x.quantile(lower_percentile)\n",
    "        )\n",
    "        upper_bound = df.groupby('trade_date')[col].transform(\n",
    "            lambda x: x.quantile(upper_percentile)\n",
    "        )\n",
    "        # 截断超出范围的值\n",
    "        df[col] = np.clip(df[col], lower_bound, upper_bound)\n",
    "    return df\n",
    "\n",
    "\n",
    "from scipy.stats import iqr\n",
    "\n",
    "\n",
    "def iqr_filter(df, features):\n",
    "    for col in features:\n",
    "        df[col] = df.groupby('trade_date')[col].transform(\n",
    "            lambda x: (x - x.median()) / iqr(x) if iqr(x) != 0 else x\n",
    "        )\n",
    "    return df\n",
    "\n",
    "\n",
    "def quantile_filter(df, features, lower_quantile=0.01, upper_quantile=0.99, window=60):\n",
    "    df = df.copy()\n",
    "    for col in features:\n",
    "        # 计算 rolling 统计量，需要按日期进行 groupby\n",
    "        rolling_lower = df.groupby('trade_date')[col].transform(lambda x: x.rolling(window=min(len(x), window)).quantile(lower_quantile))\n",
    "        rolling_upper = df.groupby('trade_date')[col].transform(lambda x: x.rolling(window=min(len(x), window)).quantile(upper_quantile))\n",
    "\n",
    "        # 对数据进行裁剪\n",
    "        df[col] = np.clip(df[col], rolling_lower, rolling_upper)\n",
    "    \n",
    "    return df\n"
   ],
   "outputs": [],
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "id": "47c12bb34062ae7a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T14:57:50.841165Z",
     "start_time": "2025-04-03T14:49:25.889057Z"
    }
   },
   "source": [
    "days = 2\n",
    "validation_days = 120\n",
    "\n",
    "import gc\n",
    "\n",
    "gc.collect()\n",
    "\n",
    "df = df.sort_values(by=['ts_code', 'trade_date'])\n",
    "# df['future_return'] = df.groupby('ts_code', group_keys=False)['close'].apply(lambda x: x.shift(-days) / x - 1)\n",
    "df['future_return'] = (df.groupby('ts_code')['close'].shift(-days) - df.groupby('ts_code')['open'].shift(-1)) / \\\n",
    "                      df.groupby('ts_code')['open'].shift(-1)\n",
    "df['future_volatility'] = (\n",
    "    df.groupby('ts_code')['pct_chg']\n",
    "    .transform(lambda x: x.rolling(days).std().shift(-days))\n",
    ")\n",
    "\n",
    "# df['future_score'] = (\n",
    "#         0.7 * df['future_return']\n",
    "#         * 0.3 * df['future_volatility']\n",
    "# )\n",
    "df['future_score'] = calculate_score(df, days=2, lambda_param=0.3)\n",
    "\n",
    "filter_index = df['future_return'].between(df['future_return'].quantile(0.01), df['future_return'].quantile(0.99))\n",
    "filter_index = df['future_volatility'].between(df['future_volatility'].quantile(0.01),\n",
    "                                               df['future_volatility'].quantile(0.99)) | filter_index\n",
    "\n",
    "# df['label'] = df.groupby('trade_date', group_keys=False)['future_volatility'].transform(\n",
    "#     lambda x: pd.qcut(x, q=30, labels=False, duplicates='drop')\n",
    "# )\n",
    "\n",
    "df['label'] = df.groupby('trade_date', group_keys=False)['future_score'].transform(\n",
    "    lambda x: pd.qcut(x, q=20, labels=False, duplicates='drop')\n",
    ")\n",
    "\n",
    "\n",
    "# df['1_score'] = df.groupby('ts_code', group_keys=False)['future_score'].shift(days)\n",
    "# df['2_score'] = df.groupby('ts_code', group_keys=False)['future_score'].shift(1 + days)\n",
    "# df['3_score'] = df.groupby('ts_code', group_keys=False)['future_score'].shift(3 + days - 1)\n",
    "\n",
    "def symmetric_log_transform(values):\n",
    "    return np.sign(values) * np.log1p(np.abs(values))\n",
    "\n",
    "\n",
    "train_data = df[filter_index & (df['trade_date'] <= '2023-08-01') & (df['trade_date'] >= '2000-01-01')]\n",
    "test_data = df[(df['trade_date'] >= '2023-08-01')]\n",
    "\n",
    "\n",
    "def select_pre_zt_stocks_dynamic(stock_df):\n",
    "    # 排序数据\n",
    "    stock_df = stock_df.sort_values(by=['trade_date', 'ts_code'])\n",
    "    stock_df = stock_df.groupby('trade_date', group_keys=False).apply(\n",
    "        lambda x: x.nlargest(1000, 'return_20')\n",
    "    )\n",
    "\n",
    "    return stock_df\n",
    "\n",
    "\n",
    "train_data = select_pre_zt_stocks_dynamic(train_data)\n",
    "test_data = select_pre_zt_stocks_dynamic(test_data)\n",
    "\n",
    "train_data, _ = get_simple_factor(train_data)\n",
    "test_data, _ = get_simple_factor(test_data)\n",
    "\n",
    "# train_data['label'] = train_data.groupby('trade_date', group_keys=False)['future_score'].transform(\n",
    "#     lambda x: pd.qcut(x, q=50, labels=False, duplicates='drop')\n",
    "# )\n",
    "# test_data['label'] = test_data.groupby('trade_date', group_keys=False)['future_score'].transform(\n",
    "#     lambda x: pd.qcut(x, q=50, labels=False, duplicates='drop')\n",
    "# )\n",
    "\n",
    "industry_df = industry_df.sort_values(by=['trade_date'])\n",
    "index_data = index_data.sort_values(by=['trade_date'])\n",
    "\n",
    "train_data = train_data.merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
    "# train_data = train_data.merge(index_data, on='trade_date', how='left')\n",
    "test_data = test_data.merge(industry_df, on=['cat_l2_code', 'trade_date'], how='left')\n",
    "# test_data = test_data.merge(index_data, on='trade_date', how='left')\n",
    "\n",
    "train_data, test_data = train_data.replace([np.inf, -np.inf], np.nan), test_data.replace([np.inf, -np.inf], np.nan)\n",
    "\n",
    "# feature_columns_new = feature_columns[:]\n",
    "# train_data, _ = create_deviation_within_dates(train_data, feature_columns)\n",
    "# test_data, _ = create_deviation_within_dates(test_data, feature_columns)\n",
    "\n",
    "feature_columns = [col for col in train_data.columns if col in train_data.columns]\n",
    "feature_columns = [col for col in feature_columns if col not in ['trade_date',\n",
    "                                                                 'ts_code',\n",
    "                                                                 'label']]\n",
    "feature_columns = [col for col in feature_columns if 'future' not in col]\n",
    "feature_columns = [col for col in feature_columns if 'label' not in col]\n",
    "feature_columns = [col for col in feature_columns if 'score' not in col]\n",
    "feature_columns = [col for col in feature_columns if 'gen' not in col]\n",
    "feature_columns = [col for col in feature_columns if 'cat_l2_code' not in col]\n",
    "feature_columns = [col for col in feature_columns if col not in origin_columns]\n",
    "feature_columns = [col for col in feature_columns if not col.startswith('_')]\n",
    "print(f'feature_columns size: {len(feature_columns)}')\n",
    "\n",
    "numeric_columns = train_data.select_dtypes(include=['float64', 'int64']).columns\n",
    "numeric_columns = [col for col in numeric_columns if col in feature_columns]\n",
    "print('去极值')\n",
    "train_data = quantile_filter(train_data, numeric_columns)  # 去极值\n",
    "# print('中性化')\n",
    "# train_data = neutralize_manual(train_data, numeric_columns, industry_col='cat_l2_code', mkt_cap_col='log(circ_mv)')  # 中性化\n",
    "print('去极值')\n",
    "test_data = quantile_filter(test_data, numeric_columns)  # 去极值\n",
    "# print('中性化')\n",
    "# test_data = neutralize_manual(test_data, numeric_columns, industry_col='cat_l2_code', mkt_cap_col='log(circ_mv)')\n",
    "all_dates = train_data['trade_date'].unique()  # 获取所有唯一的 trade_date\n",
    "split_date = all_dates[-validation_days]  # 划分点为倒数第 validation_days 天\n",
    "train_data_split = train_data[train_data['trade_date'] < split_date]  # 训练集\n",
    "val_data_split = train_data[train_data['trade_date'] >= split_date]  # 验证集\n",
    "\n",
    "feature_columns, _ = remove_shifted_features(\n",
    "    train_data_split,\n",
    "    val_data_split,\n",
    "    feature_columns)\n",
    "\n",
    "feature_columns = remove_highly_correlated_features(train_data,\n",
    "                                                    feature_columns)\n",
    "keep_columns = [col for col in train_data.columns if\n",
    "                col in feature_columns or col in ['ts_code', 'trade_date', 'label', 'future_return',\n",
    "                                                  'future_score', 'future_volatility']]\n",
    "# train_data = train_data[keep_columns]\n",
    "print(f'feature_columns: {feature_columns}')\n",
    "\n",
    "train_data = train_data.dropna(subset=feature_columns)\n",
    "train_data = train_data.dropna(subset=['label'])\n",
    "train_data = train_data.reset_index(drop=True)\n",
    "\n",
    "# print(test_data.tail())\n",
    "test_data = test_data.dropna(subset=feature_columns)\n",
    "# test_data = test_data.dropna(subset=['label'])\n",
    "test_data = test_data.reset_index(drop=True)\n",
    "\n",
    "print(len(train_data))\n",
    "print(f\"最小日期: {train_data['trade_date'].min().strftime('%Y-%m-%d')}\")\n",
    "print(f\"最大日期: {train_data['trade_date'].max().strftime('%Y-%m-%d')}\")\n",
    "print(len(test_data))\n",
    "print(f\"最小日期: {test_data['trade_date'].min().strftime('%Y-%m-%d')}\")\n",
    "print(f\"最大日期: {test_data['trade_date'].max().strftime('%Y-%m-%d')}\")\n",
    "\n",
    "cat_columns = [col for col in feature_columns if col.startswith('cat')]\n",
    "for col in cat_columns:\n",
    "    train_data[col] = train_data[col].astype('category')\n",
    "    test_data[col] = test_data[col].astype('category')\n",
    "\n",
    "\n",
    "\n",
    "# feature_columns_new.remove('cat_l2_code')"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature_columns size: 114\n",
      "去极值\n",
      "去极值\n",
      "检测到 13 个可能漂移的特征: ['vol', 'pct_chg', 'turnover_rate', 'obv', 'log(circ_mv)', 'alpha_003', 'log_close', 'up', 'down', 'mv_turnover_ratio', 'mv_adjusted_volume', 'mv_weighted_turnover', 'nonlinear_mv_volume']\n",
      "feature_columns: ['pe_ttm', 'volume_ratio', 'winner_rate', 'upside_volatility_5', 'downside_volatility_5', 'volatility_ratio_5', 'upside_semi_variance_5', 'downside_semi_variance_5', 'positive_negative_days_ratio_5', 'avg_positive_return_magnitude_5', 'avg_negative_return_magnitude_5', 'return_skew', 'return_kurtosis', 'volume_change_rate', 'cat_volume_breakout', 'turnover_deviation', 'cat_turnover_spike', 'avg_volume_ratio', 'cat_volume_ratio_breakout', 'vol_spike', 'vol_std_5', 'atr_14', 'maobv_6', 'rsi_3', 'return_5', 'return_20', 'std_return_5', 'std_return_90', 'act_factor1', 'act_factor2', 'act_factor3', 'act_factor4', 'rank_act_factor1', 'rank_act_factor2', 'rank_act_factor3', 'cov', 'delta_cov', 'alpha_007', 'alpha_013', 'cat_up_limit', 'cat_down_limit', 'up_limit_count_10d', 'down_limit_count_10d', 'consecutive_up_limit', 'vol_break', 'weight_roc5', 'smallcap_concentration', 'cost_stability', 'high_cost_break_days', 'liquidity_risk', 'turnover_std', 'mv_volatility', 'volume_growth', 'mv_growth', 'arbr', 'momentum_factor', 'resonance_factor', 'cat_vol_spike', 'obv-maobv_6', 'std_return_5 / std_return_90', 'std_return_90 - std_return_90_2', 'cat_af2', 'cat_af3', 'cat_af4', 'act_factor5', 'act_factor6', 'active_buy_volume_large', 'active_buy_volume_big', 'active_buy_volume_small', 'buy_lg_vol_minus_sell_lg_vol', 'buy_elg_vol_minus_sell_elg_vol', 'ctrl_strength', 'low_cost_dev', 'asymmetry', 'lock_factor', 'cat_vol_break', 'cost_atr_adj', 'cat_golden_resonance', 'mv_momentum', 'industry_obv', 'industry_return_5', 'industry_return_20', 'industry__ema_5', 'industry_act_factor1', 'industry_act_factor2', 'industry_act_factor3', 'industry_act_factor4', 'industry_act_factor5', 'industry_act_factor6', 'industry_rank_act_factor1', 'industry_rank_act_factor2', 'industry_rank_act_factor3', 'industry_return_5_percentile', 'industry_return_20_percentile']\n",
      "590201\n",
      "最小日期: 2018-06-04\n",
      "最大日期: 2023-08-01\n",
      "179005\n",
      "最小日期: 2023-08-01\n",
      "最大日期: 2025-03-28\n"
     ]
    }
   ],
   "execution_count": 55
  },
  {
   "cell_type": "code",
   "id": "8f134d435f71e9e2",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "ExecuteTime": {
     "end_time": "2025-04-03T14:57:51.050696Z",
     "start_time": "2025-04-03T14:57:51.034030Z"
    }
   },
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "import lightgbm as lgb\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "\n",
    "def train_light_model(train_data_df, params, feature_columns, callbacks, evals,\n",
    "                      print_feature_importance=True, num_boost_round=100,\n",
    "                      validation_days=180, use_pca=False, split_date=None):  # 新增参数：validation_days\n",
    "    # 确保数据按时间排序\n",
    "    train_data_df = train_data_df.sort_values(by='trade_date')\n",
    "\n",
    "    numeric_columns = train_data_df.select_dtypes(include=['float64', 'int64']).columns\n",
    "    numeric_columns = [col for col in numeric_columns if col in feature_columns]\n",
    "    # X_train.loc[:, numeric_columns] = scaler.fit_transform(X_train[numeric_columns])\n",
    "    # X_val.loc[:, numeric_columns] = scaler.transform(X_val[numeric_columns])\n",
    "    train_data_df = cross_sectional_standardization(train_data_df, numeric_columns)\n",
    "\n",
    "    # 去除标签为空的样本\n",
    "    train_data_df = train_data_df.dropna(subset=['label'])\n",
    "    print('原始训练集大小: ', len(train_data_df))\n",
    "\n",
    "    # 按时间顺序划分训练集和验证集\n",
    "    if split_date is None:\n",
    "        all_dates = train_data_df['trade_date'].unique()  # 获取所有唯一的 trade_date\n",
    "        split_date = all_dates[-validation_days]  # 划分点为倒数第 validation_days 天\n",
    "    train_data_split = train_data_df[train_data_df['trade_date'] < split_date]  # 训练集\n",
    "    val_data_split = train_data_df[train_data_df['trade_date'] >= split_date]  # 验证集\n",
    "\n",
    "    # 打印划分结果\n",
    "    print(f\"划分后的训练集大小: {len(train_data_split)}, 验证集大小: {len(val_data_split)}\")\n",
    "\n",
    "    # 提取特征和标签\n",
    "    X_train = train_data_split[feature_columns]\n",
    "    y_train = train_data_split['label']\n",
    "\n",
    "    X_val = val_data_split[feature_columns]\n",
    "    y_val = val_data_split['label']\n",
    "\n",
    "    # 标准化数值特征\n",
    "    scaler = StandardScaler()\n",
    "\n",
    "    # 计算每个 trade_date 内的样本数（LTR 需要 group 信息）\n",
    "    train_groups = train_data_split.groupby('trade_date').size().tolist()\n",
    "    val_groups = val_data_split.groupby('trade_date').size().tolist()\n",
    "\n",
    "    # 处理类别特征\n",
    "    categorical_feature = [col for col in feature_columns if 'cat' in col]\n",
    "\n",
    "    pca = None\n",
    "    if use_pca:\n",
    "        pca = PCA(n_components=0.95)  # 或指定 n_components=固定值（如 10）\n",
    "        numeric_features = [col for col in feature_columns if col not in categorical_feature]\n",
    "        numeric_pca = pca.fit_transform(X_train[numeric_features])\n",
    "        X_train = pd.concat([pd.DataFrame(numeric_pca, index=X_train.index), X_train[categorical_feature]], axis=1)\n",
    "\n",
    "        numeric_pca = pca.transform(X_val[numeric_features])\n",
    "        X_val = pd.concat([pd.DataFrame(numeric_pca, index=X_val.index), X_val[categorical_feature]], axis=1)\n",
    "\n",
    "    # 计算权重（基于时间）\n",
    "    # trade_date = train_data_split['trade_date']  # 交易日期\n",
    "    # weights = (trade_date - trade_date.min()).dt.days / (trade_date.max() - trade_date.min()).days + 1\n",
    "    # weights = train_data_split.groupby('trade_date')['std_return_5'].transform(\n",
    "    #     lambda x: x / x.mean()\n",
    "    # )\n",
    "    ud = sorted(train_data_split[\"trade_date\"].unique().tolist())\n",
    "    date_weights = {date: weight * weight for date, weight in zip(ud, np.linspace(1, 10, len(ud)))}\n",
    "    params['weight'] = train_data_split[\"trade_date\"].map(date_weights).tolist()\n",
    "\n",
    "    print('feature_columns size: ', len(X_train.columns.tolist()))\n",
    "\n",
    "    train_dataset = lgb.Dataset(\n",
    "        X_train, label=y_train, group=train_groups,\n",
    "        categorical_feature=categorical_feature\n",
    "    )\n",
    "\n",
    "    # weights = val_data_split.groupby('trade_date')['std_return_5'].transform(\n",
    "    #     lambda x: x / x.mean()\n",
    "    # )\n",
    "    val_dataset = lgb.Dataset(\n",
    "        X_val, label=y_val, group=val_groups,\n",
    "        categorical_feature=categorical_feature\n",
    "    )\n",
    "\n",
    "    # 训练模型\n",
    "    model = lgb.train(\n",
    "        params, train_dataset, num_boost_round=num_boost_round,\n",
    "        valid_sets=[train_dataset, val_dataset], valid_names=['train', 'valid'],\n",
    "        callbacks=callbacks\n",
    "    )\n",
    "\n",
    "    # 打印特征重要性（如果需要）\n",
    "    if print_feature_importance:\n",
    "        lgb.plot_metric(evals)\n",
    "        lgb.plot_importance(model, importance_type='split', max_num_features=20)\n",
    "        plt.show()\n",
    "\n",
    "    return model, scaler, pca\n",
    "\n",
    "\n",
    "from catboost import CatBoostRanker, Pool\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def train_catboost(train_data_df, test_data_df, feature_columns, params=None, plot=False):\n",
    "    X_train = train_data_df[feature_columns]\n",
    "    y_train = train_data_df['label']\n",
    "\n",
    "    X_val = test_data_df[feature_columns]\n",
    "    y_val = test_data_df['label']\n",
    "\n",
    "    scaler = StandardScaler()\n",
    "    numeric_columns = X_train.select_dtypes(include=['float64', 'int64']).columns\n",
    "    X_train.loc[:, numeric_columns] = scaler.fit_transform(X_train[numeric_columns])\n",
    "    X_val.loc[:, numeric_columns] = scaler.transform(X_val[numeric_columns])\n",
    "\n",
    "    group_train = train_data_df['trade_date'].factorize()[0]\n",
    "    group_val = test_data_df['trade_date'].factorize()[0]\n",
    "\n",
    "    cat_features = [i for i, col in enumerate(feature_columns) if col.startswith('cat')]\n",
    "    print(f'cat_features: {cat_features}')\n",
    "\n",
    "    train_pool = Pool(\n",
    "        data=X_train,\n",
    "        label=y_train,\n",
    "        group_id=group_train,\n",
    "        cat_features=cat_features\n",
    "    )\n",
    "\n",
    "    val_pool = Pool(\n",
    "        data=X_val,\n",
    "        label=y_val,\n",
    "        group_id=group_val,\n",
    "        cat_features=cat_features\n",
    "    )\n",
    "\n",
    "    # CatBoost 排序学习模型\n",
    "    model = CatBoostRanker(**params)\n",
    "    model.fit(train_pool, eval_set=val_pool, plot=plot, use_best_model=True)\n",
    "\n",
    "    return model, scaler\n"
   ],
   "outputs": [],
   "execution_count": 56
  },
  {
   "cell_type": "code",
   "id": "c6eb5cd4-e714-420a-ac48-39af3e11ee81",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T15:03:18.426481Z",
     "start_time": "2025-04-03T15:02:19.926352Z"
    }
   },
   "source": [
    "print('train data size: ', len(train_data))\n",
    "\n",
    "label_gain = list(range(len(train_data['label'].unique())))\n",
    "label_gain = [gain * gain for gain in label_gain]\n",
    "light_params = {\n",
    "    'label_gain': label_gain,\n",
    "    'objective': 'lambdarank',\n",
    "    'metric': 'ndcg',\n",
    "    'learning_rate': 0.03,\n",
    "    'num_leaves': 32,\n",
    "    # 'min_data_in_leaf': 128,\n",
    "    'max_depth': 8,\n",
    "    'max_bin': 32,\n",
    "    'feature_fraction': 0.7,\n",
    "    'bagging_fraction': 0.7,\n",
    "    'bagging_freq': 5,\n",
    "    'lambda_l1': 0.1,\n",
    "    'lambda_l2': 0.1,\n",
    "    'boosting': 'goss',\n",
    "    'verbosity': -1,\n",
    "    'extra_trees': True,\n",
    "    'max_position': 5,\n",
    "    'ndcg_at': 1,\n",
    "    'quant_train_renew_leaf': True,\n",
    "    'lambdarank_truncation_level': 1,\n",
    "    'lambdarank_position_bias_regularization': 1,\n",
    "    'seed': 7\n",
    "}\n",
    "evals = {}\n",
    "\n",
    "gc.collect()\n",
    "\n",
    "use_pca = False\n",
    "# feature_contri = [2 if feat.startswith('act_factor') or 'buy' in feat or 'sell' in feat else 1 for feat in feature_columns]\n",
    "# light_params['feature_contri'] = feature_contri\n",
    "# print(f'feature_contri: {feature_contri}')\n",
    "model, scaler, pca = train_light_model(train_data.dropna(subset=['label']),\n",
    "                                       light_params, feature_columns,\n",
    "                                       [lgb.log_evaluation(period=100),\n",
    "                                        lgb.callback.record_evaluation(evals),\n",
    "                                        lgb.early_stopping(100, first_metric_only=True)\n",
    "                                        ], evals,\n",
    "                                       num_boost_round=1000, validation_days=120,\n",
    "                                       print_feature_importance=True, use_pca=use_pca)\n",
    "\n",
    "print('train data size: ', len(train_data))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train data size:  590201\n",
      "原始训练集大小:  590201\n",
      "划分后的训练集大小: 537198, 验证集大小: 53003\n",
      "feature_columns size:  94\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\ttrain's ndcg@1: 0.727839\tvalid's ndcg@1: 0.35097\n",
      "Early stopping, best iteration is:\n",
      "[67]\ttrain's ndcg@1: 0.702439\tvalid's ndcg@1: 0.403463\n",
      "Evaluated only: ndcg@1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAkbpJREFUeJzs3Xd4k9XbwPFvku6WbjqAQtl7L9mgIIqiOBEQARUXuPi5UJn6ihNxggPEheLAybIgUxCw7L03bSmle6XJ8/5xmqShg7aUpk3uz3X1SvKsnJO2yZ1z7nOOTtM0DSGEEEIIJ6F3dAGEEEIIISqSBDdCCCGEcCoS3AghhBDCqUhwI4QQQginIsGNEEIIIZyKBDdCCCGEcCoS3AghhBDCqUhwI4QQQginIsGNEEIIIZyKBDdCCObPn49Op+P48eNX7TmmTp2KTqerNtd1tOPHj6PT6Zg/f365ztfpdEydOrVCyyREdSHBjRCVyBJE6HQ61q9fX2i/pmlERUWh0+m4+eaby/UcH3/8cbk/EEXZLFiwgFmzZjm6GEKIS0hwI4QDeHl5sWDBgkLb16xZw+nTp/H09Cz3tcsT3IwcOZKsrCzq1atX7ud1lJdffpmsrCyHPPfVDG7q1atHVlYWI0eOLNf5WVlZvPzyyxVcKiGqBwluhHCAQYMG8eOPP5KXl2e3fcGCBXTs2JGIiIhKKUdGRgYABoMBLy+vatW9Yym7m5sbXl5eDi7N5WVnZ2M2m0t9vE6nw8vLC4PBUK7n8/Lyws3NrVznClHdSXAjhAMMGzaMCxcuEBMTY92Wm5vLTz/9xPDhw4s8x2w2M2vWLFq2bImXlxfh4eE8/PDDXLx40XpMdHQ0e/bsYc2aNdbur759+wK2LrE1a9bw2GOPERYWRp06dez2XZpzs3TpUvr06UONGjXw9/enc+fORbY4XWr9+vV07twZLy8vGjZsyCeffFLomJJySi7NF7Hk1ezdu5fhw4cTFBREz5497fZdev748eP59ddfadWqFZ6enrRs2ZJly5YVeq7Vq1fTqVMnu7KWJo+nb9++LF68mBMnTlhf6+joaOs1dTod33//PS+//DK1a9fGx8eH1NRUkpKSeOaZZ2jdujV+fn74+/tz4403smPHjsu+PqNHj8bPz48zZ84wZMgQ/Pz8qFmzJs888wwmk6lUr+Hhw4cZPXo0gYGBBAQEMGbMGDIzM+3OzcrK4oknniA0NJQaNWpwyy23cObMGcnjEdWGhPVCOEB0dDTdunXju+++48YbbwRUIJGSksI999zD+++/X+ichx9+mPnz5zNmzBieeOIJjh07xocffsi2bdv4559/cHd3Z9asWTz++OP4+fnx0ksvARAeHm53nccee4yaNWsyefJka+tHUebPn8/9999Py5YtmThxIoGBgWzbto1ly5YVG4AB7Nq1i+uvv56aNWsydepU8vLymDJlSqFylMddd91F48aNee2119A0rcRj169fz6JFi3jssceoUaMG77//PnfccQcnT54kJCQEgG3btnHDDTcQGRnJtGnTMJlMTJ8+nZo1a162LC+99BIpKSmcPn2ad999FwA/Pz+7Y1555RU8PDx45plnyMnJwcPDg7179/Lrr79y1113Ub9+feLj4/nkk0/o06cPe/fupVatWiU+r8lkYuDAgXTt2pW3336bFStW8M4779CwYUMeffTRy5b77rvvpn79+syYMYOtW7fy+eefExYWxhtvvGE9ZvTo0fzwww+MHDmSa665hjVr1nDTTTdd9tpCVBmaEKLSfPHFFxqgbdmyRfvwww+1GjVqaJmZmZqmadpdd92l9evXT9M0TatXr5520003Wc9bt26dBmjffvut3fWWLVtWaHvLli21Pn36FPvcPXv21PLy8orcd+zYMU3TNC05OVmrUaOG1rVrVy0rK8vuWLPZXGIdhwwZonl5eWknTpywbtu7d69mMBi0gm85x44d0wDtiy++KHQNQJsyZYr18ZQpUzRAGzZsWKFjLfsuPd/Dw0M7fPiwdduOHTs0QPvggw+s2wYPHqz5+PhoZ86csW47dOiQ5ubmVuiaRbnpppu0evXqFdq+atUqDdAaNGhg/f1aZGdnayaTyW7bsWPHNE9PT2369Ol22y59fUaNGqUBdsdpmqa1b99e69ixY6HXoKjX8P7777c77rbbbtNCQkKsj2NjYzVAe+qpp+yOGz16dKFrClFVSbeUEA5y9913k5WVxZ9//klaWhp//vlnsS0iP/74IwEBAQwYMIDExETrT8eOHfHz82PVqlWlft6xY8deNo8jJiaGtLQ0XnjhhUL5LCV115hMJpYvX86QIUOoW7eudXvz5s0ZOHBgqctYnEceeaTUx/bv35+GDRtaH7dp0wZ/f3+OHj1qLeuKFSsYMmSIXWtJo0aNrK1pV2rUqFF4e3vbbfP09ESv11vLcOHCBfz8/GjatClbt24t1XUvfR169eplrVd5zr1w4QKpqakA1q67xx57zO64xx9/vFTXF6IqkG4pIRykZs2a9O/fnwULFpCZmYnJZOLOO+8s8thDhw6RkpJCWFhYkfsTEhJK/bz169e/7DFHjhwBoFWrVqW+LsD58+fJysqicePGhfY1bdqUJUuWlOl6lypN2S0KBlcWQUFB1hylhIQEsrKyaNSoUaHjitpWHkWV12w289577/Hxxx9z7Ngxu1wZS3dZSby8vAp1mxWs1+Vc+roEBQUBcPHiRfz9/Tlx4gR6vb5Q2SvqNRGiMkhwI4QDDR8+nLFjxxIXF8eNN95IYGBgkceZzWbCwsL49ttvi9xfmhwRi0tbEhyluBagSxNjCypL2YtrndIuk6tTkYoq72uvvcakSZO4//77eeWVVwgODkav1/PUU0+VajRVeUdPXe78ynxdhLjaJLgRwoFuu+02Hn74Yf79918WLlxY7HENGzZkxYoV9OjR47If8BUxnNvSnbN79+4yfWOvWbMm3t7eHDp0qNC+AwcO2D22tBgkJyfbbT9x4kQZS1s+YWFheHl5cfjw4UL7itpWlPK81j/99BP9+vVj7ty5dtuTk5MJDQ0t8/UqWr169TCbzRw7dsyuBa60r4kQVYHk3AjhQH5+fsyePZupU6cyePDgYo+7++67MZlMvPLKK4X25eXl2QUIvr6+hQKGsrr++uupUaMGM2bMIDs7225fSd/wDQYDAwcO5Ndff+XkyZPW7fv27WP58uV2x/r7+xMaGsratWvttn/88cdXVPbSMhgM9O/fn19//ZWzZ89atx8+fJilS5eW6hq+vr6kpKSU+XkvfQ1//PFHzpw5U6brXC2W3KhLfw8ffPCBI4ojRLlIy40QDjZq1KjLHtOnTx8efvhhZsyYwfbt27n++utxd3fn0KFD/Pjjj7z33nvWfJ2OHTsye/ZsXn31VRo1akRYWBjXXnttmcrk7+/Pu+++y4MPPkjnzp2tc8vs2LGDzMxMvvzyy2LPnTZtGsuWLaNXr1489thj5OXl8cEHH9CyZUt27txpd+yDDz7I66+/zoMPPkinTp1Yu3YtBw8eLFNZr8TUqVP566+/6NGjB48++igmk4kPP/yQVq1asX379sue37FjRxYuXMiECRPo3Lkzfn5+JQapADfffDPTp09nzJgxdO/enV27dvHtt9/SoEGDCqrVlenYsSN33HEHs2bN4sKFC9ah4JbfS3Wa6FG4LgluhKgm5syZQ8eOHfnkk0948cUXcXNzIzo6mnvvvZcePXpYj5s8eTInTpzgzTffJC0tjT59+pQ5uAF44IEHCAsL4/XXX+eVV17B3d2dZs2a8fTTT5d4Xps2bVi+fDkTJkxg8uTJ1KlTh2nTpnHu3LlCwc3kyZM5f/48P/30Ez/88AM33ngjS5cuLTZxuqJ17NiRpUuX8swzzzBp0iSioqKYPn06+/btY//+/Zc9/7HHHmP79u188cUXvPvuu9SrV++ywc2LL75IRkYGCxYsYOHChXTo0IHFixfzwgsvVFS1rthXX31FREQE3333Hb/88gv9+/dn4cKFNG3atFrMBi2ETpMsMiGEsDNkyBD27NlTZO6Qq9q+fTvt27f
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwwAAAHHCAYAAAASz98lAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAA88xJREFUeJzs3Xtczuf/wPHX3VHnlEhEDkkOEYYck0NEjtPQkDkMa05zyiHlmNPEmG1YOS4zhzE5FIvNMTbmmDHJNhaGVJNb3b8//Lq/bhV34U55Px+PHvpc9/W5Pu/Pu3Tf1+e6rs9HoVKpVAghhBBCCCFELvQKOwAhhBBCCCHEm0s6DEIIIYQQQog8SYdBCCGEEEIIkSfpMAghhBBCCCHyJB0GIYQQQgghRJ6kwyCEEEIIIYTIk3QYhBBCCCGEEHmSDoMQQgghhBAiT9JhEEIIIYQQQuRJOgxCCCHEWyQyMhKFQkFiYmJhhyKEKCKkwyCEEKJYy/6AnNvXxIkTX8sxDx8+TEhICPfu3Xst7b/N0tPTCQkJIS4urrBDEeKtYVDYAQghhBC6MH36dCpVqqRRVqtWrddyrMOHDxMaGkpAQADW1tav5RgF1bdvX3r16oWxsXFhh1Ig6enphIaGAuDp6Vm4wQjxlpAOgxBCiLdChw4daNCgQWGH8VLS0tIwMzN7qTb09fXR19d/RRHpTlZWFo8ePSrsMIR4K8mUJCGEEALYtWsXzZs3x8zMDAsLCzp27Mi5c+c06vz2228EBARQuXJlSpQogb29PR988AF37txR1wkJCWHcuHEAVKpUST39KTExkcTERBQKBZGRkTmOr1AoCAkJ0WhHoVBw/vx5+vTpQ8mSJWnWrJn69XXr1lG/fn1MTEywsbGhV69eXL9+/YXnmdsaBicnJzp16kRcXBwNGjTAxMSE2rVrq6f9bNmyhdq1a1OiRAnq16/Pr7/+qtFmQEAA5ubm/PHHH3h7e2NmZoaDgwPTp09HpVJp1E1LS+OTTz7B0dERY2NjXFxcWLBgQY56CoWCwMBA1q9fT82aNTE2NuaLL77Azs4OgNDQUHVus/Omzc/n6dxevnxZPQpkZWXFgAEDSE9Pz5GzdevW0bBhQ0xNTSlZsiQtWrRg7969GnW0+f0RoqiSEQYhhBBvhfv373P79m2NslKlSgGwdu1a+vfvj7e3N3PnziU9PZ3ly5fTrFkzfv31V5ycnACIiYnhjz/+YMCAAdjb23Pu3Dm++uorzp07x9GjR1EoFHTv3p1Lly7xzTffsGjRIvUx7OzsuHXrVr7j7tmzJ87OzsyePVv9oXrWrFlMnToVPz8/Bg0axK1bt/jss89o0aIFv/76a4GmQV2+fJk+ffrw4Ycf8v7777NgwQJ8fX354osvmDRpEsOHDwdgzpw5+Pn5kZCQgJ7e/647ZmZm0r59exo3bsy8efPYvXs306ZN4/Hjx0yfPh0AlUpF586d+fHHHxk4cCB169Zlz549jBs3jr/++otFixZpxLR//36+/fZbAgMDKVWqFHXq1GH58uUMGzaMbt260b17dwDc3NwA7X4+T/Pz86NSpUrMmTOHX375hZUrV1K6dGnmzp2rrhMaGkpISAhNmjRh+vTpGBkZcezYMfbv30+7du0A7X9/hCiyVEIIIUQxFhERoQJy/VKpVKoHDx6orK2tVYMHD9bY7+bNmyorKyuN8vT09Bztf/PNNypAdfDgQXXZ/PnzVYDq6tWrGnWvXr2qAlQRERE52gFU06ZNU29PmzZNBah69+6tUS8xMVGlr6+vmjVrlkb5mTNnVAYGBjnK88rH07FVrFhRBagOHz6sLtuzZ48KUJmYmKiuXbumLv/yyy9VgOrHH39Ul/Xv318FqD7++GN1WVZWlqpjx44qIyMj1a1bt1QqlUq1bds2FaCaOXOmRkzvvvuuSqFQqC5fvqyRDz09PdW5c+c06t66dStHrrJp+/PJzu0HH3ygUbdbt24qW1tb9fbvv/+u0tPTU3Xr1k2VmZmpUTcrK0ulUuXv90eIokqmJAkhhHgrLFu2jJiYGI0veHJV+t69e/Tu3Zvbt2+rv/T19WnUqBE//vijug0TExP19w8fPuT27ds0btwYgF9++eW1xD106FCN7S1btpCVlYWfn59GvPb29jg7O2vEmx81atTAw8NDvd2oUSMAvLy8qFChQo7yP/74I0cbgYGB6u+zpxQ9evSI2NhYAKKjo9HX12fEiBEa+33yySeoVCp27dqlUd6yZUtq1Kih9Tnk9+fzbG6bN2/OnTt3SElJAWDbtm1kZWURHBysMZqSfX6Qv98fIYoqmZIkhBDirdCwYcNcFz3//vvvwJMPxrmxtLRUf//vv/8SGhpKVFQUycnJGvXu37//CqP9n2fv7PT777+jUqlwdnbOtb6hoWGBjvN0pwDAysoKAEdHx1zL7969q1Gup6dH5cqVNcqqVasGoF4vce3aNRwcHLCwsNCo5+rqqn79ac+e+4vk9+fz7DmXLFkSeHJulpaWXLlyBT09ved2WvLz+yNEUSUdBiGEEG+1rKws4Mk8dHt7+xyvGxj8763Sz8+Pw4cPM27cOOrWrYu5uTlZWVm0b99e3c7zPDuHPltmZmae+zx91Tw7XoVCwa5du3K925G5ufkL48hNXndOyqtc9cwi5dfh2XN/kfz+fF7FueXn90eIokp+i4UQQrzVqlSpAkDp0qVp06ZNnvXu3r3Lvn37CA0NJTg4WF2efYX5aXl1DLKvYD/7QLdnr6y/KF6VSkWlSpXUV/DfBFlZWfzxxx8aMV26dAlAvei3YsWKxMbG8uDBA41RhosXL6pff5G8cpufn4+2qlSpQlZWFufPn6du3bp51oEX//4IUZTJGgYhhBBvNW9vbywtLZk9ezZKpTLH69l3Nsq+Gv3s1efw8PAc+2Q/K+HZjoGlpSWlSpXi4MGDGuWff/651vF2794dfX19QkNDc8SiUqly3EJUl5YuXaoRy9KlSzE0NKR169YA+Pj4kJmZqVEPYNGiRSgUCjp06PDCY5iamgI5c5ufn4+2unbtip6eHtOnT88xQpF9HG1/f4QoymSEQQghxFvN0tKS5cuX07dvX+rVq0evXr2ws7MjKSmJnTt30rRpU5YuXYqlpSUtWrRg3rx5KJVKypUrx969e7l69WqONuvXrw/A5MmT6dWrF4aGhvj6+mJmZsagQYMICwtj0KBBNGjQgIMHD6qvxGujSpUqzJw5k6CgIBITE+natSsWFhZcvXqVrVu3MmTIEMaOHfvK8qOtEiVKsHv3bvr370+jRo3YtWsXO3fuZNKkSepnJ/j6+tKqVSsmT55MYmIiderUYe/evXz//feMGjVKfbX+eUxMTKhRowYbN26kWrVq2NjYUKtWLWrVqqX1z0dbVatWZfLkycyYMYPmzZvTvXt3jI2NiY+Px8HBgTlz5mj9+yNEkVZId2cSQgghdCL7NqLx8fHPrffjjz+qvL29VVZWVqoSJUqoqlSpogoICFCdOHFCXefPP/9UdevWTWVtba2ysrJS9ezZU/X333/nepvPGTNmqMqVK6fS09PTuI1penq6auDAgSorKyuVhYWFys/PT5WcnJznbVWzb0n6rM2bN6uaNWumMjMzU5mZmamqV6+u+uijj1QJCQla5ePZ26p27NgxR11A9dFHH2mUZd8adv78+eqy/v37q8zMzFRXrlxRtWvXTmVqaqoqU6aMatq0aTluR/rgwQPV6NGjVQ4ODipDQ0OVs7Ozav78+erblD7v2NkOHz6sql+/vsrIyEgjb9r+fPLKbW65UalUqq+//lrl7u6uMjY2VpUsWVLVsmVLVUxMjEYdbX5/hCiqFCqVDlYtCSGEEKLYCggI4LvvviM1NbWwQxFCvAayhkEIIYQQQgiRJ+kwCCGEEEIIIfIkHQYhhBBCCCFEnmQNgxBCCCGEECJPMsIghBBCCCGEyJN0GIQQQgghhBB5kge3CSFeSlZWFn///TcWFhYoFIrCDkcIIYQQWlCpVDx48AAHBwf09J4/hiAdBiHES/n7779xdHQs7DCEEEIIUQDXr1+nfPnyz60jHQYhxEu
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train data size:  590201\n"
     ]
    }
   ],
   "execution_count": 62
  },
  {
   "cell_type": "code",
   "id": "5d1522a7538db91b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T15:04:39.656944Z",
     "start_time": "2025-04-03T15:04:39.298483Z"
    }
   },
   "source": [
    "# train_data = train_data.sort_values(by='trade_date')\n",
    "# all_dates = train_data['trade_date'].unique()  # 获取所有唯一的 trade_date\n",
    "# split_date = all_dates[-120]  # 划分点为倒数第 validation_days 天\n",
    "# print(split_date)\n",
    "# print(all_dates)\n",
    "# val_data_split = train_data[train_data['trade_date'] >= split_date]  # 验证集\n",
    "\n",
    "score_df = test_data\n",
    "numeric_columns = score_df.select_dtypes(include=['float64', 'int64']).columns\n",
    "numeric_columns = [col for col in numeric_columns if col in feature_columns]\n",
    "# score_df.loc[:, numeric_columns] = scaler.transform(score_df[numeric_columns])\n",
    "score_df = cross_sectional_standardization(score_df, numeric_columns)\n",
    "\n",
    "if use_pca and pca is not None:\n",
    "    categorical_feature = [col for col in feature_columns if 'cat' in col]\n",
    "    numeric_features = [col for col in feature_columns if col not in categorical_feature]\n",
    "    numeric_pca = pca.transform(score_df[numeric_features])\n",
    "    score_df = pd.concat([pd.DataFrame(numeric_pca), score_df[categorical_feature],\n",
    "                          score_df[['trade_date', 'ts_code', 'future_return', 'future_score', 'label']]], axis=1)\n",
    "    score_df['score'] = model.predict(score_df[[col for col in score_df.columns if\n",
    "                                                col not in ['trade_date', 'ts_code', 'future_return', 'future_score',\n",
    "                                                            'label']]])\n",
    "else:\n",
    "    score_df['score'] = model.predict(score_df[feature_columns])\n",
    "# train_data['score'] = catboost_model.predict(train_data[feature_columns])\n",
    "score_df = score_df.loc[score_df.groupby('trade_date')['score'].idxmax()]\n",
    "# score_df = score_df[score_df['score'] > 0]\n",
    "score_df[['trade_date', 'score', 'ts_code']].to_csv('predictions_test.tsv', index=False)\n",
    "print(score_df['label'].mean())\n",
    "print(score_df[['trade_date', 'ts_code', 'future_return', 'future_score', 'label']].head(10))"
   ],
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'test_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[1], line 8\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# train_data = train_data.sort_values(by='trade_date')\u001B[39;00m\n\u001B[0;32m      2\u001B[0m \u001B[38;5;66;03m# all_dates = train_data['trade_date'].unique()  # 获取所有唯一的 trade_date\u001B[39;00m\n\u001B[0;32m      3\u001B[0m \u001B[38;5;66;03m# split_date = all_dates[-120]  # 划分点为倒数第 validation_days 天\u001B[39;00m\n\u001B[0;32m      4\u001B[0m \u001B[38;5;66;03m# print(split_date)\u001B[39;00m\n\u001B[0;32m      5\u001B[0m \u001B[38;5;66;03m# print(all_dates)\u001B[39;00m\n\u001B[0;32m      6\u001B[0m \u001B[38;5;66;03m# val_data_split = train_data[train_data['trade_date'] >= split_date]  # 验证集\u001B[39;00m\n\u001B[1;32m----> 8\u001B[0m score_df \u001B[38;5;241m=\u001B[39m test_data\n\u001B[0;32m      9\u001B[0m numeric_columns \u001B[38;5;241m=\u001B[39m score_df\u001B[38;5;241m.\u001B[39mselect_dtypes(include\u001B[38;5;241m=\u001B[39m[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mfloat64\u001B[39m\u001B[38;5;124m'\u001B[39m, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mint64\u001B[39m\u001B[38;5;124m'\u001B[39m])\u001B[38;5;241m.\u001B[39mcolumns\n\u001B[0;32m     10\u001B[0m numeric_columns \u001B[38;5;241m=\u001B[39m [col \u001B[38;5;28;01mfor\u001B[39;00m col \u001B[38;5;129;01min\u001B[39;00m numeric_columns \u001B[38;5;28;01mif\u001B[39;00m col \u001B[38;5;129;01min\u001B[39;00m feature_columns]\n",
      "\u001B[1;31mNameError\u001B[0m: name 'test_data' is not defined"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "id": "d86af99d15cb3bdd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T15:03:25.791021Z",
     "start_time": "2025-04-03T15:03:25.537833Z"
    }
   },
   "source": [
    "print(df[(df['ts_code'] == '603577.SH') & (df['trade_date'] >= '2018-06-04')][\n",
    "          ['trade_date', 'ts_code', 'close', 'open', 'future_return']])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        trade_date    ts_code  close   open  future_return\n",
      "255411  2018-06-04  603577.SH  24.57  23.98      -0.044408\n",
      "257982  2018-06-05  603577.SH  23.45  24.32      -0.037819\n",
      "260550  2018-06-06  603577.SH  23.24  22.74      -0.052262\n",
      "263114  2018-06-07  603577.SH  21.88  22.77      -0.028451\n",
      "265676  2018-06-08  603577.SH  21.58  21.44      -0.002314\n",
      "...            ...        ...    ...    ...            ...\n",
      "5105528 2025-03-24  603577.SH  20.82  20.67      -0.001921\n",
      "5108613 2025-03-25  603577.SH  20.33  20.82      -0.005411\n",
      "5111699 2025-03-26  603577.SH  20.78  20.33      -0.032771\n",
      "5114784 2025-03-27  603577.SH  20.22  20.75            NaN\n",
      "5117867 2025-03-28  603577.SH  20.07  20.20            NaN\n",
      "\n",
      "[1655 rows x 5 columns]\n"
     ]
    }
   ],
   "execution_count": 64
  },
  {
   "cell_type": "code",
   "id": "ef9d068e-67f7-412c-bbd8-cdee7492dbc9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-03T15:03:25.893508Z",
     "start_time": "2025-04-03T15:03:25.878525Z"
    }
   },
   "source": [
    "print(train_data[\"future_score\"].corr(train_data[\"label\"]))\n",
    "print(test_data[\"future_score\"].corr(test_data[\"label\"]))\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7684793250465913\n",
      "0.6965949564132026\n"
     ]
    }
   ],
   "execution_count": 65
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}