Files
NewStock/main/data/update/update_daily_basic.ipynb

489 lines
19 KiB
Plaintext
Raw Normal View History

2025-02-12 00:21:33 +08:00
{
"cells": [
{
"cell_type": "code",
2025-05-06 23:42:40 +08:00
"execution_count": 1,
2025-02-12 00:21:33 +08:00
"id": "18d1d622-b083-4cc4-a6f8-7c1ed2d0edd2",
"metadata": {
"ExecuteTime": {
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:57:36.913044Z",
"start_time": "2025-04-09T14:57:36.159612Z"
2025-02-12 00:21:33 +08:00
}
},
2025-05-06 23:42:40 +08:00
"outputs": [],
2025-02-12 00:21:33 +08:00
"source": [
"import tushare as ts\n",
"ts.set_token('3a0741c702ee7e5e5f2bf1f0846bafaafe4e320833240b2a7e4a685f')\n",
"pro = ts.pro_api()"
2025-05-06 23:42:40 +08:00
]
2025-02-12 00:21:33 +08:00
},
{
2025-02-15 23:33:34 +08:00
"cell_type": "code",
2025-05-06 23:42:40 +08:00
"execution_count": 2,
2025-02-15 23:33:34 +08:00
"id": "14671a7f72de2564",
2025-02-12 00:21:33 +08:00
"metadata": {
"ExecuteTime": {
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:57:39.128278Z",
"start_time": "2025-04-09T14:57:36.918051Z"
2025-02-12 00:21:33 +08:00
}
},
2025-05-06 23:42:40 +08:00
"outputs": [],
2025-02-12 00:21:33 +08:00
"source": [
"from datetime import datetime\n",
"import pandas as pd\n",
2025-03-31 23:08:03 +08:00
"import warnings\n",
"\n",
"warnings.filterwarnings(\"ignore\")\n",
"def filter_rows(df):\n",
" # 按照 name 和 start_date 分组\n",
" def select_row(group):\n",
" # 如果有 end_date 不为 NaT 的行,优先保留这些行\n",
" valid_rows = group[group['end_date'].notna()]\n",
" if not valid_rows.empty:\n",
" return valid_rows.iloc[0] # 返回第一个有效行\n",
" else:\n",
" return group.iloc[0] # 如果没有有效行,返回第一行\n",
"\n",
" filtered_df = df.groupby(['name', 'start_date'], group_keys=False).apply(select_row)\n",
" filtered_df = filtered_df.reset_index(drop=True)\n",
" return filtered_df\n",
2025-02-12 00:21:33 +08:00
"\n",
"def is_st(name_change_dict, stock_code, target_date):\n",
" target_date = datetime.strptime(target_date, '%Y%m%d')\n",
" if stock_code not in name_change_dict.keys():\n",
" return False\n",
" df = name_change_dict[stock_code]\n",
" for i in range(len(df)):\n",
" sds = df.iloc[i, 2]\n",
" eds = df.iloc[i, 3]\n",
" if eds is None or eds is pd.NaT:\n",
" eds = datetime.now()\n",
" if (target_date - sds).days >= 0 and (target_date - eds).days <= 0:\n",
" return True\n",
" return False\n",
"\n",
"name_change_df = pd.read_hdf('../../../data/name_change.h5', key='name_change')\n",
"name_change_df = name_change_df.drop_duplicates(keep='first')\n",
"\n",
"# 确保 name_change_df 的日期格式正确\n",
"name_change_df['start_date'] = pd.to_datetime(name_change_df['start_date'], format='%Y%m%d')\n",
"name_change_df['end_date'] = pd.to_datetime(name_change_df['end_date'], format='%Y%m%d', errors='coerce')\n",
"name_change_df = name_change_df[name_change_df.name.str.contains('ST')]\n",
"name_change_dict = {}\n",
"for ts_code, group in name_change_df.groupby('ts_code'):\n",
" # 只保留 'ST' 和 '*ST' 的记录\n",
2025-05-06 23:42:40 +08:00
" # st_data = group[(group['change_reason'] == 'ST') | (group['change_reason'] == '*ST')]\n",
" st_data = group[group['name'].str.contains('ST')]\n",
2025-02-12 00:21:33 +08:00
" if not st_data.empty:\n",
2025-03-31 23:08:03 +08:00
" name_change_dict[ts_code] = filter_rows(st_data)"
2025-05-06 23:42:40 +08:00
]
2025-02-12 00:21:33 +08:00
},
{
2025-02-15 23:33:34 +08:00
"cell_type": "code",
2025-05-06 23:42:40 +08:00
"execution_count": 3,
2025-02-15 23:33:34 +08:00
"id": "e7f8cce2f80e2f20",
2025-02-12 00:21:33 +08:00
"metadata": {
"ExecuteTime": {
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:09.296046Z",
"start_time": "2025-04-09T14:57:39.339423Z"
2025-02-12 00:21:33 +08:00
}
},
2025-05-06 23:42:40 +08:00
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"Index: 8599138 entries, 0 to 8599137\n",
"Data columns (total 2 columns):\n",
" # Column Dtype \n",
"--- ------ ----- \n",
" 0 ts_code object\n",
" 1 trade_date object\n",
"dtypes: object(2)\n",
"memory usage: 196.8+ MB\n",
"None\n",
"20250430\n",
"20250506\n"
]
}
],
2025-02-12 00:21:33 +08:00
"source": [
"import time\n",
"from concurrent.futures import ThreadPoolExecutor, as_completed\n",
"\n",
"h5_filename = '../../../data/daily_basic.h5'\n",
"key = '/daily_basic'\n",
"max_date = None\n",
"with pd.HDFStore(h5_filename, mode='r') as store:\n",
" df = store[key][['ts_code', 'trade_date']]\n",
" print(df.info())\n",
" max_date = df['trade_date'].max()\n",
"\n",
"print(max_date)\n",
2025-05-06 23:42:40 +08:00
"trade_cal = pro.trade_cal(exchange='', start_date='20170101', end_date='20250720')\n",
2025-02-12 00:21:33 +08:00
"trade_cal = trade_cal[trade_cal['is_open'] == 1] # 只保留交易日\n",
"trade_dates = trade_cal[trade_cal['cal_date'] > max_date]['cal_date'].tolist()\n",
"start_date = min(trade_dates)\n",
"print(start_date)"
2025-05-06 23:42:40 +08:00
]
2025-04-09 22:57:01 +08:00
},
{
"cell_type": "code",
2025-05-06 23:42:40 +08:00
"execution_count": 4,
2025-04-09 22:57:01 +08:00
"id": "553cfb36-f560-4cc4-b2bc-68323ccc5072",
"metadata": {
"ExecuteTime": {
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:16.817010Z",
"start_time": "2025-04-09T14:58:09.326485Z"
2025-05-06 23:42:40 +08:00
},
"scrolled": true
2025-04-09 22:57:01 +08:00
},
2025-05-06 23:42:40 +08:00
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"任务 20250718 完成\n",
"任务 20250717 完成\n",
"任务 20250715 完成\n",
"任务 20250716 完成\n",
"任务 20250711 完成\n",
"任务 20250714 完成\n",
"任务 20250709 完成\n",
"任务 20250710 完成\n",
"任务 20250707 完成\n",
"任务 20250708 完成\n",
"任务 20250704 完成\n",
"任务 20250703 完成\n",
"任务 20250702 完成\n",
"任务 20250701 完成\n",
"任务 20250630 完成\n",
"任务 20250627 完成\n",
"任务 20250626 完成\n",
"任务 20250625 完成\n",
"任务 20250624 完成\n",
"任务 20250623 完成\n",
"任务 20250619 完成\n",
"任务 20250620 完成\n",
"任务 20250618 完成\n",
"任务 20250617 完成\n",
"任务 20250616 完成\n",
"任务 20250613 完成\n",
"任务 20250612 完成\n",
"任务 20250611 完成\n",
"任务 20250610 完成\n",
"任务 20250609 完成\n",
"任务 20250606 完成\n",
"任务 20250605 完成\n",
"任务 20250604 完成\n",
"任务 20250603 完成\n",
"任务 20250530 完成\n",
"任务 20250529 完成\n",
"任务 20250528 完成\n",
"任务 20250527 完成\n",
"任务 20250526 完成\n",
"任务 20250523 完成\n",
"任务 20250522 完成\n",
"任务 20250521 完成\n",
"任务 20250520 完成\n",
"任务 20250519 完成\n",
"任务 20250516 完成\n",
"任务 20250515 完成\n",
"任务 20250514 完成\n",
"任务 20250513 完成\n",
"任务 20250512 完成\n",
"任务 20250509 完成\n",
"任务 20250508 完成\n",
"任务 20250507 完成\n",
"任务 20250506 完成\n"
]
}
],
2025-02-12 00:21:33 +08:00
"source": [
"\n",
"\n",
"# 使用 HDFStore 存储数据\n",
"all_daily_data = []\n",
"\n",
"# API 调用计数和时间控制变量\n",
"api_call_count = 0\n",
"batch_start_time = time.time()\n",
"\n",
"\n",
"def get_data(trade_date):\n",
" daily_basic_data = pro.daily_basic(ts_code='', trade_date=trade_date)\n",
" if daily_basic_data is not None and not daily_basic_data.empty:\n",
" # 添加交易日期列标识\n",
" daily_basic_data['trade_date'] = trade_date\n",
" daily_basic_data['is_st'] = daily_basic_data.apply(\n",
" lambda row: is_st(name_change_dict, row['ts_code'], row['trade_date']), axis=1\n",
" )\n",
" time.sleep(0.2)\n",
" # print(f\"成功获取并保存 {trade_date} 的每日基础数据\")\n",
" return daily_basic_data\n",
"\n",
"\n",
"# 遍历每个交易日期并获取数据\n",
"with ThreadPoolExecutor(max_workers=2) as executor:\n",
" future_to_date = {executor.submit(get_data, td): td for td in trade_dates}\n",
"\n",
" for future in as_completed(future_to_date):\n",
" trade_date = future_to_date[future] # 获取对应的交易日期\n",
" try:\n",
" result = future.result() # 获取任务执行的结果\n",
" all_daily_data.append(result)\n",
" print(f\"任务 {trade_date} 完成\")\n",
" except Exception as e:\n",
" print(f\"获取 {trade_date} 数据时出错: {e}\")\n",
" # 计数一次 API 调用\n",
" api_call_count += 1\n",
"\n",
" # 每调用 300 次,检查时间是否少于 1 分钟,如果少于则等待剩余时间\n",
" if api_call_count % 150 == 0:\n",
" elapsed = time.time() - batch_start_time\n",
" if elapsed < 60:\n",
" sleep_time = 60 - elapsed\n",
" print(f\"已调用 150 次 API等待 {sleep_time:.2f} 秒以满足速率限制...\")\n",
" time.sleep(sleep_time)\n",
" # 重置批次起始时间\n",
" batch_start_time = time.time()\n",
"\n"
2025-05-06 23:42:40 +08:00
]
2025-02-12 00:21:33 +08:00
},
{
2025-02-15 23:33:34 +08:00
"cell_type": "code",
2025-05-06 23:42:40 +08:00
"execution_count": 5,
2025-02-15 23:33:34 +08:00
"id": "919023c693d7a47a",
2025-02-12 00:21:33 +08:00
"metadata": {
"ExecuteTime": {
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:16.864178Z",
"start_time": "2025-04-09T14:58:16.855084Z"
2025-02-12 00:21:33 +08:00
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
2025-05-06 23:42:40 +08:00
" ts_code trade_date close turnover_rate turnover_rate_f \\\n",
"0 002390.SZ 20250506 3.48 0.7696 1.3833 \n",
"1 300708.SZ 20250506 11.64 2.8994 3.2217 \n",
"2 301171.SZ 20250506 27.73 9.9120 10.7228 \n",
"3 301662.SZ 20250506 52.50 17.0926 17.0926 \n",
"4 001309.SZ 20250506 129.63 5.7123 6.3388 \n",
"... ... ... ... ... ... \n",
"5381 000551.SZ 20250506 12.39 2.0213 3.1432 \n",
"5382 600792.SH 20250506 3.17 0.8036 2.3531 \n",
"5383 300176.SZ 20250506 6.62 1.7530 2.5325 \n",
"5384 000016.SZ 20250506 5.57 13.9545 20.7669 \n",
"5385 300339.SZ 20250506 56.53 11.3184 11.9579 \n",
2025-02-12 00:21:33 +08:00
"\n",
2025-05-06 23:42:40 +08:00
" volume_ratio pe pe_ttm pb ps ps_ttm dv_ratio \\\n",
"0 1.02 66.7242 80.7223 1.0020 1.1214 1.1483 2.5321 \n",
"1 1.14 40.4767 37.8935 2.9328 2.8689 2.7390 1.3334 \n",
"2 0.95 56.4451 55.0565 3.6159 5.1380 4.3691 0.4867 \n",
"3 0.79 20.2143 23.5423 2.7909 2.0091 2.2310 NaN \n",
"4 1.02 59.8205 243.9150 8.6523 4.3939 4.0221 0.0702 \n",
"... ... ... ... ... ... ... ... \n",
"5381 1.20 19.9692 18.7030 1.8602 1.1939 1.1927 0.5650 \n",
"5382 0.89 NaN NaN 1.1995 0.5271 0.5777 2.1767 \n",
"5383 1.12 92.1443 96.5538 2.7208 1.4839 1.4627 0.0000 \n",
"5384 3.66 NaN NaN 5.6643 1.2067 1.1979 0.0000 \n",
"5385 2.40 279.4392 270.1037 12.8967 13.2445 13.0061 0.0000 \n",
2025-02-12 00:21:33 +08:00
"\n",
2025-05-06 23:42:40 +08:00
" dv_ttm total_share float_share free_share total_mv \\\n",
"0 2.5321 194385.1868 185230.5076 103045.2550 6.764605e+05 \n",
"1 1.3003 68015.2346 52260.4413 47031.2918 7.916973e+05 \n",
"2 0.4867 47188.5905 30877.5025 28542.8345 1.308540e+06 \n",
"3 NaN 8000.0000 1577.6325 1577.6325 4.200000e+05 \n",
"4 NaN 16177.0306 8763.6153 7897.4398 2.097028e+06 \n",
"... ... ... ... ... ... \n",
"5381 0.5650 40394.4205 40263.2044 25893.0990 5.004869e+05 \n",
"5382 2.1767 110992.3600 105986.8113 36194.3684 3.518458e+05 \n",
"5383 NaN 38728.0800 38728.0800 26808.2764 2.563799e+05 \n",
"5384 NaN 240794.5408 159659.3800 107284.6868 1.341226e+06 \n",
"5385 NaN 79641.0841 77768.6667 73609.4256 4.502110e+06 \n",
2025-02-12 00:21:33 +08:00
"\n",
2025-04-09 22:57:01 +08:00
" circ_mv is_st \n",
2025-05-06 23:42:40 +08:00
"0 6.446022e+05 False \n",
"1 6.083115e+05 False \n",
"2 8.562331e+05 False \n",
"3 8.282571e+04 False \n",
"4 1.136027e+06 False \n",
2025-04-09 22:57:01 +08:00
"... ... ... \n",
2025-05-06 23:42:40 +08:00
"5381 4.988611e+05 False \n",
"5382 3.359782e+05 False \n",
"5383 2.563799e+05 False \n",
"5384 8.893027e+05 False \n",
"5385 4.396263e+06 False \n",
2025-02-12 00:21:33 +08:00
"\n",
2025-05-06 23:42:40 +08:00
"[5386 rows x 19 columns]\n"
2025-02-12 00:21:33 +08:00
]
}
],
2025-05-06 23:42:40 +08:00
"source": [
"all_daily_data_df = pd.concat(all_daily_data, ignore_index=True)\n",
"print(all_daily_data_df)"
]
2025-02-12 00:21:33 +08:00
},
{
2025-02-15 23:33:34 +08:00
"cell_type": "code",
2025-05-06 23:42:40 +08:00
"execution_count": 6,
2025-02-15 23:33:34 +08:00
"id": "28cb78d032671b20",
2025-02-12 00:21:33 +08:00
"metadata": {
"ExecuteTime": {
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:16.881685Z",
"start_time": "2025-04-09T14:58:16.871184Z"
2025-02-12 00:21:33 +08:00
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" ts_code trade_date close turnover_rate turnover_rate_f \\\n",
2025-05-06 23:42:40 +08:00
"23 000820.SZ 20250506 2.04 11.8279 12.1552 \n",
"33 300506.SZ 20250506 3.27 0.6104 0.8597 \n",
"82 839680.BJ 20250506 7.25 34.6648 39.7153 \n",
"105 300159.SZ 20250506 1.83 3.6351 4.0740 \n",
"114 300301.SZ 20250506 1.82 1.3707 1.4819 \n",
"... ... ... ... ... ... \n",
2025-05-06 23:42:40 +08:00
"5259 600243.SH 20250506 2.43 6.7484 8.1172 \n",
"5264 002528.SZ 20250506 2.35 2.0592 4.3961 \n",
"5294 300044.SZ 20250506 3.31 12.8866 13.4490 \n",
"5324 300097.SZ 20250506 4.36 2.5814 3.0107 \n",
"5345 600200.SH 20250506 3.04 0.2013 0.2433 \n",
2025-02-12 00:21:33 +08:00
"\n",
2025-05-06 23:42:40 +08:00
" volume_ratio pe pe_ttm pb ps ps_ttm dv_ratio \\\n",
"23 3.99 NaN NaN 9.0141 10.6452 13.5427 0.0 \n",
"33 0.77 NaN NaN 28.5038 19.4588 19.2499 0.0 \n",
"82 1.96 NaN NaN 7.4242 9.3299 11.0451 NaN \n",
"105 1.34 NaN NaN NaN 4.1337 4.1261 0.0 \n",
"114 1.22 NaN NaN 120.9449 2.9900 3.1074 0.0 \n",
"... ... ... ... ... ... ... ... \n",
"5259 0.73 NaN NaN 1.6685 4.5071 4.6210 0.0 \n",
"5264 1.52 NaN NaN 15.5269 2.9812 3.6083 0.0 \n",
"5294 2.91 NaN NaN 24.3171 17.6463 26.1361 0.0 \n",
"5324 0.99 NaN NaN 2.7137 3.2758 3.8102 0.0 \n",
"5345 0.05 30.7156 NaN 1.2351 1.3543 1.7858 0.0 \n",
2025-02-12 00:21:33 +08:00
"\n",
2025-05-06 23:42:40 +08:00
" dv_ttm total_share float_share free_share total_mv circ_mv \\\n",
"23 NaN 64362.0201 29403.1899 28611.4718 131298.5210 59982.5074 \n",
"33 NaN 69559.6569 57572.5450 40880.9749 227460.0781 188262.2222 \n",
"82 NaN 6699.9900 4689.3344 4093.0077 48574.9275 33997.6744 \n",
"105 NaN 150196.5923 147183.9203 131325.6306 274859.7639 269346.5741 \n",
"114 NaN 82986.8769 78987.6719 73061.8561 151036.1160 143757.5629 \n",
"... ... ... ... ... ... ... \n",
"5259 NaN 43885.0000 43885.0000 36485.0000 106640.5500 106640.5500 \n",
"5264 NaN 119867.5082 104974.0608 49171.2582 281688.6443 246689.0429 \n",
"5294 NaN 76386.9228 76375.7508 73182.1277 252840.7145 252803.7351 \n",
"5324 NaN 28854.9669 27000.9948 23150.5534 125807.6557 117724.3373 \n",
"5345 NaN 71215.1832 71087.9480 58808.3718 216494.1569 216107.3619 \n",
2025-02-12 00:21:33 +08:00
"\n",
2025-05-06 23:42:40 +08:00
" is_st \n",
"23 True \n",
"33 True \n",
"82 True \n",
"105 True \n",
"114 True \n",
"... ... \n",
"5259 True \n",
"5264 True \n",
"5294 True \n",
"5324 True \n",
"5345 True \n",
2025-02-12 00:21:33 +08:00
"\n",
2025-05-06 23:42:40 +08:00
"[196 rows x 19 columns]\n"
2025-02-12 00:21:33 +08:00
]
}
],
2025-05-06 23:42:40 +08:00
"source": [
"print(all_daily_data_df[all_daily_data_df['is_st']])"
]
2025-02-12 00:21:33 +08:00
},
{
2025-02-15 23:33:34 +08:00
"cell_type": "code",
2025-05-06 23:42:40 +08:00
"execution_count": 7,
2025-02-15 23:33:34 +08:00
"id": "692b58674b7462c9",
2025-02-12 00:21:33 +08:00
"metadata": {
"ExecuteTime": {
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:17.773453Z",
"start_time": "2025-04-09T14:58:16.903459Z"
2025-02-12 00:21:33 +08:00
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"所有每日基础数据获取并保存完毕!\n"
]
}
],
2025-05-06 23:42:40 +08:00
"source": [
"# 将数据保存为 HDF5 文件table 格式)\n",
"all_daily_data_df.to_hdf(h5_filename, key='daily_basic', mode='a', format='table', append=True, data_columns=True)\n",
"\n",
"print(\"所有每日基础数据获取并保存完毕!\")\n"
]
2025-02-12 00:21:33 +08:00
},
{
2025-02-15 23:33:34 +08:00
"cell_type": "code",
2025-05-06 23:42:40 +08:00
"execution_count": 8,
2025-02-15 23:33:34 +08:00
"id": "d7a773fc20293477",
2025-02-12 00:21:33 +08:00
"metadata": {
"ExecuteTime": {
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:24.305403Z",
"start_time": "2025-04-09T14:58:17.816332Z"
2025-02-12 00:21:33 +08:00
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
2025-05-06 23:42:40 +08:00
"Index: 8604524 entries, 0 to 5385\n",
2025-02-12 00:21:33 +08:00
"Data columns (total 3 columns):\n",
" # Column Dtype \n",
"--- ------ ----- \n",
" 0 ts_code object\n",
" 1 trade_date object\n",
" 2 is_st bool \n",
"dtypes: bool(1), object(2)\n",
2025-05-06 23:42:40 +08:00
"memory usage: 205.1+ MB\n",
2025-02-12 00:21:33 +08:00
"None\n"
]
}
],
2025-05-06 23:42:40 +08:00
"source": [
"with pd.HDFStore(h5_filename, mode='r') as store:\n",
" df = store[key][['ts_code', 'trade_date', 'is_st']]\n",
" print(df.info())"
]
2025-02-12 00:21:33 +08:00
}
],
"metadata": {
"kernelspec": {
2025-05-06 23:42:40 +08:00
"display_name": "new_trader",
2025-02-12 00:21:33 +08:00
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
2025-03-31 23:08:03 +08:00
"version": "3.11.11"
2025-02-12 00:21:33 +08:00
}
},
"nbformat": 4,
"nbformat_minor": 5
}