liaozhaorun
e407225d29
- 添加 is_scheduled_reconnecting 标志位协调重连逻辑 - 增强定时重连任务的日志前缀便于追踪 - 改进异常处理和资源清理日志 - 优化代码格式和注释
292 lines
10 KiB
Plaintext
292 lines
10 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "b94bb1f2-5332-485e-ae1b-eea01f938106",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-04-09T14:57:40.184418Z",
|
||
"start_time": "2025-04-09T14:57:39.137312Z"
|
||
}
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"import tushare as ts\n",
|
||
"\n",
|
||
"ts.set_token('3a0741c702ee7e5e5f2bf1f0846bafaafe4e320833240b2a7e4a685f')\n",
|
||
"pro = ts.pro_api()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "742c29d453b9bb38",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-04-09T14:58:10.515830Z",
|
||
"start_time": "2025-04-09T14:57:40.190466Z"
|
||
}
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"<class 'pandas.core.frame.DataFrame'>\n",
|
||
"Index: 9387872 entries, 0 to 25863\n",
|
||
"Data columns (total 2 columns):\n",
|
||
" # Column Dtype \n",
|
||
"--- ------ ----- \n",
|
||
" 0 ts_code object\n",
|
||
" 1 trade_date object\n",
|
||
"dtypes: object(2)\n",
|
||
"memory usage: 214.9+ MB\n",
|
||
"None\n",
|
||
"20260130\n",
|
||
"start_date: 20260202\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"import time\n",
|
||
"\n",
|
||
"h5_filename = '/mnt/d/PyProject/NewStock/data/money_flow.h5'\n",
|
||
"key = '/money_flow'\n",
|
||
"max_date = None\n",
|
||
"with pd.HDFStore(h5_filename, mode='r') as store:\n",
|
||
" df = store[key][['ts_code', 'trade_date']]\n",
|
||
" print(df.info())\n",
|
||
" max_date = df['trade_date'].max()\n",
|
||
"\n",
|
||
"print(max_date)\n",
|
||
"trade_cal = pro.trade_cal(exchange='', start_date='20170101', end_date='20260310')\n",
|
||
"trade_cal = trade_cal[trade_cal['is_open'] == 1] # 只保留交易日\n",
|
||
"trade_dates = trade_cal[trade_cal['cal_date'] > max_date]['cal_date'].tolist()\n",
|
||
"start_date = min(trade_dates)\n",
|
||
"print(f'start_date: {start_date}')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "679ce40e-8d62-4887-970c-e1d8cbdeee6b",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-04-09T14:58:17.197319Z",
|
||
"start_time": "2025-04-09T14:58:10.724923Z"
|
||
},
|
||
"scrolled": true
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"任务 20260310 完成\n",
|
||
"任务 20260309 完成\n",
|
||
"任务 20260306 完成\n",
|
||
"任务 20260305 完成\n",
|
||
"任务 20260304 完成\n",
|
||
"任务 20260303 完成\n",
|
||
"任务 20260302 完成\n",
|
||
"任务 20260227 完成\n",
|
||
"任务 20260226 完成\n",
|
||
"任务 20260225 完成\n",
|
||
"任务 20260224 完成\n",
|
||
"任务 20260213 完成\n",
|
||
"任务 20260212 完成\n",
|
||
"任务 20260211 完成\n",
|
||
"任务 20260210 完成\n",
|
||
"任务 20260209 完成\n",
|
||
"任务 20260206 完成\n",
|
||
"任务 20260205 完成\n",
|
||
"任务 20260204 完成\n",
|
||
"任务 20260203 完成\n",
|
||
"任务 20260202 完成\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from concurrent.futures import ThreadPoolExecutor, as_completed\n",
|
||
"\n",
|
||
"all_daily_data = []\n",
|
||
"\n",
|
||
"# API 调用计数和时间控制变量\n",
|
||
"api_call_count = 0\n",
|
||
"batch_start_time = time.time()\n",
|
||
"\n",
|
||
"\n",
|
||
"def get_data(trade_date):\n",
|
||
" time.sleep(0.1)\n",
|
||
" money_flow_data = pro.moneyflow(trade_date=trade_date)\n",
|
||
" if money_flow_data is not None and not money_flow_data.empty:\n",
|
||
" return money_flow_data\n",
|
||
"\n",
|
||
"\n",
|
||
"with ThreadPoolExecutor(max_workers=2) as executor:\n",
|
||
" future_to_date = {executor.submit(get_data, td): td for td in trade_dates}\n",
|
||
"\n",
|
||
" for future in as_completed(future_to_date):\n",
|
||
" trade_date = future_to_date[future] # 获取对应的交易日期\n",
|
||
" try:\n",
|
||
" result = future.result() # 获取任务执行的结果\n",
|
||
" all_daily_data.append(result)\n",
|
||
" print(f\"任务 {trade_date} 完成\")\n",
|
||
" except Exception as e:\n",
|
||
" print(f\"获取 {trade_date} 数据时出错: {e}\")\n",
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "9af80516849d4e80",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-04-09T14:58:17.214168Z",
|
||
"start_time": "2025-04-09T14:58:17.210734Z"
|
||
}
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"all_daily_data_df = pd.concat(all_daily_data, ignore_index=True)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "a2b05187-437f-4053-bc43-bd80d4cf8b0e",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-04-09T14:58:19.633456Z",
|
||
"start_time": "2025-04-09T14:58:17.229837Z"
|
||
}
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"所有每日基础数据获取并保存完毕！\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"\n",
|
||
"# 将所有数据合并为一个 DataFrame\n",
|
||
"\n",
|
||
"# 将数据保存为 HDF5 文件（table 格式）\n",
|
||
"all_daily_data_df.to_hdf(h5_filename, key='money_flow', mode='a', format='table', append=True, data_columns=True)\n",
|
||
"\n",
|
||
"print(\"所有每日基础数据获取并保存完毕！\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "e6f2a2fe",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" ts_code trade_date buy_sm_vol buy_sm_amount sell_sm_vol \\\n",
|
||
"0 002397.SZ 20260206 143016 6566.04 142640 \n",
|
||
"1 603882.SH 20260206 32234 10009.25 31819 \n",
|
||
"2 002824.SZ 20260206 58407 13856.33 47164 \n",
|
||
"3 603379.SH 20260206 31545 21744.99 30955 \n",
|
||
"4 600223.SH 20260206 73778 5582.50 67552 \n",
|
||
"... ... ... ... ... ... \n",
|
||
"25871 002774.SZ 20260202 32854 3675.61 20566 \n",
|
||
"25872 300188.SZ 20260202 54005 7416.58 48176 \n",
|
||
"25873 688173.SH 20260202 64149 10373.20 68049 \n",
|
||
"25874 300145.SZ 20260202 132990 6043.37 133529 \n",
|
||
"25875 300205.SZ 20260202 8361 364.75 6236 \n",
|
||
"\n",
|
||
" sell_sm_amount buy_md_vol buy_md_amount sell_md_vol sell_md_amount \\\n",
|
||
"0 6550.08 114346 5247.29 121777 5580.76 \n",
|
||
"1 9880.13 15560 4828.56 14519 4506.56 \n",
|
||
"2 11195.49 27988 6622.95 30015 7118.27 \n",
|
||
"3 21323.49 21708 14968.66 21044 14503.09 \n",
|
||
"4 5110.61 53914 4075.02 53354 4037.48 \n",
|
||
"... ... ... ... ... ... \n",
|
||
"25871 2300.99 20761 2320.61 21785 2437.00 \n",
|
||
"25872 6623.07 52821 7250.13 53449 7347.52 \n",
|
||
"25873 11000.03 35320 5706.28 30270 4895.56 \n",
|
||
"25874 6066.65 114646 5208.71 97260 4422.03 \n",
|
||
"25875 272.25 9374 408.60 9827 427.93 \n",
|
||
"\n",
|
||
" buy_lg_vol buy_lg_amount sell_lg_vol sell_lg_amount buy_elg_vol \\\n",
|
||
"0 85477 3926.44 87204 3989.37 28400 \n",
|
||
"1 5474 1697.68 6390 1982.08 721 \n",
|
||
"2 11723 2778.17 15330 3630.71 3939 \n",
|
||
"3 13069 9000.74 15789 10882.14 7583 \n",
|
||
"4 27279 2062.59 22583 1709.48 10118 \n",
|
||
"... ... ... ... ... ... \n",
|
||
"25871 8975 1002.99 18793 2100.36 0 \n",
|
||
"25872 31050 4271.36 35903 4923.97 7655 \n",
|
||
"25873 11829 1915.52 12687 2050.68 2000 \n",
|
||
"25874 81164 3682.66 113414 5149.25 41421 \n",
|
||
"25875 6887 302.35 8559 375.51 0 \n",
|
||
"\n",
|
||
" buy_elg_amount sell_elg_vol sell_elg_amount net_mf_vol \\\n",
|
||
"0 1293.83 19618 913.39 42201 \n",
|
||
"1 223.34 1261 390.07 -823 \n",
|
||
"2 935.53 9548 2248.51 2997 \n",
|
||
"3 5221.72 6117 4227.39 6215 \n",
|
||
"4 766.07 21600 1628.62 -9839 \n",
|
||
"... ... ... ... ... \n",
|
||
"25871 0.00 1446 160.86 -20182 \n",
|
||
"25872 1053.35 8003 1096.85 -12903 \n",
|
||
"25873 319.00 2291 367.73 -5670 \n",
|
||
"25874 1881.32 26018 1178.15 -70082 \n",
|
||
"25875 0.00 0 0.00 -1678 \n",
|
||
"\n",
|
||
" net_mf_amount \n",
|
||
"0 1943.09 \n",
|
||
"1 -246.33 \n",
|
||
"2 705.13 \n",
|
||
"3 4367.94 \n",
|
||
"4 -741.40 \n",
|
||
"... ... \n",
|
||
"25871 -2251.19 \n",
|
||
"25872 -1769.73 \n",
|
||
"25873 -918.58 \n",
|
||
"25874 -3146.31 \n",
|
||
"25875 -68.42 \n",
|
||
"\n",
|
||
"[25876 rows x 20 columns]\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(all_daily_data_df)"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "stock",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.12.11"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|