239 lines
15 KiB
Plaintext
239 lines
15 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"id": "500802dc-7a20-48b7-a470-a4bae3ec534b",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-02-11T15:18:36.892437Z",
|
||
"start_time": "2025-02-11T15:18:36.020822Z"
|
||
}
|
||
},
|
||
"source": [
|
||
"import tushare as ts\n",
|
||
"\n",
|
||
"ts.set_token('3a0741c702ee7e5e5f2bf1f0846bafaafe4e320833240b2a7e4a685f')\n",
|
||
"pro = ts.pro_api()"
|
||
],
|
||
"outputs": [],
|
||
"execution_count": 1
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"id": "5a84bc9da6d54868",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-02-11T15:20:12.573607Z",
|
||
"start_time": "2025-02-11T15:20:00.110127Z"
|
||
}
|
||
},
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"import time\n",
|
||
"\n",
|
||
"h5_filename = '../../../data/stk_limit.h5'\n",
|
||
"key = '/stk_limit'\n",
|
||
"max_date = None\n",
|
||
"with pd.HDFStore(h5_filename, mode='r') as store:\n",
|
||
" df = store[key][['ts_code', 'trade_date']]\n",
|
||
" print(df.sort_values(by='trade_date', ascending=True).tail())\n",
|
||
" print(df.info())\n",
|
||
" max_date = df['trade_date'].max()\n",
|
||
"\n",
|
||
"print(max_date)\n",
|
||
"trade_cal = pro.trade_cal(exchange='', start_date='20170101', end_date='20250220')\n",
|
||
"trade_cal = trade_cal[trade_cal['is_open'] == 1] # 只保留交易日\n",
|
||
"trade_dates = trade_cal[trade_cal['cal_date'] > max_date]['cal_date'].tolist()\n",
|
||
"start_date = min(trade_dates)\n",
|
||
"print(start_date)"
|
||
],
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" ts_code trade_date\n",
|
||
"4682 600310.SH 20250211\n",
|
||
"4683 600312.SH 20250211\n",
|
||
"4684 600313.SH 20250211\n",
|
||
"4673 600299.SH 20250211\n",
|
||
"0 000001.SZ 20250211\n",
|
||
"<class 'pandas.core.frame.DataFrame'>\n",
|
||
"Index: 10040878 entries, 0 to 10040877\n",
|
||
"Data columns (total 2 columns):\n",
|
||
" # Column Dtype \n",
|
||
"--- ------ ----- \n",
|
||
" 0 ts_code object\n",
|
||
" 1 trade_date object\n",
|
||
"dtypes: object(2)\n",
|
||
"memory usage: 229.8+ MB\n",
|
||
"None\n",
|
||
"20250211\n",
|
||
"20250212\n"
|
||
]
|
||
}
|
||
],
|
||
"execution_count": 5
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"id": "bb3191de-27a2-4c89-a3b5-32a0d7b9496f",
|
||
"metadata": {
|
||
"scrolled": true,
|
||
"ExecuteTime": {
|
||
"end_time": "2025-02-11T15:21:27.831699Z",
|
||
"start_time": "2025-02-11T15:21:26.665039Z"
|
||
}
|
||
},
|
||
"source": [
|
||
"from concurrent.futures import ThreadPoolExecutor, as_completed\n",
|
||
"\n",
|
||
"all_daily_data = []\n",
|
||
"\n",
|
||
"# API 调用计数和时间控制变量\n",
|
||
"api_call_count = 0\n",
|
||
"batch_start_time = time.time()\n",
|
||
"\n",
|
||
"\n",
|
||
"def get_data(trade_date):\n",
|
||
" time.sleep(0.1)\n",
|
||
" stk_limit_data = pro.stk_limit(trade_date=trade_date)\n",
|
||
" if stk_limit_data is not None and not stk_limit_data.empty:\n",
|
||
" return stk_limit_data\n",
|
||
"\n",
|
||
"\n",
|
||
"with ThreadPoolExecutor(max_workers=2) as executor:\n",
|
||
" future_to_date = {executor.submit(get_data, td): td for td in trade_dates}\n",
|
||
"\n",
|
||
" for future in as_completed(future_to_date):\n",
|
||
" trade_date = future_to_date[future] # 获取对应的交易日期\n",
|
||
" try:\n",
|
||
" result = future.result() # 获取任务执行的结果\n",
|
||
" if result is not None:\n",
|
||
" all_daily_data.append(result)\n",
|
||
" print(f\"任务 {trade_date} 完成\")\n",
|
||
" except Exception as e:\n",
|
||
" print(f\"获取 {trade_date} 数据时出错: {e}\")\n",
|
||
"\n"
|
||
],
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"任务 20250220 完成\n",
|
||
"任务 20250219 完成\n",
|
||
"任务 20250217 完成\n",
|
||
"任务 20250218 完成\n",
|
||
"任务 20250214 完成\n",
|
||
"任务 20250213 完成\n",
|
||
"任务 20250212 完成\n"
|
||
]
|
||
}
|
||
],
|
||
"execution_count": 10
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-02-11T15:21:29.294283Z",
|
||
"start_time": "2025-02-11T15:21:29.247112Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"print(all_daily_data)\n",
|
||
"# 将所有数据合并为一个 DataFrame\n",
|
||
"all_daily_data_df = pd.concat(all_daily_data, ignore_index=True)"
|
||
],
|
||
"id": "96a81aa5890ea3c3",
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[]\n"
|
||
]
|
||
},
|
||
{
|
||
"ename": "ValueError",
|
||
"evalue": "No objects to concatenate",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
|
||
"\u001B[1;31mValueError\u001B[0m Traceback (most recent call last)",
|
||
"Cell \u001B[1;32mIn[11], line 3\u001B[0m\n\u001B[0;32m 1\u001B[0m \u001B[38;5;28mprint\u001B[39m(all_daily_data)\n\u001B[0;32m 2\u001B[0m \u001B[38;5;66;03m# 将所有数据合并为一个 DataFrame\u001B[39;00m\n\u001B[1;32m----> 3\u001B[0m all_daily_data_df \u001B[38;5;241m=\u001B[39m \u001B[43mpd\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mconcat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mall_daily_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mignore_index\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m)\u001B[49m\n",
|
||
"File \u001B[1;32mE:\\Python\\anaconda\\envs\\try_trader\\lib\\site-packages\\pandas\\core\\reshape\\concat.py:372\u001B[0m, in \u001B[0;36mconcat\u001B[1;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001B[0m\n\u001B[0;32m 369\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m copy \u001B[38;5;129;01mand\u001B[39;00m using_copy_on_write():\n\u001B[0;32m 370\u001B[0m copy \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m--> 372\u001B[0m op \u001B[38;5;241m=\u001B[39m \u001B[43m_Concatenator\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 373\u001B[0m \u001B[43m \u001B[49m\u001B[43mobjs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 374\u001B[0m \u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 375\u001B[0m \u001B[43m \u001B[49m\u001B[43mignore_index\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mignore_index\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 376\u001B[0m \u001B[43m \u001B[49m\u001B[43mjoin\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mjoin\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 377\u001B[0m \u001B[43m \u001B[49m\u001B[43mkeys\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mkeys\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 378\u001B[0m \u001B[43m \u001B[49m\u001B[43mlevels\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlevels\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 379\u001B[0m \u001B[43m \u001B[49m\u001B[43mnames\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mnames\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 380\u001B[0m \u001B[43m \u001B[49m\u001B[43mverify_integrity\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mverify_integrity\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 381\u001B[0m \u001B[43m \u001B[49m\u001B[43mcopy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcopy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 382\u001B[0m \u001B[43m \u001B[49m\u001B[43msort\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msort\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 383\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 385\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m op\u001B[38;5;241m.\u001B[39mget_result()\n",
|
||
"File \u001B[1;32mE:\\Python\\anaconda\\envs\\try_trader\\lib\\site-packages\\pandas\\core\\reshape\\concat.py:429\u001B[0m, in \u001B[0;36m_Concatenator.__init__\u001B[1;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001B[0m\n\u001B[0;32m 426\u001B[0m objs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m(objs)\n\u001B[0;32m 428\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(objs) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[1;32m--> 429\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNo objects to concatenate\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m 431\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m keys \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m 432\u001B[0m objs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m(com\u001B[38;5;241m.\u001B[39mnot_none(\u001B[38;5;241m*\u001B[39mobjs))\n",
|
||
"\u001B[1;31mValueError\u001B[0m: No objects to concatenate"
|
||
]
|
||
}
|
||
],
|
||
"execution_count": 11
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"id": "ad9733a1-2f42-43ee-a98c-0bf699304c21",
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-02-11T15:20:37.999493Z",
|
||
"start_time": "2025-02-11T15:20:37.375220Z"
|
||
}
|
||
},
|
||
"source": [
|
||
"\n",
|
||
"\n",
|
||
"# 将数据保存为 HDF5 文件（table 格式）\n",
|
||
"all_daily_data_df.to_hdf(h5_filename, key='stk_limit', mode='a', format='table', append=True, data_columns=True)\n",
|
||
"\n",
|
||
"print(\"所有每日基础数据获取并保存完毕！\")"
|
||
],
|
||
"outputs": [
|
||
{
|
||
"ename": "ValueError",
|
||
"evalue": "All objects passed were None",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
|
||
"\u001B[1;31mValueError\u001B[0m Traceback (most recent call last)",
|
||
"Cell \u001B[1;32mIn[7], line 2\u001B[0m\n\u001B[0;32m 1\u001B[0m \u001B[38;5;66;03m# 将所有数据合并为一个 DataFrame\u001B[39;00m\n\u001B[1;32m----> 2\u001B[0m all_daily_data_df \u001B[38;5;241m=\u001B[39m \u001B[43mpd\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mconcat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mall_daily_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mignore_index\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[0;32m 4\u001B[0m \u001B[38;5;66;03m# 将数据保存为 HDF5 文件（table 格式）\u001B[39;00m\n\u001B[0;32m 5\u001B[0m all_daily_data_df\u001B[38;5;241m.\u001B[39mto_hdf(h5_filename, key\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mstk_limit\u001B[39m\u001B[38;5;124m'\u001B[39m, mode\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124ma\u001B[39m\u001B[38;5;124m'\u001B[39m, \u001B[38;5;28mformat\u001B[39m\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mtable\u001B[39m\u001B[38;5;124m'\u001B[39m, append\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m, data_columns\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n",
|
||
"File \u001B[1;32mE:\\Python\\anaconda\\envs\\try_trader\\lib\\site-packages\\pandas\\core\\reshape\\concat.py:372\u001B[0m, in \u001B[0;36mconcat\u001B[1;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001B[0m\n\u001B[0;32m 369\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m copy \u001B[38;5;129;01mand\u001B[39;00m using_copy_on_write():\n\u001B[0;32m 370\u001B[0m copy \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m--> 372\u001B[0m op \u001B[38;5;241m=\u001B[39m \u001B[43m_Concatenator\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 373\u001B[0m \u001B[43m \u001B[49m\u001B[43mobjs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 374\u001B[0m \u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 375\u001B[0m \u001B[43m \u001B[49m\u001B[43mignore_index\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mignore_index\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 376\u001B[0m \u001B[43m \u001B[49m\u001B[43mjoin\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mjoin\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 377\u001B[0m \u001B[43m \u001B[49m\u001B[43mkeys\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mkeys\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 378\u001B[0m \u001B[43m \u001B[49m\u001B[43mlevels\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlevels\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 379\u001B[0m \u001B[43m \u001B[49m\u001B[43mnames\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mnames\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 380\u001B[0m \u001B[43m \u001B[49m\u001B[43mverify_integrity\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mverify_integrity\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 381\u001B[0m \u001B[43m \u001B[49m\u001B[43mcopy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcopy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 382\u001B[0m \u001B[43m \u001B[49m\u001B[43msort\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msort\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 383\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 385\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m op\u001B[38;5;241m.\u001B[39mget_result()\n",
|
||
"File \u001B[1;32mE:\\Python\\anaconda\\envs\\try_trader\\lib\\site-packages\\pandas\\core\\reshape\\concat.py:452\u001B[0m, in \u001B[0;36m_Concatenator.__init__\u001B[1;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001B[0m\n\u001B[0;32m 449\u001B[0m keys \u001B[38;5;241m=\u001B[39m Index(clean_keys, name\u001B[38;5;241m=\u001B[39mname, dtype\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mgetattr\u001B[39m(keys, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdtype\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m))\n\u001B[0;32m 451\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(objs) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[1;32m--> 452\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mAll objects passed were None\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m 454\u001B[0m \u001B[38;5;66;03m# figure out what our result ndim is going to be\u001B[39;00m\n\u001B[0;32m 455\u001B[0m ndims \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mset\u001B[39m()\n",
|
||
"\u001B[1;31mValueError\u001B[0m: All objects passed were None"
|
||
]
|
||
}
|
||
],
|
||
"execution_count": 7
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7e777f1f-4d54-4a74-b916-691ede6af055",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.8.19"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|