{ "cells": [ { "cell_type": "code", "id": "500802dc-7a20-48b7-a470-a4bae3ec534b", "metadata": { "ExecuteTime": { "end_time": "2025-02-11T15:18:36.892437Z", "start_time": "2025-02-11T15:18:36.020822Z" } }, "source": [ "import tushare as ts\n", "\n", "ts.set_token('3a0741c702ee7e5e5f2bf1f0846bafaafe4e320833240b2a7e4a685f')\n", "pro = ts.pro_api()" ], "outputs": [], "execution_count": 1 }, { "cell_type": "code", "id": "5a84bc9da6d54868", "metadata": { "ExecuteTime": { "end_time": "2025-02-11T15:20:12.573607Z", "start_time": "2025-02-11T15:20:00.110127Z" } }, "source": [ "import pandas as pd\n", "import time\n", "\n", "h5_filename = '../../../data/stk_limit.h5'\n", "key = '/stk_limit'\n", "max_date = None\n", "with pd.HDFStore(h5_filename, mode='r') as store:\n", " df = store[key][['ts_code', 'trade_date']]\n", " print(df.sort_values(by='trade_date', ascending=True).tail())\n", " print(df.info())\n", " max_date = df['trade_date'].max()\n", "\n", "print(max_date)\n", "trade_cal = pro.trade_cal(exchange='', start_date='20170101', end_date='20250220')\n", "trade_cal = trade_cal[trade_cal['is_open'] == 1] # 只保留交易日\n", "trade_dates = trade_cal[trade_cal['cal_date'] > max_date]['cal_date'].tolist()\n", "start_date = min(trade_dates)\n", "print(start_date)" ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " ts_code trade_date\n", "4682 600310.SH 20250211\n", "4683 600312.SH 20250211\n", "4684 600313.SH 20250211\n", "4673 600299.SH 20250211\n", "0 000001.SZ 20250211\n", "\n", "Index: 10040878 entries, 0 to 10040877\n", "Data columns (total 2 columns):\n", " # Column Dtype \n", "--- ------ ----- \n", " 0 ts_code object\n", " 1 trade_date object\n", "dtypes: object(2)\n", "memory usage: 229.8+ MB\n", "None\n", "20250211\n", "20250212\n" ] } ], "execution_count": 5 }, { "cell_type": "code", "id": "bb3191de-27a2-4c89-a3b5-32a0d7b9496f", "metadata": { "scrolled": true, "ExecuteTime": { "end_time": "2025-02-11T15:21:27.831699Z", "start_time": "2025-02-11T15:21:26.665039Z" } }, "source": [ "from concurrent.futures import ThreadPoolExecutor, as_completed\n", "\n", "all_daily_data = []\n", "\n", "# API 调用计数和时间控制变量\n", "api_call_count = 0\n", "batch_start_time = time.time()\n", "\n", "\n", "def get_data(trade_date):\n", " time.sleep(0.1)\n", " stk_limit_data = pro.stk_limit(trade_date=trade_date)\n", " if stk_limit_data is not None and not stk_limit_data.empty:\n", " return stk_limit_data\n", "\n", "\n", "with ThreadPoolExecutor(max_workers=2) as executor:\n", " future_to_date = {executor.submit(get_data, td): td for td in trade_dates}\n", "\n", " for future in as_completed(future_to_date):\n", " trade_date = future_to_date[future] # 获取对应的交易日期\n", " try:\n", " result = future.result() # 获取任务执行的结果\n", " if result is not None:\n", " all_daily_data.append(result)\n", " print(f\"任务 {trade_date} 完成\")\n", " except Exception as e:\n", " print(f\"获取 {trade_date} 数据时出错: {e}\")\n", "\n" ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "任务 20250220 完成\n", "任务 20250219 完成\n", "任务 20250217 完成\n", "任务 20250218 完成\n", "任务 20250214 完成\n", "任务 20250213 完成\n", "任务 20250212 完成\n" ] } ], "execution_count": 10 }, { "metadata": { "ExecuteTime": { "end_time": "2025-02-11T15:21:29.294283Z", "start_time": "2025-02-11T15:21:29.247112Z" } }, "cell_type": "code", "source": [ "print(all_daily_data)\n", "# 将所有数据合并为一个 DataFrame\n", "all_daily_data_df = pd.concat(all_daily_data, ignore_index=True)" ], "id": "96a81aa5890ea3c3", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[]\n" ] }, { "ename": "ValueError", "evalue": "No objects to concatenate", "output_type": "error", "traceback": [ "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", "\u001B[1;31mValueError\u001B[0m Traceback (most recent call last)", "Cell \u001B[1;32mIn[11], line 3\u001B[0m\n\u001B[0;32m 1\u001B[0m \u001B[38;5;28mprint\u001B[39m(all_daily_data)\n\u001B[0;32m 2\u001B[0m \u001B[38;5;66;03m# 将所有数据合并为一个 DataFrame\u001B[39;00m\n\u001B[1;32m----> 3\u001B[0m all_daily_data_df \u001B[38;5;241m=\u001B[39m \u001B[43mpd\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mconcat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mall_daily_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mignore_index\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m)\u001B[49m\n", "File \u001B[1;32mE:\\Python\\anaconda\\envs\\try_trader\\lib\\site-packages\\pandas\\core\\reshape\\concat.py:372\u001B[0m, in \u001B[0;36mconcat\u001B[1;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001B[0m\n\u001B[0;32m 369\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m copy \u001B[38;5;129;01mand\u001B[39;00m using_copy_on_write():\n\u001B[0;32m 370\u001B[0m copy \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m--> 372\u001B[0m op \u001B[38;5;241m=\u001B[39m \u001B[43m_Concatenator\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 373\u001B[0m \u001B[43m \u001B[49m\u001B[43mobjs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 374\u001B[0m \u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 375\u001B[0m \u001B[43m \u001B[49m\u001B[43mignore_index\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mignore_index\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 376\u001B[0m \u001B[43m \u001B[49m\u001B[43mjoin\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mjoin\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 377\u001B[0m \u001B[43m \u001B[49m\u001B[43mkeys\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mkeys\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 378\u001B[0m \u001B[43m \u001B[49m\u001B[43mlevels\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlevels\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 379\u001B[0m \u001B[43m \u001B[49m\u001B[43mnames\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mnames\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 380\u001B[0m \u001B[43m \u001B[49m\u001B[43mverify_integrity\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mverify_integrity\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 381\u001B[0m \u001B[43m \u001B[49m\u001B[43mcopy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcopy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 382\u001B[0m \u001B[43m \u001B[49m\u001B[43msort\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msort\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 383\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 385\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m op\u001B[38;5;241m.\u001B[39mget_result()\n", "File \u001B[1;32mE:\\Python\\anaconda\\envs\\try_trader\\lib\\site-packages\\pandas\\core\\reshape\\concat.py:429\u001B[0m, in \u001B[0;36m_Concatenator.__init__\u001B[1;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001B[0m\n\u001B[0;32m 426\u001B[0m objs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m(objs)\n\u001B[0;32m 428\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(objs) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[1;32m--> 429\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNo objects to concatenate\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m 431\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m keys \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m 432\u001B[0m objs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m(com\u001B[38;5;241m.\u001B[39mnot_none(\u001B[38;5;241m*\u001B[39mobjs))\n", "\u001B[1;31mValueError\u001B[0m: No objects to concatenate" ] } ], "execution_count": 11 }, { "cell_type": "code", "id": "ad9733a1-2f42-43ee-a98c-0bf699304c21", "metadata": { "ExecuteTime": { "end_time": "2025-02-11T15:20:37.999493Z", "start_time": "2025-02-11T15:20:37.375220Z" } }, "source": [ "\n", "\n", "# 将数据保存为 HDF5 文件（table 格式）\n", "all_daily_data_df.to_hdf(h5_filename, key='stk_limit', mode='a', format='table', append=True, data_columns=True)\n", "\n", "print(\"所有每日基础数据获取并保存完毕！\")" ], "outputs": [ { "ename": "ValueError", "evalue": "All objects passed were None", "output_type": "error", "traceback": [ "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", "\u001B[1;31mValueError\u001B[0m Traceback (most recent call last)", "Cell \u001B[1;32mIn[7], line 2\u001B[0m\n\u001B[0;32m 1\u001B[0m \u001B[38;5;66;03m# 将所有数据合并为一个 DataFrame\u001B[39;00m\n\u001B[1;32m----> 2\u001B[0m all_daily_data_df \u001B[38;5;241m=\u001B[39m \u001B[43mpd\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mconcat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mall_daily_data\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mignore_index\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[0;32m 4\u001B[0m \u001B[38;5;66;03m# 将数据保存为 HDF5 文件（table 格式）\u001B[39;00m\n\u001B[0;32m 5\u001B[0m all_daily_data_df\u001B[38;5;241m.\u001B[39mto_hdf(h5_filename, key\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mstk_limit\u001B[39m\u001B[38;5;124m'\u001B[39m, mode\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124ma\u001B[39m\u001B[38;5;124m'\u001B[39m, \u001B[38;5;28mformat\u001B[39m\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mtable\u001B[39m\u001B[38;5;124m'\u001B[39m, append\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m, data_columns\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n", "File \u001B[1;32mE:\\Python\\anaconda\\envs\\try_trader\\lib\\site-packages\\pandas\\core\\reshape\\concat.py:372\u001B[0m, in \u001B[0;36mconcat\u001B[1;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001B[0m\n\u001B[0;32m 369\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m copy \u001B[38;5;129;01mand\u001B[39;00m using_copy_on_write():\n\u001B[0;32m 370\u001B[0m copy \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m--> 372\u001B[0m op \u001B[38;5;241m=\u001B[39m \u001B[43m_Concatenator\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 373\u001B[0m \u001B[43m \u001B[49m\u001B[43mobjs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 374\u001B[0m \u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 375\u001B[0m \u001B[43m \u001B[49m\u001B[43mignore_index\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mignore_index\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 376\u001B[0m \u001B[43m \u001B[49m\u001B[43mjoin\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mjoin\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 377\u001B[0m \u001B[43m \u001B[49m\u001B[43mkeys\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mkeys\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 378\u001B[0m \u001B[43m \u001B[49m\u001B[43mlevels\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlevels\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 379\u001B[0m \u001B[43m \u001B[49m\u001B[43mnames\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mnames\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 380\u001B[0m \u001B[43m \u001B[49m\u001B[43mverify_integrity\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mverify_integrity\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 381\u001B[0m \u001B[43m \u001B[49m\u001B[43mcopy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcopy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 382\u001B[0m \u001B[43m \u001B[49m\u001B[43msort\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msort\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 383\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 385\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m op\u001B[38;5;241m.\u001B[39mget_result()\n", "File \u001B[1;32mE:\\Python\\anaconda\\envs\\try_trader\\lib\\site-packages\\pandas\\core\\reshape\\concat.py:452\u001B[0m, in \u001B[0;36m_Concatenator.__init__\u001B[1;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001B[0m\n\u001B[0;32m 449\u001B[0m keys \u001B[38;5;241m=\u001B[39m Index(clean_keys, name\u001B[38;5;241m=\u001B[39mname, dtype\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mgetattr\u001B[39m(keys, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdtype\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m))\n\u001B[0;32m 451\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(objs) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[1;32m--> 452\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mAll objects passed were None\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m 454\u001B[0m \u001B[38;5;66;03m# figure out what our result ndim is going to be\u001B[39;00m\n\u001B[0;32m 455\u001B[0m ndims \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mset\u001B[39m()\n", "\u001B[1;31mValueError\u001B[0m: All objects passed were None" ] } ], "execution_count": 7 }, { "cell_type": "code", "execution_count": null, "id": "7e777f1f-4d54-4a74-b916-691ede6af055", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.19" } }, "nbformat": 4, "nbformat_minor": 5 }