liaozhaorun/NewStock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
6387b30d107184e5178ee0335e694416eee4d6a0
NewStock/main/data/update/update_stk_limit.ipynb

284 lines
12 KiB
Plaintext
Raw Normal View History

init
2025-02-12 00:21:33 +08:00
{
"cells": [
{
"cell_type": "code",
Classify2
2025-05-06 23:42:40 +08:00
"execution_count": 1,
init
2025-02-12 00:21:33 +08:00
"id": "500802dc-7a20-48b7-a470-a4bae3ec534b",
"metadata": {
"ExecuteTime": {
RollingRank勉强赚钱
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:57:41.532210Z",
"start_time": "2025-04-09T14:57:40.584930Z"
init
2025-02-12 00:21:33 +08:00
}
},
Classify2
2025-05-06 23:42:40 +08:00
"outputs": [],
init
2025-02-12 00:21:33 +08:00
"source": [
"import tushare as ts\n",
"\n",
"ts.set_token('3a0741c702ee7e5e5f2bf1f0846bafaafe4e320833240b2a7e4a685f')\n",
"pro = ts.pro_api()"
Classify2
2025-05-06 23:42:40 +08:00
]
init
2025-02-12 00:21:33 +08:00
},
{
"cell_type": "code",
Classify2
2025-05-06 23:42:40 +08:00
"execution_count": 2,
init
2025-02-12 00:21:33 +08:00
"id": "5a84bc9da6d54868",
"metadata": {
"ExecuteTime": {
RollingRank勉强赚钱
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:04.911924Z",
"start_time": "2025-04-09T14:57:41.540345Z"
init
2025-02-12 00:21:33 +08:00
}
},
Classify2
2025-05-06 23:42:40 +08:00
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" ts_code trade_date\n",
"4745 600276.SH 20250506\n",
"4746 600278.SH 20250506\n",
"4747 600279.SH 20250506\n",
"4736 600262.SH 20250506\n",
"281 000791.SZ 20250506\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"Index: 10436295 entries, 0 to 113592\n",
"Data columns (total 2 columns):\n",
" # Column Dtype \n",
"--- ------ ----- \n",
" 0 ts_code object\n",
" 1 trade_date object\n",
"dtypes: object(2)\n",
"memory usage: 238.9+ MB\n",
"None\n",
"20250506\n",
"20250507\n"
]
}
],
2.15
2025-02-15 23:33:34 +08:00
"source": [
"import pandas as pd\n",
"import time\n",
"\n",
"h5_filename = '../../../data/stk_limit.h5'\n",
"key = '/stk_limit'\n",
"max_date = None\n",
"with pd.HDFStore(h5_filename, mode='r') as store:\n",
" df = store[key][['ts_code', 'trade_date']]\n",
" print(df.sort_values(by='trade_date', ascending=True).tail())\n",
" print(df.info())\n",
" max_date = df['trade_date'].max()\n",
"\n",
"print(max_date)\n",
Classify2
2025-05-06 23:42:40 +08:00
"trade_cal = pro.trade_cal(exchange='', start_date='20170101', end_date='20250720')\n",
2.15
2025-02-15 23:33:34 +08:00
"trade_cal = trade_cal[trade_cal['is_open'] == 1] # 只保留交易日\n",
"trade_dates = trade_cal[trade_cal['cal_date'] > max_date]['cal_date'].tolist()\n",
"start_date = min(trade_dates)\n",
"print(start_date)"
Classify2
2025-05-06 23:42:40 +08:00
]
(data leak)RollingRank-7.0，赚钱
2025-04-09 22:57:01 +08:00
},
{
"cell_type": "code",
Classify2
2025-05-06 23:42:40 +08:00
"execution_count": 3,
(data leak)RollingRank-7.0，赚钱
2025-04-09 22:57:01 +08:00
"id": "bb3191de-27a2-4c89-a3b5-32a0d7b9496f",
"metadata": {
"ExecuteTime": {
RollingRank勉强赚钱
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:09.342522Z",
"start_time": "2025-04-09T14:58:05.259974Z"
Classify2
2025-05-06 23:42:40 +08:00
},
"scrolled": true
(data leak)RollingRank-7.0，赚钱
2025-04-09 22:57:01 +08:00
},
Classify2
2025-05-06 23:42:40 +08:00
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"任务 20250718 完成\n",
"任务 20250717 完成\n",
"任务 20250715 完成\n",
"任务 20250716 完成\n",
"任务 20250714 完成\n",
"任务 20250711 完成\n",
"任务 20250709 完成\n",
"任务 20250710 完成\n",
"任务 20250708 完成\n",
"任务 20250707 完成\n",
"任务 20250703 完成\n",
"任务 20250704 完成\n",
"任务 20250701 完成\n",
"任务 20250702 完成\n",
"任务 20250630 完成\n",
"任务 20250627 完成\n",
"任务 20250626 完成\n",
"任务 20250625 完成\n",
"任务 20250624 完成\n",
"任务 20250623 完成\n",
"任务 20250620 完成\n",
"任务 20250619 完成\n",
"任务 20250618 完成\n",
"任务 20250617 完成\n",
"任务 20250616 完成\n",
"任务 20250613 完成\n",
"任务 20250612 完成\n",
"任务 20250611 完成\n",
"任务 20250610 完成\n",
"任务 20250609 完成\n",
"任务 20250606 完成\n",
"任务 20250605 完成\n",
"任务 20250604 完成\n",
"任务 20250603 完成\n",
"任务 20250530 完成\n",
"任务 20250529 完成\n",
"任务 20250528 完成\n",
"任务 20250527 完成\n",
"任务 20250526 完成\n",
"任务 20250523 完成\n",
"任务 20250522 完成\n",
"任务 20250521 完成\n",
"任务 20250520 完成\n",
"任务 20250519 完成\n",
"任务 20250516 完成\n",
"任务 20250515 完成\n",
"任务 20250514 完成\n",
"任务 20250513 完成\n",
"任务 20250512 完成\n",
"任务 20250509 完成\n",
"任务 20250508 完成\n",
"任务 20250507 完成\n"
]
}
],
init
2025-02-12 00:21:33 +08:00
"source": [
"from concurrent.futures import ThreadPoolExecutor, as_completed\n",
"\n",
"all_daily_data = []\n",
"\n",
"# API 调用计数和时间控制变量\n",
"api_call_count = 0\n",
"batch_start_time = time.time()\n",
"\n",
"\n",
"def get_data(trade_date):\n",
" time.sleep(0.1)\n",
" stk_limit_data = pro.stk_limit(trade_date=trade_date)\n",
" if stk_limit_data is not None and not stk_limit_data.empty:\n",
" return stk_limit_data\n",
"\n",
"\n",
"with ThreadPoolExecutor(max_workers=2) as executor:\n",
" future_to_date = {executor.submit(get_data, td): td for td in trade_dates}\n",
"\n",
" for future in as_completed(future_to_date):\n",
" trade_date = future_to_date[future] # 获取对应的交易日期\n",
" try:\n",
" result = future.result() # 获取任务执行的结果\n",
" if result is not None:\n",
" all_daily_data.append(result)\n",
" print(f\"任务 {trade_date} 完成\")\n",
" except Exception as e:\n",
" print(f\"获取 {trade_date} 数据时出错: {e}\")\n",
"\n"
Classify2
2025-05-06 23:42:40 +08:00
]
init
2025-02-12 00:21:33 +08:00
},
{
2.15
2025-02-15 23:33:34 +08:00
"cell_type": "code",
Classify2
2025-05-06 23:42:40 +08:00
"execution_count": 4,
2.15
2025-02-15 23:33:34 +08:00
"id": "96a81aa5890ea3c3",
init
2025-02-12 00:21:33 +08:00
"metadata": {
"ExecuteTime": {
RollingRank勉强赚钱
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:09.353560Z",
"start_time": "2025-04-09T14:58:09.346528Z"
init
2025-02-12 00:21:33 +08:00
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
Classify2
2025-05-06 23:42:40 +08:00
"[]\n"
]
},
{
"ename": "ValueError",
"evalue": "No objects to concatenate",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[1;32mIn[4], line 3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28mprint\u001b[39m(all_daily_data)\n\u001b[0;32m 2\u001b[0m \u001b[38;5;66;03m# 将所有数据合并为一个 DataFrame\u001b[39;00m\n\u001b[1;32m----> 3\u001b[0m all_daily_data_df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat(all_daily_data, ignore_index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
"File \u001b[1;32me:\\Python\\anaconda\\envs\\new_trader\\Lib\\site-packages\\pandas\\core\\reshape\\concat.py:382\u001b[0m, in \u001b[0;36mconcat\u001b[1;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[0;32m 379\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m copy \u001b[38;5;129;01mand\u001b[39;00m using_copy_on_write():\n\u001b[0;32m 380\u001b[0m copy \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m--> 382\u001b[0m op \u001b[38;5;241m=\u001b[39m _Concatenator(\n\u001b[0;32m 383\u001b[0m objs,\n\u001b[0;32m 384\u001b[0m axis\u001b[38;5;241m=\u001b[39maxis,\n\u001b[0;32m 385\u001b[0m ignore_index\u001b[38;5;241m=\u001b[39mignore_index,\n\u001b[0;32m 386\u001b[0m join\u001b[38;5;241m=\u001b[39mjoin,\n\u001b[0;32m 387\u001b[0m keys\u001b[38;5;241m=\u001b[39mkeys,\n\u001b[0;32m 388\u001b[0m levels\u001b[38;5;241m=\u001b[39mlevels,\n\u001b[0;32m 389\u001b[0m names\u001b[38;5;241m=\u001b[39mnames,\n\u001b[0;32m 390\u001b[0m verify_integrity\u001b[38;5;241m=\u001b[39mverify_integrity,\n\u001b[0;32m 391\u001b[0m copy\u001b[38;5;241m=\u001b[39mcopy,\n\u001b[0;32m 392\u001b[0m sort\u001b[38;5;241m=\u001b[39msort,\n\u001b[0;32m 393\u001b[0m )\n\u001b[0;32m 395\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m op\u001b[38;5;241m.\u001b[39mget_result()\n",
"File \u001b[1;32me:\\Python\\anaconda\\envs\\new_trader\\Lib\\site-packages\\pandas\\core\\reshape\\concat.py:445\u001b[0m, in \u001b[0;36m_Concatenator.__init__\u001b[1;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001b[0m\n\u001b[0;32m 442\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverify_integrity \u001b[38;5;241m=\u001b[39m verify_integrity\n\u001b[0;32m 443\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcopy \u001b[38;5;241m=\u001b[39m copy\n\u001b[1;32m--> 445\u001b[0m objs, keys \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_clean_keys_and_objs(objs, keys)\n\u001b[0;32m 447\u001b[0m \u001b[38;5;66;03m# figure out what our result ndim is going to be\u001b[39;00m\n\u001b[0;32m 448\u001b[0m ndims \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_ndims(objs)\n",
"File \u001b[1;32me:\\Python\\anaconda\\envs\\new_trader\\Lib\\site-packages\\pandas\\core\\reshape\\concat.py:507\u001b[0m, in \u001b[0;36m_Concatenator._clean_keys_and_objs\u001b[1;34m(self, objs, keys)\u001b[0m\n\u001b[0;32m 504\u001b[0m objs_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(objs)\n\u001b[0;32m 506\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(objs_list) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m--> 507\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo objects to concatenate\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 509\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m keys \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m 510\u001b[0m objs_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(com\u001b[38;5;241m.\u001b[39mnot_none(\u001b[38;5;241m*\u001b[39mobjs_list))\n",
"\u001b[1;31mValueError\u001b[0m: No objects to concatenate"
init
2025-02-12 00:21:33 +08:00
]
}
],
Classify2
2025-05-06 23:42:40 +08:00
"source": [
"print(all_daily_data)\n",
"# 将所有数据合并为一个 DataFrame\n",
"all_daily_data_df = pd.concat(all_daily_data, ignore_index=True)"
]
init
2025-02-12 00:21:33 +08:00
},
{
"cell_type": "code",
Classify2
2025-05-06 23:42:40 +08:00
"execution_count": null,
init
2025-02-12 00:21:33 +08:00
"id": "ad9733a1-2f42-43ee-a98c-0bf699304c21",
"metadata": {
"ExecuteTime": {
RollingRank勉强赚钱
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:09.674078Z",
"start_time": "2025-04-09T14:58:09.366441Z"
init
2025-02-12 00:21:33 +08:00
}
},
2.15
2025-02-15 23:33:34 +08:00
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"所有每日基础数据获取并保存完毕！\n"
]
}
],
Classify2
2025-05-06 23:42:40 +08:00
"source": [
"\n",
"\n",
"# 将数据保存为 HDF5 文件（table 格式）\n",
"all_daily_data_df.to_hdf(h5_filename, key='stk_limit', mode='a', format='table', append=True, data_columns=True)\n",
"\n",
"print(\"所有每日基础数据获取并保存完毕！\")"
]
init
2025-02-12 00:21:33 +08:00
},
{
"cell_type": "code",
Classify2
2025-05-06 23:42:40 +08:00
"execution_count": null,
init
2025-02-12 00:21:33 +08:00
"id": "7e777f1f-4d54-4a74-b916-691ede6af055",
(exception)勉强赚钱rank
2025-03-31 23:08:03 +08:00
"metadata": {
"ExecuteTime": {
RollingRank勉强赚钱
2025-04-10 23:17:22 +08:00
"end_time": "2025-04-09T14:58:09.689422Z",
"start_time": "2025-04-09T14:58:09.686524Z"
(exception)勉强赚钱rank
2025-03-31 23:08:03 +08:00
}
},
init
2025-02-12 00:21:33 +08:00
"outputs": [],
Classify2
2025-05-06 23:42:40 +08:00
"source": []
init
2025-02-12 00:21:33 +08:00
}
],
"metadata": {
"kernelspec": {
Classify2
2025-05-06 23:42:40 +08:00
"display_name": "new_trader",
init
2025-02-12 00:21:33 +08:00
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
(exception)勉强赚钱rank
2025-03-31 23:08:03 +08:00
"version": "3.11.11"
init
2025-02-12 00:21:33 +08:00
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue Copy Permalink