init
This commit is contained in:
liaozhaorun
193
code/data/update/update_money_flow.ipynb
Normal file
193
code/data/update/update_money_flow.ipynb
Normal file
@@ -0,0 +1,193 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "b94bb1f2-5332-485e-ae1b-eea01f938106",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2025-02-11T15:21:54.821950Z",
|
||||
"start_time": "2025-02-11T15:21:54.050569Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"import tushare as ts\n",
|
||||
"\n",
|
||||
"ts.set_token('3a0741c702ee7e5e5f2bf1f0846bafaafe4e320833240b2a7e4a685f')\n",
|
||||
"pro = ts.pro_api()"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2025-02-11T15:22:32.726905Z",
|
||||
"start_time": "2025-02-11T15:22:25.018135Z"
|
||||
}
|
||||
},
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import time\n",
|
||||
"\n",
|
||||
"h5_filename = '../../../data/money_flow.h5'\n",
|
||||
"key = '/money_flow'\n",
|
||||
"max_date = None\n",
|
||||
"with pd.HDFStore(h5_filename, mode='r') as store:\n",
|
||||
" df = store[key][['ts_code', 'trade_date']]\n",
|
||||
" print(df.info())\n",
|
||||
" max_date = df['trade_date'].max()\n",
|
||||
"\n",
|
||||
"print(max_date)\n",
|
||||
"trade_cal = pro.trade_cal(exchange='', start_date='20170101', end_date='20250220')\n",
|
||||
"trade_cal = trade_cal[trade_cal['is_open'] == 1] # 只保留交易日\n",
|
||||
"trade_dates = trade_cal[trade_cal['cal_date'] > max_date]['cal_date'].tolist()\n",
|
||||
"start_date = min(trade_dates)\n",
|
||||
"print(f'start_date: {start_date}')"
|
||||
],
|
||||
"id": "742c29d453b9bb38",
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"<class 'pandas.core.frame.DataFrame'>\n",
|
||||
"Index: 8153941 entries, 0 to 5120\n",
|
||||
"Data columns (total 2 columns):\n",
|
||||
" # Column Dtype \n",
|
||||
"--- ------ ----- \n",
|
||||
" 0 ts_code object\n",
|
||||
" 1 trade_date object\n",
|
||||
"dtypes: object(2)\n",
|
||||
"memory usage: 186.6+ MB\n",
|
||||
"None\n",
|
||||
"20250211\n",
|
||||
"start_date: 20250212\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_count": 6
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "679ce40e-8d62-4887-970c-e1d8cbdeee6b",
|
||||
"metadata": {
|
||||
"scrolled": true,
|
||||
"ExecuteTime": {
|
||||
"end_time": "2025-02-11T15:22:14.513527Z",
|
||||
"start_time": "2025-02-11T15:22:12.973331Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"from concurrent.futures import ThreadPoolExecutor, as_completed\n",
|
||||
"\n",
|
||||
"all_daily_data = []\n",
|
||||
"\n",
|
||||
"# API 调用计数和时间控制变量\n",
|
||||
"api_call_count = 0\n",
|
||||
"batch_start_time = time.time()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_data(trade_date):\n",
|
||||
" time.sleep(0.1)\n",
|
||||
" money_flow_data = pro.moneyflow(trade_date=trade_date)\n",
|
||||
" if money_flow_data is not None and not money_flow_data.empty:\n",
|
||||
" return money_flow_data\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"with ThreadPoolExecutor(max_workers=2) as executor:\n",
|
||||
" future_to_date = {executor.submit(get_data, td): td for td in trade_dates}\n",
|
||||
"\n",
|
||||
" for future in as_completed(future_to_date):\n",
|
||||
" trade_date = future_to_date[future] # 获取对应的交易日期\n",
|
||||
" try:\n",
|
||||
" result = future.result() # 获取任务执行的结果\n",
|
||||
" all_daily_data.append(result)\n",
|
||||
" print(f\"任务 {trade_date} 完成\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"获取 {trade_date} 数据时出错: {e}\")\n",
|
||||
"\n"
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"任务 20250219 完成\n",
|
||||
"任务 20250220 完成\n",
|
||||
"任务 20250218 完成\n",
|
||||
"任务 20250217 完成\n",
|
||||
"任务 20250214 完成\n",
|
||||
"任务 20250213 完成\n",
|
||||
"任务 20250212 完成\n",
|
||||
"任务 20250211 完成\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_count": 3
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2025-02-11T15:22:16.656650Z",
|
||||
"start_time": "2025-02-11T15:22:16.639271Z"
|
||||
}
|
||||
},
|
||||
"cell_type": "code",
|
||||
"source": "all_daily_data_df = pd.concat(all_daily_data, ignore_index=True)\n",
|
||||
"id": "9af80516849d4e80",
|
||||
"outputs": [],
|
||||
"execution_count": 4
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"id": "a2b05187-437f-4053-bc43-bd80d4cf8b0e",
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2025-02-11T15:22:20.447350Z",
|
||||
"start_time": "2025-02-11T15:22:19.145561Z"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"\n",
|
||||
"# 将所有数据合并为一个 DataFrame\n",
|
||||
"\n",
|
||||
"# 将数据保存为 HDF5 文件（table 格式）\n",
|
||||
"all_daily_data_df.to_hdf(h5_filename, key='money_flow', mode='a', format='table', append=True, data_columns=True)\n",
|
||||
"\n",
|
||||
"print(\"所有每日基础数据获取并保存完毕！\")"
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"所有每日基础数据获取并保存完毕！\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_count": 5
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.19"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user