efactor(qmt): 重构架构 - 统一信号发送和消息处理

- 修复 qmt_engine.py 中 initialize() 的重复代码
- 新增 message_processor.py: Redis Stream 消息处理器
- 新增 logger.py: 细粒度日志模块
- 新增 qmt_sender.py: 统一信号发送端（槽位+百分比模式）
- 新增 backtest_consumer.py: 回测消息消费者
- 删除旧模块: qmt_trader.py, qmt_signal_sender.py, qmt_percentage_sender.py
- 更新文档: qmt_functionality.md 反映新架构

qmt/qmt_engine.py

@@ -17,6 +17,15 @@ from xtquant.xttrader import XtQuantTrader, XtQuantTraderCallback
 from xtquant.xttype import StockAccount
 from xtquant import xtconstant
 
+# 导入 Redis Stream 消息处理器和日志模块
+try:
+    from .message_processor import StreamMessageProcessor
+    from .logger import QMTLogger
+except ImportError:
+    # 当作为脚本直接运行时
+    from message_processor import StreamMessageProcessor
+    from logger import QMTLogger
+
 # ================= 0. Windows 补丁 =================
 try:
     import ctypes
@@ -473,19 +482,66 @@ class MultiEngineManager:
         self.start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         self.is_scheduled_reconnecting = False  # 定时重连调度器正在执行标志
         self._initialized = True
+        # Stream 处理器和日志器将在 initialize 中创建
+        self.stream_processor: Optional[StreamMessageProcessor] = None
+        self.qmt_logger: Optional[QMTLogger] = None
 
     def initialize(self, config_file="config.json"):
-        self._setup_logger()
+        self._setup_logger()  # 先初始化 logger
         with open(config_file, "r", encoding="utf-8") as f:
             self.config = json.load(f)
 
-        self.r = redis.Redis(**self.config["redis"], decode_responses=True)
+        # 从 .env.local 加载 Redis 配置
+        redis_config = self._load_redis_config()
+        self.r = redis.Redis(**redis_config, decode_responses=True)
         self.pos_manager = PositionManager(self.r)
 
+        # 初始化 Redis Stream 处理器
+        self.stream_processor = StreamMessageProcessor(redis_client=self.r)
+        self.qmt_logger = QMTLogger(name="QMT_Engine_Stream")
+        self.logger.info("Redis Stream 处理器初始化完成")
+
         for t_cfg in self.config.get("qmt_terminals", []):
             unit = TradingUnit(t_cfg)
             unit.connect()
             self.units[unit.qmt_id] = unit
+    def _load_redis_config(self) -> Dict[str, Any]:
+        """从 .env.local 加载 Redis 配置"""
+        # 尝试加载 python-dotenv
+        try:
+            from dotenv import load_dotenv
+
+            # 尝试多个路径
+            env_paths = [
+                os.path.join(os.path.dirname(os.path.abspath(__file__)), "config", ".env.local"),
+                os.path.join(os.path.dirname(__file__), "config", ".env.local"),
+                os.path.join(os.path.dirname(__file__), "..", "config", ".env.local"),
+                "/qmt/config/.env.local",
+                "config/.env.local",
+            ]
+
+            loaded = False
+            for env_path in env_paths:
+                if os.path.exists(env_path):
+                    load_dotenv(env_path)
+                    self.logger.info(f"[Config] 加载 Redis 配置: {env_path}")
+                    loaded = True
+                    break
+
+            if not loaded:
+                self.logger.warning("[Config] 警告: 未找到 .env.local 文件，使用默认配置")
+        except ImportError:
+            self.logger.warning("[Config] 警告: 未安装 python-dotenv，使用默认配置")
+
+        # 从环境变量读取 Redis 配置
+        redis_config = {
+            "host": os.getenv("REDIS_HOST", "localhost"),
+            "port": int(os.getenv("REDIS_PORT", "6379")),
+            "password": os.getenv("REDIS_PASSWORD") or None,
+            "db": int(os.getenv("REDIS_DB", "0")),
+        }
+
+        return redis_config
 
     def _setup_logger(self):
         log_dir = "logs"
@@ -494,8 +550,8 @@ class MultiEngineManager:
         log_file = os.path.join(
             log_dir, f"{datetime.date.today().strftime('%Y-%m-%d')}.log"
         )
-        logger = logging.getLogger("QMT_Engine")
-        logger.setLevel(logging.INFO)
+        self.logger = logging.getLogger("QMT_Engine")
+        self.logger.setLevel(logging.INFO)
         # 确保日志流为 UTF-8
         fmt = logging.Formatter(
             "[%(asctime)s] [%(threadName)s] %(message)s", "%H:%M:%S"
@@ -504,9 +560,8 @@ class MultiEngineManager:
         fh.setFormatter(fmt)
         sh = logging.StreamHandler(sys.stdout)
         sh.setFormatter(fmt)
-        logger.addHandler(fh)
-        logger.addHandler(sh)
-
+        self.logger.addHandler(fh)
+        self.logger.addHandler(sh)
     def get_strategies_by_terminal(self, qmt_id):
         return [
             s
@@ -632,37 +687,147 @@ class MultiEngineManager:
                 time.sleep(10)
 
     def process_route(self, strategy_name):
+        """处理策略消息路由 - 使用 Redis Stream
+
+        从 Redis Stream 消费消息，处理成功后 ACK，失败则进入失败队列。
+        """
         strat_cfg = self.config["strategies"].get(strategy_name)
         unit = self.units.get(strat_cfg.get("qmt_id"))
         if not unit or not unit.callback or not unit.callback.is_connected:
             return
 
-        msg_json = self.r.lpop(f"{strategy_name}_real")
-        if not msg_json:
+        # 使用 StreamMessageProcessor 消费消息
+        if not self.stream_processor:
+            self.logger.error("[process_route] Stream处理器未初始化")
             return
 
         try:
-            data = json.loads(msg_json)
-            # 严格校验消息日期
-            if data.get("timestamp", "").split(" ")[
-                0
-            ] != datetime.date.today().strftime("%Y-%m-%d"):
+            # 消费消息 (非阻塞，立即返回)
+            messages = self.stream_processor.consume_message(
+                strategy_name=strategy_name,
+                consumer_id=None,  # 自动生成
+                is_backtest=False,
+                block_ms=100,  # 短阻塞，快速返回
+            )
+
+            if not messages:
                 return
 
-            if data["action"] == "BUY":
-                self._execute_buy(unit, strategy_name, data)
-            elif data["action"] == "SELL":
-                self._execute_sell(unit, strategy_name, data)
-        except json.JSONDecodeError as e:
-            self.logger.error(
-                f"[{strategy_name}] JSON解析失败: {e}, 消息: {msg_json[:200]}"
-            )
-        except KeyError as e:
-            self.logger.error(f"[{strategy_name}] 消息缺少必要字段: {e}")
+            # 处理每条消息
+            for msg_id, data in messages:
+                try:
+                    # 1. 记录消息接收日志
+                    stream_key = f"qmt:{strategy_name}:real"
+                    self.qmt_logger.log_message_receive(stream_key, msg_id, data)
+
+                    # 2. 严格校验消息日期
+                    msg_date = data.get("timestamp", "").split(" ")[0]
+                    today_str = datetime.date.today().strftime("%Y-%m-%d")
+
+                    if msg_date != today_str:
+                        self.qmt_logger.log_validation(
+                            validation_type="date_check",
+                            strategy_name=strategy_name,
+                            details={"msg_date": msg_date, "today": today_str},
+                            result=False,
+                        )
+                        # 日期不符的消息也 ACK，避免重复处理
+                        self.stream_processor.ack_message(
+                            strategy_name, msg_id, is_backtest=False
+                        )
+                        continue
+
+                    self.qmt_logger.log_validation(
+                        validation_type="date_check",
+                        strategy_name=strategy_name,
+                        details={"msg_date": msg_date},
+                        result=True,
+                    )
+
+                    # 3. 执行交易动作
+                    action = data.get("action")
+                    
+                    # 获取策略配置，确定下单模式
+                    strat_cfg = self.config["strategies"].get(strategy_name, {})
+                    order_mode = strat_cfg.get("order_mode", "slots")
+                    
+                    if action == "BUY":
+                        self.qmt_logger.log_validation(
+                            validation_type="action_check",
+                            strategy_name=strategy_name,
+                            details={"action": "BUY", "code": data.get("stock_code"), "order_mode": order_mode},
+                            result=True,
+                        )
+                        # 根据下单模式执行相应逻辑
+                        if order_mode == "percentage":
+                            self._execute_percentage_buy(unit, strategy_name, data)
+                        else:
+                            self._execute_buy(unit, strategy_name, data)
+                    elif action == "SELL":
+                        self.qmt_logger.log_validation(
+                            validation_type="action_check",
+                            strategy_name=strategy_name,
+                            details={"action": "SELL", "code": data.get("stock_code"), "order_mode": order_mode},
+                            result=True,
+                        )
+                        # 根据下单模式执行相应逻辑
+                        if order_mode == "percentage":
+                            self._execute_percentage_sell(unit, strategy_name, data)
+                        else:
+                            self._execute_sell(unit, strategy_name, data)
+                    else:
+                        self.qmt_logger.log_validation(
+                            validation_type="action_check",
+                            strategy_name=strategy_name,
+                            details={"action": action},
+                            result=False,
+                        )
+                        self.logger.warning(f"[{strategy_name}] 未知动作: {action}")
+
+                    # 4. 确认消息已处理
+                    ack_success = self.stream_processor.ack_message(
+                        strategy_name, msg_id, is_backtest=False
+                    )
+                    self.qmt_logger.log_ack(stream_key, msg_id, ack_success)
+
+                except json.JSONDecodeError as e:
+                    error_msg = f"JSON解析失败: {e}"
+                    self.logger.error(
+                        f"[{strategy_name}] {error_msg}, 消息ID: {msg_id}"
+                    )
+                    self.qmt_logger.log_failure(stream_key, msg_id, error_msg)
+                    # 解析失败的消息也发送 ACK，避免死循环
+                    self.stream_processor.ack_message(
+                        strategy_name, msg_id, is_backtest=False
+                    )
+
+                except KeyError as e:
+                    error_msg = f"消息缺少必要字段: {e}"
+                    self.logger.error(f"[{strategy_name}] {error_msg}")
+                    self.qmt_logger.log_failure(stream_key, msg_id, error_msg)
+                    self.stream_processor.send_to_failure_queue(
+                        strategy_name, data, error_msg
+                    )
+                    # 发送失败队列后 ACK
+                    self.stream_processor.ack_message(
+                        strategy_name, msg_id, is_backtest=False
+                    )
+
+                except Exception as e:
+                    error_msg = f"消息处理异常: {str(e)}"
+                    self.logger.error(f"[{strategy_name}] {error_msg}", exc_info=True)
+                    self.qmt_logger.log_failure(stream_key, msg_id, error_msg)
+                    # 异常消息进入失败队列
+                    self.stream_processor.send_to_failure_queue(
+                        strategy_name, data, error_msg
+                    )
+                    # 发送失败队列后 ACK
+                    self.stream_processor.ack_message(
+                        strategy_name, msg_id, is_backtest=False
+                    )
+
         except Exception as e:
-            self.logger.error(
-                f"[{strategy_name}] 消息处理异常: {str(e)}", exc_info=True
-            )
+            self.logger.error(f"[process_route] 消费消息异常: {str(e)}", exc_info=True)
 
     def _execute_buy(self, unit, strategy_name, data):
         strat_cfg = self.config["strategies"][strategy_name]
@@ -719,6 +884,15 @@ class MultiEngineManager:
             if vol < 100:
                 return
 
+            # 记录订单执行请求
+            self.qmt_logger.log_order_execution(
+                strategy_name=strategy_name,
+                stock_code=data["stock_code"],
+                action="BUY",
+                volume=vol,
+                price=price,
+            )
+
             oid = unit.xt_trader.order_stock(
                 unit.acc_obj,
                 data["stock_code"],
@@ -735,6 +909,26 @@ class MultiEngineManager:
                 self.logger.info(
                     f"√√√ [{unit.alias}] {strategy_name} 下单买入: {data['stock_code']} {vol}股 @ {price}"
                 )
+                # 记录订单执行成功
+                self.qmt_logger.log_order_execution(
+                    strategy_name=strategy_name,
+                    stock_code=data["stock_code"],
+                    action="BUY",
+                    volume=vol,
+                    price=price,
+                    order_id=oid,
+                )
+            else:
+                error_msg = "下单请求被拒绝 (Result=-1)"
+                self.logger.error(f"[{strategy_name}] {error_msg}")
+                self.qmt_logger.log_order_execution(
+                    strategy_name=strategy_name,
+                    stock_code=data["stock_code"],
+                    action="BUY",
+                    volume=vol,
+                    price=price,
+                    error=error_msg,
+                )
         except:
             self.logger.error(traceback.format_exc())
 
@@ -785,6 +979,15 @@ class MultiEngineManager:
             )
             price = round(float(data["price"]) + offset, 3)
 
+            # 记录订单执行请求
+            self.qmt_logger.log_order_execution(
+                strategy_name=strategy_name,
+                stock_code=data["stock_code"],
+                action="SELL",
+                volume=final_vol,
+                price=price,
+            )
+
             oid = unit.xt_trader.order_stock(
                 unit.acc_obj,
                 data["stock_code"],
@@ -800,9 +1003,198 @@ class MultiEngineManager:
                 self.logger.info(
                     f"√√√ [{unit.alias}] {strategy_name} 下单卖出: {data['stock_code']} {final_vol}股 @ {price}"
                 )
+                # 记录订单执行成功
+                self.qmt_logger.log_order_execution(
+                    strategy_name=strategy_name,
+                    stock_code=data["stock_code"],
+                    action="SELL",
+                    volume=final_vol,
+                    price=price,
+                    order_id=oid,
+                )
+            else:
+                error_msg = "下单请求被拒绝 (Result=-1)"
+                self.logger.error(f"[{strategy_name}] {error_msg}")
+                self.qmt_logger.log_order_execution(
+                    strategy_name=strategy_name,
+                    stock_code=data["stock_code"],
+                    action="SELL",
+                    volume=final_vol,
+                    price=price,
+                    error=error_msg,
+                )
         except:
             self.logger.error(traceback.format_exc())
 
+    def _execute_percentage_buy(self, unit, strategy_name, data):
+        """处理百分比模式的买入逻辑"""
+        strat_cfg = self.config["strategies"][strategy_name]
+        
+        # 获取目标持仓百分比
+        position_pct = float(data.get("position_pct", 0))
+        if position_pct <= 0:
+            self.logger.warning(f"[{strategy_name}] 百分比模式买入: position_pct 无效 ({position_pct})")
+            return
+        
+        self.logger.info(f"[{strategy_name}] [百分比模式] 处理买入: {data['stock_code']}, 目标占比: {position_pct}")
+        
+        try:
+            asset = unit.xt_trader.query_stock_asset(unit.acc_obj)
+            if not asset:
+                self.logger.error(f"[{strategy_name}] API 错误: query_stock_asset 返回 None")
+                return
+            
+            total_asset = asset.total_asset
+            available_cash = asset.cash
+            
+            self.logger.info(f"[{strategy_name}] [百分比模式] 账户总资产: {total_asset:.2f}, 可用资金: {available_cash:.2f}")
+            
+            # 计算目标金额
+            target_amount = total_asset * position_pct
+            actual_amount = min(target_amount, available_cash * 0.98)  # 预留手续费滑点
+            
+            self.logger.info(f"[{strategy_name}] [百分比模式] 目标金额: {target_amount:.2f}, 实际可用: {actual_amount:.2f}")
+            
+            # 检查最小金额限制
+            if actual_amount < 2000:
+                self.logger.warning(f"[{strategy_name}] [百分比模式] 拦截买入: 金额过小 ({actual_amount:.2f} < 2000)")
+                return
+            
+            # 价格校验
+            price = float(data.get("price", 0))
+            offset = strat_cfg.get("execution", {}).get("buy_price_offset", 0.0)
+            price = round(price + offset, 3)
+            
+            if price <= 0:
+                self.logger.warning(f"[{strategy_name}] [百分比模式] 价格异常: {price}，强制设为1.0")
+                price = 1.0
+            
+            # 计算股数
+            vol = int(actual_amount / price / 100) * 100
+            self.logger.info(f"[{strategy_name}] [百分比模式] 计算股数: 资金{actual_amount:.2f} / 价格{price} -> {vol}股")
+            
+            if vol < 100:
+                self.logger.warning(f"[{strategy_name}] [百分比模式] 拦截买入: 股数不足 100 ({vol})")
+                return
+            
+            # 记录订单执行请求
+            self.qmt_logger.log_order_execution(
+                strategy_name=strategy_name,
+                stock_code=data["stock_code"],
+                action="BUY",
+                volume=vol,
+                price=price,
+            )
+            
+            oid = unit.xt_trader.order_stock(
+                unit.acc_obj,
+                data["stock_code"],
+                xtconstant.STOCK_BUY,
+                vol,
+                xtconstant.FIX_PRICE,
+                price,
+                strategy_name,
+                "PyBuyPct",
+            )
+            
+            if oid != -1:
+                unit.order_cache[oid] = (strategy_name, data["stock_code"], "BUY")
+                self.logger.info(f"√√√ [{unit.alias}] [{strategy_name}] [百分比模式] 下单买入: {data['stock_code']} {vol}股 @ {price}")
+                # 记录订单执行成功
+                self.qmt_logger.log_order_execution(
+                    strategy_name=strategy_name,
+                    stock_code=data["stock_code"],
+                    action="BUY",
+                    volume=vol,
+                    price=price,
+                    order_id=oid,
+                )
+            else:
+                error_msg = "下单请求被拒绝 (Result=-1)"
+                self.logger.error(f"[{strategy_name}] [百分比模式] {error_msg}")
+                self.qmt_logger.log_order_execution(
+                    strategy_name=strategy_name,
+                    stock_code=data["stock_code"],
+                    action="BUY",
+                    volume=vol,
+                    price=price,
+                    error=error_msg,
+                )
+        except:
+            self.logger.error(traceback.format_exc())
+
+    def _execute_percentage_sell(self, unit, strategy_name, data):
+        """处理百分比模式的卖出逻辑（清仓）"""
+        self.logger.info(f"[{strategy_name}] [百分比模式] 处理卖出: {data['stock_code']} (清仓)")
+        
+        try:
+            # 查询实盘持仓
+            real_pos = unit.xt_trader.query_stock_positions(unit.acc_obj)
+            if real_pos is None:
+                self.logger.error(f"[{strategy_name}] [百分比模式] API 错误: query_stock_positions 返回 None")
+                return
+            
+            rp = next((p for p in real_pos if p.stock_code == data["stock_code"]), None)
+            can_use = rp.can_use_volume if rp else 0
+            
+            self.logger.info(f"[{strategy_name}] [百分比模式] 股票 {data['stock_code']} 实盘可用持仓: {can_use}")
+            
+            if can_use <= 0:
+                self.logger.warning(f"[{strategy_name}] [百分比模式] 拦截卖出: 无可用持仓")
+                return
+            
+            # 执行清仓
+            price = float(data.get("price", 0))
+            offset = self.config["strategies"][strategy_name].get("execution", {}).get("sell_price_offset", 0.0)
+            price = round(price + offset, 3)
+            
+            self.logger.info(f"[{strategy_name}] [百分比模式] 执行清仓: {data['stock_code']} @ {price}, 数量: {can_use}")
+            
+            # 记录订单执行请求
+            self.qmt_logger.log_order_execution(
+                strategy_name=strategy_name,
+                stock_code=data["stock_code"],
+                action="SELL",
+                volume=can_use,
+                price=price,
+            )
+            
+            oid = unit.xt_trader.order_stock(
+                unit.acc_obj,
+                data["stock_code"],
+                xtconstant.STOCK_SELL,
+                can_use,
+                xtconstant.FIX_PRICE,
+                price,
+                strategy_name,
+                "PySellPct",
+            )
+            
+            if oid != -1:
+                unit.order_cache[oid] = (strategy_name, data["stock_code"], "SELL")
+                self.logger.info(f"√√√ [{unit.alias}] [{strategy_name}] [百分比模式] 下单卖出: {data['stock_code']} {can_use}股 @ {price}")
+                # 记录订单执行成功
+                self.qmt_logger.log_order_execution(
+                    strategy_name=strategy_name,
+                    stock_code=data["stock_code"],
+                    action="SELL",
+                    volume=can_use,
+                    price=price,
+                    order_id=oid,
+                )
+            else:
+                error_msg = "下单请求被拒绝 (Result=-1)"
+                self.logger.error(f"[{strategy_name}] [百分比模式] {error_msg}")
+                self.qmt_logger.log_order_execution(
+                    strategy_name=strategy_name,
+                    stock_code=data["stock_code"],
+                    action="SELL",
+                    volume=can_use,
+                    price=price,
+                    error=error_msg,
+                )
+        except:
+            self.logger.error(traceback.format_exc())
     def verify_connection(self, timeout=5):
         """验证物理连接是否有效"""
         try: