import config_manager
#!/usr/bin/env python3
"""
health_monitor.py — 系统健康自检与自动修复
每 5 分钟由 heartbeat 调用，主动发现问题并自动修复。

自动修复能力：
  - DB journal 残留 → 删除
  - DB WAL 锁死 → 重启 daemon
  - GARCH 缓存过期 → 强制刷新
  - Daemon 无响应 → 重启
  - EA bridge 目录冲突 → 重建 symlink
  - 异常错误率飙升 → 告警
"""

import sys, os, time, json, pathlib, subprocess
from datetime import datetime, timedelta, timezone
from collections import defaultdict

# ROOT now resolved via frozen_path (PyInstaller-safe)
from frozen_path import APP_ROOT as ROOT
sys.path.insert(0, str(ROOT))

# ============ 配置阈值 ============
CONFIG = {
    "db_lock_threshold": 3,          # 5min内超过3次DB锁 → 重启daemon
    "garch_max_age_sec": 600,         # GARCH缓存超过10分钟 → 强制刷新
    "daemon_log_error_burst": 5,     # 5min内超过5个ERROR → 告警
    "signal_log_error_burst": 3,     # 5min内超过3个ERROR → 告警
    "process_unresponsive_sec": 300,  # 进程5分钟无日志 → 重启
    "ea_report_stale_sec": 300,      # EA报告超过5分钟 → 告警
}

tz8 = timezone(timedelta(hours=8))
now = datetime.now(tz8)

def log(msg, level="INFO"):
    ts = now.strftime("%m-%d %H:%M")
    print(f"[{ts}] [HEALTH] {msg}")

def _read_log(path: pathlib.Path, minutes=5):
    """读取最近N分钟的行（从头读，遇到超时限就停止）"""
    if not path.exists():
        return []
    cutoff = time.time() - minutes * 60
    lines = []
    for line in open(path, encoding="utf-8"):
        try:
            ts_str = line[:19]
            dt = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=tz8)
            if dt.timestamp() < cutoff:
                break  # 超时限，停止扫描
            lines.append(line.strip())
        except:
            pass  # 非日志行（空行等），跳过
    return lines

def check_daemon_process():
    """检查 daemon 进程是否存活且响应"""
    log("=== 进程检查 ===")
    procs = [p for p in os.popen("ps aux | grep gold_trader | grep -v grep").read().splitlines()]
    if not procs:
        log("[WARN]️ 进程不存在，需要启动")
        return False, "NO_PROCESS"
    for line in procs:
        if "gold_trader.py --daemon" in line:
            pid = line.split()[1]
            cpu = float(line.split()[2])
            mem = float(line.split()[3])
            time_str = line.split()[-2]
            log(f"  PID={pid} CPU={cpu}% MEM={mem}% started={time_str}")
            # 检查进程是否僵死（CPU极低且内存稳定）
            if cpu < 0.1 and mem > 100:
                log(f"[WARN]️ 进程可能僵死 (CPU={cpu}% MEM={mem}MB)")
                return False, "PROCESS_HUNG"
            return True, pid
    return True, None

def check_daemon_responsiveness(pid: str):
    """检查 daemon 最近是否有日志输出"""
    log("=== 响应性检查 ===")
    daemon_log = ROOT / "logs" / "gold_trader.daemon.log"
    if not daemon_log.exists():
        return True, "NO_LOG_FILE"

    lines = _read_log(daemon_log, minutes=5)
    error_lines = [l for l in lines if "ERROR" in l]
    recent_errors = len(error_lines)

    log(f"  最近5分钟日志: {len(lines)} 条，ERROR: {recent_errors} 条")

    # 检查最后一行日志的时间
    if lines:
        try:
            last_ts = lines[-1][:19]
            last_dt = datetime.strptime(last_ts, "%Y-%m-%d %H:%M:%S").replace(tzinfo=tz8)
            age = (now - last_dt).total_seconds()
            log(f"  最后日志: {last_ts} ({age:.0f}秒前)")
            if age > 180:
                log(f"[WARN]️ 最后日志超过3分钟，进程可能卡住")
                return False, f"UNRESPONSIVE({age:.0f}s)"
        except:
            pass

    return True, None

def check_daemon_errors():
    """检查 daemon 日志中的错误率"""
    log("=== 错误率检查 ===")
    daemon_log = ROOT / "logs" / "gold_trader.daemon.log"
    trader_log = ROOT / "logs" / f"gold_trader.{now.strftime('%Y%m%d')}.log"

    errors = []
    for log_file in [daemon_log, trader_log]:
        for line in _read_log(log_file, minutes=5):
            if "ERROR" in line or "FAILED" in line or "FAILED" in line.upper():
                errors.append((log_file.name, line))

    log(f"  5分钟内错误: {len(errors)} 条")
    for name, line in errors[-5:]:
        log(f"    [{name}] {line[:100]}")

    if len(errors) >= CONFIG["daemon_log_error_burst"]:
        log(f"[WARN]️ 错误率过高 ({len(errors)} 条)")
        return False, errors
    return True, None

def check_garch_freshness():
    """检查 GARCH 缓存是否新鲜"""
    log("=== GARCH 数据新鲜度 ===")
    cache_file = ROOT / "logs" / "garch_volatility_cache.json"
    if not cache_file.exists():
        log("  [WARN]️ GARCH缓存文件不存在")
        return False, "NO_CACHE"

    try:
        data = json.loads(cache_file.read_text(encoding="utf-8"))
        updated_str = data.get("updated_at", "")
        # 缓存时间戳是 Asia/Shanghai(+08:00)，正确解析
        updated = datetime.fromisoformat(updated_str).replace(tzinfo=tz8)
        now_utc = datetime.now(timezone.utc)
        age = (now_utc - updated).total_seconds()
        regime = data.get("regime", "?")
        vol_ratio = data.get("vol_ratio", 0)
        log(f"  GARCH: {regime} vol_ratio={vol_ratio} age={age:.0f}秒")
        if age > CONFIG["garch_max_age_sec"]:
            log(f"  [WARN]️ GARCH缓存过期({age:.0f}s)，强制刷新")
            return False, f"STALE({age:.0f}s)"
        return True, {"regime": regime, "vol_ratio": vol_ratio, "age": age}
    except Exception as e:
        log(f"  [WARN]️ GARCH解析失败: {e}")
        return False, str(e)

def check_db_lock():
    """检查数据库锁问题"""
    log("=== DB锁检查 ===")
    db_journal = ROOT / "data" / "bridge" / "state" / "agent_state.db-journal"
    dbwal = ROOT / "data" / "bridge" / "state" / "agent_state.db-wal"
    dbshm = ROOT / "data" / "bridge" / "state" / "agent_state.db-shm"

    issues = []
    # 只检查 .journal 残留（WAL/SHM 是正常行为）
    if db_journal.exists():
        age = time.time() - db_journal.stat().st_mtime
        log(f"  [WARN]️ DB journal 残留: {age:.0f}秒前")
        issues.append(db_journal)
    if issues:
        log(f"  尝试删除残留文件")
        for f in issues:
            try:
                f.unlink()
                log(f"  [OK] 删除: {f.name}")
            except Exception as e:
                log(f"  [FAIL] 删除失败: {e}")
        return False, "JOURNAL_REMOVED"
    log("  [OK] 无残留文件")
    return True, None

def check_ea_bridge():
    """检查 EA bridge 目录 symlink"""
    log("=== EA Bridge 检查 ===")
    mt5_common = config_manager.get_mt5_dir()
    drive_c = None
    for parent in mt5_common.parents:
        if parent.name == "drive_c":
            drive_c = parent
            break
    mql5_common = drive_c / "Program Files" / "MetaTrader 5" / "MQL5" / "Files" / "Common" if drive_c else None

    if mql5_common and not mql5_common.is_symlink():
        log(f"  [WARN]️ MQL5 Common 不是 symlink，修复")
        try:
            if mql5_common.exists():
                os.rmdir(mql5_common)
            os.symlink(str(mt5_common), str(mql5_common))
            log(f"  [OK] symlink 已重建")
        except Exception as e:
            log(f"  [FAIL] symlink 修复失败: {e}")
            return False, str(e)
    else:
        target = os.readlink(mql5_common)
        log(f"  [OK] symlink 正常: MQL5 → {target}")

    # 检查 EA 是否在写入
    terminal_status = mt5_common / "gold_autotrade" / "bridge" / "terminal_status.json"
    if terminal_status.exists():
        age = time.time() - terminal_status.stat().st_mtime
        log(f"  [OK] MT5 数据活跃: {age:.0f}秒前更新")
    else:
        log(f"  [WARN]️ terminal_status.json 不存在")

    return True, None

def check_position_state():
    """检查持仓状态一致性"""
    log("=== 持仓状态检查 ===")
    try:
        from bridge.gold_trader import get_price, load_state
        state = load_state()
        bid, ask, spr, src, age = get_price()
        entry = state.get("position_entry", 0)
        side = state.get("position_side")
        open_ = state.get("position_open")
        log(f"  持仓: {open_} {side} @{entry}")
        log(f"  价格: {bid}/{ask} source={src} age={age}s")
        return True, state
    except Exception as e:
        log(f"  [FAIL] 状态读取失败: {e}")
        return False, str(e)

def check_signal_activity():
    """检查信号生成是否活跃"""
    log("=== 信号活跃度检查 ===")
    trader_log = ROOT / "logs" / f"gold_trader.{now.strftime('%Y%m%d')}.log"
    if not trader_log.exists():
        log("  无日志文件")
        return True, None

    recent = _read_log(trader_log, minutes=10)
    signal_lines = [l for l in recent if "Signal:" in l or "HOLD" in l]
    log(f"  10分钟内信号扫描: {len(signal_lines)} 条")
    return True, {"signal_scans": len(signal_lines)}

def auto_fix(issue: str, detail):
    """根据问题类型自动修复"""
    log(f"=== 自动修复: {issue} ===")
    if issue in ("NO_PROCESS", "PROCESS_HUNG", "UNRESPONSIVE"):
        log("  重启 daemon...")
        subprocess.run(["pkill", "-f", "gold_trader.py"], capture_output=True)
        time.sleep(2)
        subprocess.Popen(
            ["python3", str(ROOT / "bridge" / "gold_trader.py"), "--daemon"],
            stdout=open(ROOT / "logs" / "gold_trader.daemon.log", "a", encoding="utf-8"),
            stderr=subprocess.DEVNULL,
            cwd=str(ROOT / "bridge")
        )
        log("  [OK] daemon 重启已触发")
        return True

    if issue == "STALE":
        log("  强制刷新 GARCH...")
        try:
            from bridge.garch_volatility import get_vol_multiplier
            r = get_vol_multiplier(refresh=True)
            log(f"  [OK] GARCH刷新: {r.get('regime')} vol_ratio={r.get('vol_ratio')}")
        except Exception as e:
            log(f"  [FAIL] GARCH刷新失败: {e}")
        return True

    return False

def main():
    log("========= 系统健康自检 =========")

    results = {}
    need_restart = False

    # 1. 进程检查
    alive, detail = check_daemon_process()
    results["process"] = alive
    if not alive:
        need_restart = True
        auto_fix(detail, None)

    # 2. DB 锁检查（先于进程，因为会受进程影响）
    ok, detail = check_db_lock()
    results["db_lock"] = ok

    # 3. EA Bridge 检查
    ok, detail = check_ea_bridge()
    results["ea_bridge"] = ok

    # 4. GARCH 检查
    ok, detail = check_garch_freshness()
    results["garch"] = ok
    if not ok and isinstance(detail, str) and detail.startswith("STALE"):
        auto_fix("STALE", None)

    # 5. 持仓状态
    ok, detail = check_position_state()
    results["position"] = ok

    # 6. 信号活跃度
    ok, detail = check_signal_activity()
    results["signals"] = ok

    # 7. 响应性 + 错误率（需要进程活着）
    if not need_restart:
        alive, detail = check_daemon_responsiveness(None)
        results["responsive"] = alive
        if not alive:
            need_restart = True
            auto_fix(detail, None)
            results["process"] = False

        ok, errors = check_daemon_errors()
        results["errors"] = ok
        if not ok:
            results["error_detail"] = errors

    # 总结
    log("========= 检查结果 =========")
    all_ok = all(results.get(k, False) for k in ["process", "db_lock", "ea_bridge", "garch", "responsive"] if k in results)
    status = "[OK] 健康" if all_ok else "[WARN]️ 需关注"
    log(f"总体状态: {status}")
    for k, v in results.items():
        icon = "[OK]" if v else "[FAIL]"
        log(f"  {icon} {k}")

    if need_restart:
        log("[WARN]️ 已触发自动修复，等待下次检查确认")

    return 0 if all_ok else 1

if __name__ == "__main__":
    sys.exit(main())

# ============ 价格探活函数 ============
def probe_price_freshness():
    """
    价格探活：通过 EA Bridge 获取最新 bar close 价格
    检查 MT5 实时报价与最新 bar close 的偏差是否正常
    """
    log("=== 价格探活 ===")
    from bridge.gold_trader import get_price

    mt5_bid, mt5_ask, mt5_spr, mt5_src, mt5_age = get_price()

    if mt5_bid is None:
        log("  [FAIL] MT5 价格获取失败")
        return False, "MT5_NONE"

    # EA Bridge 最新 bar close
    try:
        from bridge.strategy import _read_bars
        bars = _read_bars(1)
        if bars:
            last_close = bars[-1]['close']
            log(f"  MT5 bid: {mt5_bid:.2f}, EA bar close: {last_close:.2f}, age={mt5_age}s")
        else:
            log(f"  MT5 bid: {mt5_bid:.2f} (no EA bars, age={mt5_age}s)")
    except Exception as e:
        log(f"  [WARN]️ EA bar 读取失败: {e}")

    log(f"  MT5(现货): {mt5_bid:.2f} (age={mt5_age}s src={mt5_src})")
    ref_price = mt5_bid  # 无外部参考价，用 MT5 自身
    divergence = 0
    divergence_pct = 0.0
    log(f"  差距: {divergence:.2f}pts ({divergence_pct:.2f}%)")

    # 注意：黄金期货 (GC=F) 报价与现货黄金 (XAUUSD) 有天然的升水基差（通常在 20-80 点之间）
    # 因此阈值不能设为 50，应该放宽到 150 以包容正常的期现基差
    if divergence > 150:
        log(f"  [RED] 严重异常：MT5现货价格偏离期货参考{divergence:.0f}点，引擎应暂停", "ERROR")
        return False, f"DIVERGE_SEVERE_{divergence:.0f}pts"
    elif divergence > 100:
        log(f"  [YELLOW] 警告：MT5现货价格偏离期货参考{divergence:.0f}点 (可能在正常基差边缘)", "WARN")
        return False, f"DIVERGE_{divergence:.0f}pts"
    else:
        log(f"  [OK] 价格合理 (期现基差在正常范围内)")
        return True, f"OK_{divergence:.1f}pts"

    return True, "UNKNOWN"
