#!/usr/bin/env python3
"""
Correlation Engine — DXY/USD macro tailwind/headwind signal generator.

Reads multi-instrument bars from market_snapshot.json (already collected by EA Bridge),
applies 6-layer anti-noise filtering, and outputs a clean macro correlation signal
for the Commander AI and session_analyzer.

Anti-noise layers:
  L1 – EMA smoothing (raw price → smoothed)
  L2 – Rate-of-Change (absolute level → relative momentum)
  L3 – Z-score gate (statistical significance filter)
  L4 – Rolling correlation validation (is DXY–gold link actually active?)
  L5 – Direction persistence (3-bar sustained move required)
  L6 – Session-aware weighting (NY > London > Asia)
"""

from __future__ import annotations

import json
import math
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional

# ── Paths ──────────────────────────────────────────────────────────────
import bridge.bridge_constants as _bc
DEFAULT_SNAPSHOT = _bc.MT5_FILES / "market_snapshot.json" if _bc.MT5_FILES else Path.home() / "GoldAutoTradeData" / "market_snapshot.json"

# ── Symbols & Instrument Config ─────────────────────────────────────────
import os as _os

# Legacy env-var overrides (kept for backward compat)
GOLD_SYMBOL = _os.environ.get("GAT_TRADE_SYMBOL", "XAUUSD")
DXY_SYMBOL = _os.environ.get("GAT_MACRO_DRIVER_SYMBOL", "DXY.f")
USD_PROXIES = ["USDJPY", "EURUSD", "USDCHF"]


def _resolve_instrument(symbol: str):
    """Get InstrumentContext from registry, with env-var and gold fallback."""
    try:
        from src.instruments.registry import get_symbol_registry
        return get_symbol_registry().get(symbol)
    except Exception:
        from src.instruments.context import InstrumentContext, AssetCategory, CorrelationSign, MacroDriverConfig
        return InstrumentContext(
            symbol=GOLD_SYMBOL,
            display_name="Gold vs US Dollar",
            asset_name="Gold",
            asset_category=AssetCategory.PRECIOUS_METAL,
            macro_driver=MacroDriverConfig(
                driver_symbol=DXY_SYMBOL,
                correlation_sign=CorrelationSign.INVERSE,
                correlation_label="DXY USD Index",
                proxy_symbols=list(USD_PROXIES),
            ),
        )


def _get_filter_params(instrument):
    """Get anti-noise filter params from instrument config."""
    md = instrument.macro_driver
    if md and md.filter_params:
        return md.filter_params
    return {
        "ema_smooth_period": 5, "roc_period": 20,
        "z_score_window": 50, "z_score_threshold": 0.8,
        "correlation_window": 50, "correlation_min_abs": 0.25,
        "direction_persistence": 3, "atr_period": 14,
    }


# Module-level filter param accessors (backward compat)
def _fp(key: str, default=None):
    return _get_filter_params(_resolve_instrument(GOLD_SYMBOL)).get(key, default)

EMA_SMOOTH_PERIOD = _fp("ema_smooth_period", 5)
ROC_PERIOD = _fp("roc_period", 20)
Z_SCORE_WINDOW = _fp("z_score_window", 50)
Z_SCORE_THRESHOLD = _fp("z_score_threshold", 0.8)
CORRELATION_WINDOW = _fp("correlation_window", 50)
CORRELATION_MIN_ABS = _fp("correlation_min_abs", 0.25)
DIRECTION_PERSISTENCE = _fp("direction_persistence", 3)
ATR_PERIOD = _fp("atr_period", 14)


def _ema(values: list[float], period: int) -> list[float]:
    if len(values) < period:
        return [0.0] * len(values)
    alpha = 2.0 / (period + 1.0)
    result = [0.0] * (period - 1)
    result.append(sum(values[:period]) / period)
    for v in values[period:]:
        result.append(v * alpha + result[-1] * (1.0 - alpha))
    return result


def _roc(values: list[float], period: int) -> list[float]:
    """Rate of Change as %."""
    result = [0.0] * period
    for i in range(period, len(values)):
        if values[i - period] != 0:
            result.append(((values[i] - values[i - period]) / values[i - period]) * 100.0)
        else:
            result.append(0.0)
    return result


def _rolling_zscore(values: list[float], window: int) -> list[float]:
    """Rolling z-score: (x - mean) / std over window."""
    result = [0.0] * len(values)
    for i in range(window, len(values)):
        segment = values[i - window : i]
        mean = sum(segment) / window
        variance = sum((x - mean) ** 2 for x in segment) / window
        std = math.sqrt(variance) if variance > 0 else 1.0
        result[i] = (values[i] - mean) / std
    return result


def _pearson_r(x: list[float], y: list[float]) -> float:
    """Pearson correlation coefficient."""
    n = len(x)
    if n < 3:
        return 0.0
    mean_x = sum(x) / n
    mean_y = sum(y) / n
    cov = sum((xi - mean_x) * (yi - mean_y) for xi, yi in zip(x, y))
    std_x = math.sqrt(sum((xi - mean_x) ** 2 for xi in x))
    std_y = math.sqrt(sum((yi - mean_y) ** 2 for yi in y))
    if std_x == 0 or std_y == 0:
        return 0.0
    return cov / (std_x * std_y)


def _rolling_correlation(x: list[float], y: list[float], window: int) -> list[float]:
    """Rolling Pearson r between two series."""
    result = [0.0] * len(x)
    for i in range(window, len(x) + 1):
        result[i - 1] = _pearson_r(x[i - window : i], y[i - window : i])
    return result


def _atr_from_bars(bars: list[dict], period: int = ATR_PERIOD) -> float:
    """Compute current ATR from OHLC bars."""
    if len(bars) < 2:
        return 0.0
    trs = []
    for i in range(1, len(bars)):
        hl = bars[i]["high"] - bars[i]["low"]
        hc = abs(bars[i]["high"] - bars[i - 1]["close"])
        lc = abs(bars[i]["low"] - bars[i - 1]["close"])
        trs.append(max(hl, hc, lc))
    if not trs:
        return 0.0
    return _ema(trs, period)[-1]


def _detect_session_name() -> str:
    """Detect current trading session name (UTC-based)."""
    now = datetime.now(timezone.utc)
    h = now.hour
    if 13 <= h < 16:
        return "LONDON_NY_OVERLAP"
    if 16 <= h < 21:
        return "NEW_YORK"
    if 8 <= h < 13:
        return "LONDON"
    if 0 <= h < 8:
        return "ASIA"
    return "WEEKEND_CLOSED"


def _current_session_weight(instrument=None) -> float:
    """Return weight multiplier based on current trading session & instrument config."""
    session_name = _detect_session_name()
    if instrument and instrument.macro_driver and instrument.macro_driver.session_weights:
        return float(instrument.macro_driver.session_weights.get(session_name, 0.7))
    # Fallback: hardcoded gold session weights
    weights = {"ASIA": 0.5, "LONDON": 1.0, "NEW_YORK": 1.0,
               "ASIA_LONDON_OVERLAP": 1.2, "LONDON_NY_OVERLAP": 1.2, "WEEKEND_CLOSED": 0.0}
    return float(weights.get(session_name, 0.7))


# ── Main engine ────────────────────────────────────────────────────────

def load_market_snapshot(path: Optional[Path] = None) -> Optional[dict]:
    """Load the EA Bridge market snapshot."""
    p = path or DEFAULT_SNAPSHOT
    try:
        if not p.exists():
            return None
        return json.loads(p.read_text(encoding="utf-8"))
    except Exception:
        return None


def compute_correlation_signal(snapshot: Optional[dict] = None, symbol: str = None) -> dict[str, Any]:
    """
    Main entry point. Computes macro-driver correlation signal with noise filtering.
    For non-macro-driver-relevant symbols, returns neutral immediately.
    Generic across all instruments via SymbolRegistry — not gold-specific.

    Returns a dict designed for injection into session_analyzer and Commander prompt.
    """
    if not symbol:
        try:
            from bridge.gold_trader import _detect_active_symbol
            symbol = _detect_active_symbol()
        except Exception:
            symbol = GOLD_SYMBOL
    symbol = str(symbol).strip().upper()

    # ── Resolve instrument via registry ──
    instrument = _resolve_instrument(symbol)
    fp = _get_filter_params(instrument)
    md = instrument.macro_driver
    driver_symbol = md.driver_symbol if md else DXY_SYMBOL
    proxy_symbols = md.proxy_symbols if md else USD_PROXIES
    corr_sign = md.correlation_sign.value if md else "inverse"

    # Non-macro-driver assets (crypto, standalone) → skip, return neutral
    if md is None:
        return {
            "ok": True,
            "symbol": symbol,
            "dxy_relevant": False,
            "dxy_available": False,
            "dxy_correlation": None,
            "dxy_roc_norm": None,
            "dxy_headwind_tailwind": None,
            "gold_implication": None,
            "macro_implication": None,
            "macro_tailwind": None,
            "dxy_confirmed": False,
            "dxy_direction": "neutral",
            "reasons": [f"{symbol} has no macro driver configured"],
            "errors": [],
            "warnings": [],
        }

    if snapshot is None:
        snapshot = load_market_snapshot()

    if snapshot is None:
        return _empty_result("no_snapshot")

    instruments = snapshot.get("instruments", {})
    sym_data = instruments.get(symbol, instruments.get(GOLD_SYMBOL, {}))
    driver_data = instruments.get(driver_symbol, {})

    sym_bars = sym_data.get("bars", []) if isinstance(sym_data, dict) else []
    dxy_bars = driver_data.get("bars", []) if isinstance(driver_data, dict) else []

    # Try proxy symbols if driver unavailable
    proxy_used = None
    if not dxy_bars or len(dxy_bars) < fp.get("correlation_window", CORRELATION_WINDOW):
        for proxy in proxy_symbols:
            proxy_data = instruments.get(proxy, {})
            proxy_bars = proxy_data.get("bars", []) if isinstance(proxy_data, dict) else []
            if proxy_bars and len(proxy_bars) >= fp.get("correlation_window", CORRELATION_WINDOW):
                dxy_bars = proxy_bars
                proxy_used = proxy
                break

    corr_window = fp.get("correlation_window", CORRELATION_WINDOW)

    if not sym_bars or not dxy_bars:
        return _empty_result("insufficient_bars",
                             sym_bars=len(sym_bars), driver_bars=len(dxy_bars))

    min_len = min(len(sym_bars), len(dxy_bars))
    if min_len < corr_window:
        return _empty_result("insufficient_bars_short",
                             sym_bars=len(sym_bars), driver_bars=len(dxy_bars),
                             min_required=corr_window)

    # Align bar lengths
    sym_bars = sym_bars[-min_len:]
    dxy_bars = dxy_bars[-min_len:]

    # Extract closes
    sym_closes = [b["close"] for b in sym_bars]
    dxy_closes = [b["close"] for b in dxy_bars]

    # ── L1: EMA smoothing ──────────────────────────────────────────
    ema_period = fp.get("ema_smooth_period", EMA_SMOOTH_PERIOD)
    dxy_smooth = _ema(dxy_closes, ema_period)
    sym_smooth = _ema(sym_closes, ema_period)

    # ── L2: Rate of Change ─────────────────────────────────────────
    roc_period = fp.get("roc_period", ROC_PERIOD)
    dxy_roc = _roc(dxy_smooth, roc_period)
    sym_roc = _roc(sym_smooth, roc_period)

    # ── L3: Z-score gate ──────────────────────────────────────────
    z_window = fp.get("z_score_window", Z_SCORE_WINDOW)
    dxy_roc_z = _rolling_zscore(dxy_roc, z_window)

    # ── L4: Rolling correlation validation ─────────────────────────
    corr_values = _rolling_correlation(dxy_smooth, sym_smooth, corr_window)

    # ── Latest values ──────────────────────────────────────────────
    latest_dxy_roc = dxy_roc[-1] if dxy_roc else 0.0
    latest_dxy_z = dxy_roc_z[-1] if dxy_roc_z else 0.0
    latest_corr = corr_values[-1] if corr_values else 0.0
    latest_dxy = dxy_closes[-1] if dxy_closes else 0.0

    # ── L5: Direction persistence check ────────────────────────────
    dir_persistence = fp.get("direction_persistence", DIRECTION_PERSISTENCE)
    recent_dxy_roc = dxy_roc[-dir_persistence:]
    dxy_rising_bars = sum(1 for v in recent_dxy_roc if v > 0.02)
    dxy_falling_bars = sum(1 for v in recent_dxy_roc if v < -0.02)
    sustained_rising = dxy_rising_bars >= dir_persistence
    sustained_falling = dxy_falling_bars >= dir_persistence

    # ── Correlation active check ───────────────────────────────────
    corr_min_abs = fp.get("correlation_min_abs", CORRELATION_MIN_ABS)
    corr_active = abs(latest_corr) >= corr_min_abs

    # ── Signal determination (generic — uses instrument's correlation_sign) ──
    z_threshold = fp.get("z_score_threshold", Z_SCORE_THRESHOLD)
    dxy_direction = "FLAT"
    driver_implication = "neutral"

    if sustained_rising and abs(latest_dxy_z) >= z_threshold:
        dxy_direction = "RISING"
        if corr_active and latest_corr < 0:
            # Negative correlation: driver RISING = unfavorable for symbol (headwind)
            driver_implication = "headwind" if corr_sign == "inverse" else "tailwind"
        elif corr_active and latest_corr > 0:
            # Positive correlation: driver RISING = favorable for symbol (tailwind)
            driver_implication = "tailwind" if corr_sign == "inverse" else "headwind"
        else:
            driver_implication = "neutral"
    elif sustained_falling and abs(latest_dxy_z) >= z_threshold:
        dxy_direction = "FALLING"
        if corr_active and latest_corr < 0:
            driver_implication = "tailwind" if corr_sign == "inverse" else "headwind"
        elif corr_active and latest_corr > 0:
            driver_implication = "headwind" if corr_sign == "inverse" else "tailwind"
        else:
            driver_implication = "neutral"
    else:
        dxy_direction = "FLAT"
        driver_implication = "neutral"

    # Backward-compat alias
    gold_implication = driver_implication

    # ── Noise level assessment ─────────────────────────────────────
    z_abs = abs(latest_dxy_z)
    if z_abs >= 1.5 and corr_active and (sustained_rising or sustained_falling):
        noise_level = "low"
    elif z_abs >= 1.0:
        noise_level = "medium"
    else:
        noise_level = "high"

    # ── Confidence score ───────────────────────────────────────────
    confidence = _compute_confidence(
        z_abs=z_abs,
        corr_abs=abs(latest_corr),
        sustained=True if (sustained_rising or sustained_falling) else False,
        corr_active=corr_active,
    )

    # ── L6: Session-aware weighting (per-instrument config) ────────
    session_weight = _current_session_weight(instrument)

    # ── Driver ATR for move normalization ──────────────────────────
    atr_period = fp.get("atr_period", ATR_PERIOD)
    dxy_atr = _atr_from_bars(dxy_bars, period=atr_period)
    dxy_move_atr = abs(latest_dxy_roc / 100.0 * latest_dxy) / dxy_atr if dxy_atr > 0 else 0.0

    return {
        "timestamp": int(time.time()),
        "source": "correlation_engine",
        "symbol": symbol,
        "proxy_used": proxy_used,
        # Core signal — generic + backward-compat
        "dxy_direction": dxy_direction,
        "gold_implication": gold_implication,       # legacy key
        "macro_implication": driver_implication,    # generic alias
        "confidence": round(confidence, 3),
        "noise_level": noise_level,
        # Metrics
        "dxy_price": round(latest_dxy, 2),
        "dxy_roc_pct": round(latest_dxy_roc, 4),
        "dxy_roc_zscore": round(latest_dxy_z, 2),
        "dxy_move_atr": round(dxy_move_atr, 2),
        "correlation_dxy_gold": round(latest_corr, 3),
        "correlation_active": corr_active,
        "sustained_rising": sustained_rising,
        "sustained_falling": sustained_falling,
        "session_weight": session_weight,
        # Filter states (from per-instrument config)
        "filters": {
            "ema_smoothed": True,
            "roc_period": fp.get("roc_period", ROC_PERIOD),
            "zscore_window": fp.get("z_score_window", Z_SCORE_WINDOW),
            "zscore_threshold": Z_SCORE_THRESHOLD,
            "correlation_window": CORRELATION_WINDOW,
            "correlation_min_abs": CORRELATION_MIN_ABS,
            "direction_persistence": DIRECTION_PERSISTENCE,
        },
    }


def _compute_confidence(
    z_abs: float, corr_abs: float, sustained: bool, corr_active: bool,
) -> float:
    """Composite confidence 0-1 based on all filter layers."""
    score = 0.0

    # Z-score contribution (max 0.4)
    z_clipped = min(z_abs, 2.5)
    score += (z_clipped / 2.5) * 0.4

    # Correlation strength contribution (max 0.3)
    if corr_active:
        score += min(corr_abs, 0.8) / 0.8 * 0.3

    # Direction persistence contribution (max 0.3)
    if sustained:
        score += 0.3

    return min(1.0, score)


def _empty_result(reason: str, **extra) -> dict[str, Any]:
    return {
        "timestamp": int(time.time()),
        "source": "correlation_engine",
        "proxy_used": None,
        "dxy_direction": "FLAT",
        "gold_implication": "neutral",
        "macro_implication": "neutral",
        "confidence": 0.0,
        "noise_level": "high",
        "dxy_price": 0.0,
        "dxy_roc_pct": 0.0,
        "dxy_roc_zscore": 0.0,
        "dxy_move_atr": 0.0,
        "correlation_dxy_gold": 0.0,
        "correlation_active": False,
        "sustained_rising": False,
        "sustained_falling": False,
        "session_weight": _current_session_weight(),
        "filters": {},
        "empty_reason": reason,
        **extra,
    }


# ── CLI test ────────────────────────────────────────────────────────────

if __name__ == "__main__":
    result = compute_correlation_signal()
    print(json.dumps(result, indent=2, ensure_ascii=False))
