#!/usr/bin/env python3
"""
Evolution Pipeline — strategy lifecycle management with AI-driven evolution.

Features:
  - auto_prune_strategies(): eliminate consistently underperforming strategies
  - _extract_ledger_insights(): extract loss patterns from real trading ledger
  - generate_strategy(): AI generates new strategy code
  - critique_and_regenerate(): AI critiques and improves strategies
  - run_evolution_cycle(): full auto-evolution orchestration

Usage:
  python3 src/evolution_pipeline.py --prune           # manual prune check
  python3 src/evolution_pipeline.py --evolve           # run one evolution cycle
  python3 src/evolution_pipeline.py --auto             # continuous auto-evolution loop
"""

from __future__ import annotations

import argparse
import copy
import json
import os
import pathlib
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional

# ROOT now resolved via frozen_path (PyInstaller-safe)
from frozen_path import APP_ROOT as ROOT
sys.path.insert(0, str(ROOT))

import config_manager

DATA_DIR = config_manager.get_data_dir()
LOGS_DIR = config_manager.get_logs_dir()
CONFIG_DIR = config_manager.get_config_dir()

# ── Paths ───────────────────────────────────────────────────────

STRATEGY_POOL_DIR = DATA_DIR / "strategy_pool"
GRAVEYARD_DIR = STRATEGY_POOL_DIR / "graveyard"
RANKING_FILE = DATA_DIR / "offline" / "strategy_ranking.json"
PRUNE_HISTORY_FILE = LOGS_DIR / "prune_history.json"
LEDGER_FILE = LOGS_DIR / "trading_ledger.ndjson"
EVOLUTION_LOG = LOGS_DIR / "evolution_pipeline.log"

# ── Built-in strategies (never pruned) ──────────────────────────

BUILTIN_STRATEGY_IDS = {
    "ema_macd", "wyckoff",
    "trend_following", "mean_reversion", "breakout",
}


def _ensure_dirs() -> None:
    STRATEGY_POOL_DIR.mkdir(parents=True, exist_ok=True)
    GRAVEYARD_DIR.mkdir(parents=True, exist_ok=True)


def _now_iso() -> str:
    return datetime.now(timezone.utc).isoformat()


def _log(msg: str) -> None:
    line = f"[{_now_iso()}] {msg}"
    print(line)
    try:
        with open(EVOLUTION_LOG, "a", encoding="utf-8") as f:
            f.write(line + "\n")
    except Exception:
        pass


# ═══════════════════════════════════════════════════════════════════
# Feedback Loop 3: Auto-Prune
# ═══════════════════════════════════════════════════════════════════

def _load_ranking() -> list[dict]:
    """Load strategy ranking from disk."""
    if not RANKING_FILE.exists():
        return []
    try:
        return json.loads(RANKING_FILE.read_text(encoding="utf-8"))
    except Exception:
        return []


def _load_prune_history() -> dict[str, list[dict]]:
    """Load prune history from disk."""
    if not PRUNE_HISTORY_FILE.exists():
        return {}
    try:
        return json.loads(PRUNE_HISTORY_FILE.read_text(encoding="utf-8"))
    except Exception:
        return {}


def _save_prune_history(history: dict[str, list[dict]]) -> None:
    PRUNE_HISTORY_FILE.write_text(json.dumps(history, indent=2, default=str))


def _strategy_ranking_to_records(ranking: list[dict]) -> dict[str, list[dict]]:
    """Convert ranking list to per-strategy history records.

    Expects ranking entries with: strategy_id, score, win_rate, total_pnl, trades, ts
    """
    records: dict[str, list[dict]] = {}
    for entry in ranking:
        sid = entry.get("strategy_id", "unknown")
        if sid not in records:
            records[sid] = []
        records[sid].append({
            "score": float(entry.get("score", 0)),
            "win_rate": float(entry.get("win_rate", 0)),
            "pnl": float(entry.get("total_pnl", 0)),
            "trades": int(entry.get("total_trades", 0)),
            "ts": entry.get("ts", _now_iso()),
        })
    return records


def auto_prune_strategies(dry_run: bool = False) -> list[str]:
    """Check strategy ranking and eliminate consistently underperforming strategies.

    Rules:
      - Built-in strategies are NEVER pruned
      - Strategy must appear in ranking 3+ times AND be bottom-ranked 3 consecutive times
      - Score must be < 0 to qualify for pruning
      - Pruned strategies are moved to strategy_pool/graveyard/

    Returns list of pruned strategy IDs.
    """
    _ensure_dirs()
    ranking = _load_ranking()
    if not ranking:
        _log("[prune] No ranking data found — skipping")
        return []

    records = _strategy_ranking_to_records(ranking)
    prune_candidates: list[str] = []

    for sid, history in records.items():
        if sid in BUILTIN_STRATEGY_IDS:
            continue
        if len(history) < 3:
            continue

        recent = history[-3:]
        scores = [r["score"] for r in recent]
        if all(s < 0 for s in scores):
            prune_candidates.append(sid)

    if not prune_candidates:
        _log(f"[prune] All {len(records)} strategies healthy — no candidates")
        return []

    pruned: list[str] = []
    prune_history = _load_prune_history()

    for sid in prune_candidates:
        _log(f"[prune] Eliminating {sid}: last 3 scores {[r['score'] for r in records[sid][-3:]]}")

        if not dry_run:
            # Move strategy files to graveyard
            strategy_files = list(STRATEGY_POOL_DIR.glob(f"{sid}*"))
            for sf in strategy_files:
                dest = GRAVEYARD_DIR / sf.name
                sf.rename(dest)
                _log(f"[prune] Moved {sf.name} → graveyard/")

            # Record in prune history
            prune_history[sid] = prune_history.get(sid, [])
            prune_history[sid].append({
                "pruned_at": _now_iso(),
                "last_scores": [r["score"] for r in records[sid][-3:]],
                "last_win_rate": records[sid][-1]["win_rate"],
                "last_pnl": records[sid][-1]["pnl"],
            })
            pruned.append(sid)

    if not dry_run and pruned:
        _save_prune_history(prune_history)
        _log(f"[prune] Done — {len(pruned)} strategies pruned")

    return pruned


# ═══════════════════════════════════════════════════════════════════
# Feedback Loop 5: Ledger Insights
# ═══════════════════════════════════════════════════════════════════

def _extract_ledger_insights(max_trades: int = 200) -> dict[str, Any]:
    """Extract loss patterns and key metrics from the real trading ledger.

    Returns dict with:
      - loss_rate: percentage of losing trades in recent window
      - worst_session: session with most losses
      - worst_side: side with most losses (BUY/SELL)
      - worst_zone: timezone with most losses
      - sample_losses: 3 representative loss examples
      - recent_win_rate: win rate in last N trades
    """
    if not LEDGER_FILE.exists():
        return {"available": False, "reason": "Ledger file not found"}

    try:
        trades: list[dict] = []
        with open(LEDGER_FILE, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                try:
                    trades.append(json.loads(line))
                except json.JSONDecodeError:
                    continue

        if not trades:
            return {"available": False, "reason": "Ledger empty"}

        recent = trades[-max_trades:]
        losses = [t for t in recent if float(t.get("pnl", 0) or 0) < 0]
        wins = [t for t in recent if float(t.get("pnl", 0) or 0) > 0]

        loss_rate = len(losses) / max(1, len(recent)) * 100
        recent_wr = len(wins) / max(1, len(recent)) * 100

        # Per-side loss analysis
        side_losses: dict[str, int] = {}
        for t in losses:
            side = str(t.get("side", "BUY")).upper()
            side_losses[side] = side_losses.get(side, 0) + 1
        worst_side = max(side_losses, key=side_losses.get) if side_losses else "N/A"

        # Per-session loss analysis (extract hour from ts)
        session_losses: dict[str, int] = {"ASIA": 0, "LONDON": 0, "NEW_YORK": 0}
        for t in losses:
            ts_str = str(t.get("ts", ""))
            hour = 0
            try:
                if "T" in ts_str:
                    hour = int(ts_str.split("T")[1].split(":")[0])
                elif " " in ts_str:
                    hour = int(ts_str.split(" ")[1].split(":")[0])
            except Exception:
                pass
            if 0 <= hour < 7:
                session_losses["ASIA"] += 1
            elif 7 <= hour < 13:
                session_losses["LONDON"] += 1
            else:
                session_losses["NEW_YORK"] += 1
        worst_session = max(session_losses, key=session_losses.get) if session_losses else "N/A"

        # Sample representative losses
        sample_losses = sorted(losses, key=lambda t: float(t.get("pnl", 0) or 0))[:3]
        samples = []
        for t in sample_losses:
            samples.append({
                "side": t.get("side", ""),
                "entry": t.get("entry", 0),
                "exit": t.get("exit", 0),
                "pnl": t.get("pnl", 0),
                "ts": str(t.get("ts", ""))[:19],
            })

        return {
            "available": True,
            "total_trades": len(recent),
            "loss_count": len(losses),
            "win_count": len(wins),
            "loss_rate": round(loss_rate, 1),
            "recent_win_rate": round(recent_wr, 1),
            "worst_side": worst_side,
            "worst_side_losses": side_losses.get(worst_side, 0),
            "worst_session": worst_session,
            "worst_session_losses": session_losses.get(worst_session, 0),
            "sample_losses": samples,
        }
    except Exception as e:
        return {"available": False, "reason": str(e)}


# ═══════════════════════════════════════════════════════════════════
# Strategy Generation & Critique
# ═══════════════════════════════════════════════════════════════════

def build_generation_prompt(ranking: list[dict], ledger_insights: dict) -> str:
    """Build a prompt for generating a new trading strategy."""
    top_strategies = sorted(
        ranking, key=lambda r: float(r.get("score", 0)), reverse=True
    )[:3] if ranking else []

    top_info = ""
    for i, s in enumerate(top_strategies):
        top_info += (
            f"  #{i+1}: {s.get('strategy_id', 'N/A')} — "
            f"score={float(s.get('score', 0)):.2f}, "
            f"win_rate={float(s.get('win_rate', 0)):.1f}%, "
            f"pnl=${float(s.get('total_pnl', 0)):.2f}\n"
        )

    ledger_info = ""
    if ledger_insights.get("available"):
        ledger_info = f"""
Real Trading Ledger Insights (last {ledger_insights.get('total_trades', 0)} trades):
  Loss Rate: {ledger_insights.get('loss_rate', 0)}%
  Recent Win Rate: {ledger_insights.get('recent_win_rate', 0)}%
  Worst Side: {ledger_insights.get('worst_side', 'N/A')} ({ledger_insights.get('worst_side_losses', 0)} losses)
  Worst Session: {ledger_insights.get('worst_session', 'N/A')} ({ledger_insights.get('worst_session_losses', 0)} losses)
  Sample Losses: {json.dumps(ledger_insights.get('sample_losses', []), indent=2)}
"""

    return f"""You are a professional quantitative trading strategist for XAUUSD (Gold).

## Current Strategy Rankings
{top_info if top_info else "No ranking data available."}

{ledger_info}

## Task
Generate a NEW Python strategy for XAUUSD gold trading. The strategy should:

1. Define a signal_generator class with:
   - __init__() → configure parameters (EMA periods, RSI thresholds, ATR multiplier, etc.)
   - generate_signal(market_data: dict) → return dict with direction (LONG/SHORT/NEUTRAL), confidence (0-1), reason
   - get_params() → return current parameters dict
   - update_params(params: dict) → update parameters

2. Use technical indicators that address the weakness patterns identified in the ledger

3. Include proper risk management (SL/TP in ATR multiples, position sizing by volatility)

4. Be self-contained — all imports in the class file

5. Include docstring explaining the strategy logic and when it works best

Output the complete Python code for the strategy class. Use strategy class name: EvolvedStrategy."""


def build_critique_prompt(strategy_code: str, ranking: list[dict],
                          ledger_insights: dict) -> str:
    """Build a prompt for critiquing an existing strategy."""
    return f"""You are a senior quantitative trading strategist reviewing a candidate strategy for XAUUSD.

## Ledger Context
Recent Win Rate: {ledger_insights.get('recent_win_rate', 'N/A')}%
Worst Session: {ledger_insights.get('worst_session', 'N/A')}
Worst Side: {ledger_insights.get('worst_side', 'N/A')}

## Candidate Strategy Code
```python
{strategy_code}
```

## Review Criteria
1. **Overfitting**: Does it overfit to specific market conditions?
2. **Risk Management**: Are SL/TP correctly implemented?
3. **Edge Cases**: Does it handle low-volume, spread spikes, news events?
4. **Addressing Weaknesses**: Does it specifically address the loss patterns identified?
5. **Code Quality**: Is it clean, efficient, and maintainable?

## Output Format
```json
{{
  "verdict": "APPROVE|REVISE|REJECT",
  "confidence": 0.8,
  "issues": ["issue1", "issue2"],
  "improvements": "Suggested code changes if REVISE...",
  "risk_assessment": "Brief risk evaluation"
}}
```"""


def generate_strategy(ranking: list[dict],
                      ledger_insights: Optional[dict] = None) -> Optional[str]:
    """Use LLM to generate a new strategy based on current performance data."""
    from src.agent_role_router import call_llm, ROLE_TRAINER

    insights = ledger_insights or _extract_ledger_insights()
    prompt = build_generation_prompt(ranking, insights)

    _log("[generate] Calling LLM to generate new strategy...")
    try:
        code = call_llm(
            ROLE_TRAINER,
            prompt,
            temperature=0.7,
            max_tokens=2000,
            timeout=180,
        )
        # Extract code block if present
        if "```python" in code:
            code = code.split("```python")[1].split("```")[0].strip()
        elif "```" in code:
            parts = code.split("```")
            if len(parts) >= 2:
                code = parts[1].strip()
        _log(f"[generate] Generated {len(code)} chars of strategy code")
        return code
    except Exception as e:
        _log(f"[generate] Failed: {e}")
        return None


def critique_and_regenerate(strategy_code: str, ranking: list[dict],
                            ledger_insights: Optional[dict] = None,
                            max_iterations: int = 3) -> Optional[str]:
    """Critique strategy and iteratively improve it."""
    from src.agent_role_router import call_llm, ROLE_TRAINER

    insights = ledger_insights or _extract_ledger_insights()
    current_code = strategy_code

    for iteration in range(max_iterations):
        _log(f"[critique] Iteration {iteration + 1}/{max_iterations}")
        critique_prompt = build_critique_prompt(current_code, ranking, insights)

        try:
            response = call_llm(
                ROLE_TRAINER,
                critique_prompt,
                temperature=0.3,
                max_tokens=1500,
                timeout=120,
            )

            # Parse JSON from response
            start = response.find("{")
            end = response.rfind("}")
            if start == -1 or end == -1:
                _log("[critique] No JSON found in response — keeping original")
                break

            verdict = json.loads(response[start:end+1])
            verdict_type = verdict.get("verdict", "REVISE")

            if verdict_type == "APPROVE":
                _log("[critique] Strategy APPROVED")
                break
            elif verdict_type == "REJECT":
                _log(f"[critique] Strategy REJECTED: {verdict.get('risk_assessment', '')}")
                return None
            else:  # REVISE
                improvements = verdict.get("improvements", "")
                if not improvements:
                    _log("[critique] No improvements suggested — keeping current")
                    break

                # Apply improvements (simple: prepend as comment, then regenerate)
                revision_prompt = f"""Revise the following strategy based on this feedback:

FEEDBACK:
{improvements}

ORIGINAL CODE:
```python
{current_code}
```

Output the complete revised code:"""

                revised = call_llm(
                    ROLE_TRAINER,
                    revision_prompt,
                    temperature=0.5,
                    max_tokens=2000,
                    timeout=180,
                )

                if "```python" in revised:
                    revised = revised.split("```python")[1].split("```")[0].strip()
                elif "```" in revised:
                    parts = revised.split("```")
                    if len(parts) >= 2:
                        revised = parts[1].strip()

                if revised and len(revised) > 100:
                    current_code = revised
                    _log(f"[critique] Revised code: {len(revised)} chars")
                else:
                    _log("[critique] Revision too short — keeping original")
                    break

        except Exception as e:
            _log(f"[critique] Error: {e}")
            break

    return current_code


# ═══════════════════════════════════════════════════════════════════
# Evolution Orchestration
# ═══════════════════════════════════════════════════════════════════

def run_evolution_cycle(dry_run: bool = False) -> dict[str, Any]:
    """Run one complete evolution cycle: prune → generate → critique → save.

    Returns:
        {"pruned": [...], "generated": bool, "strategy_file": str|None}
    """
    _ensure_dirs()
    _log("=== Starting evolution cycle ===")

    result: dict[str, Any] = {"pruned": [], "generated": False, "strategy_file": None}

    # Step 1: Prune underperformers
    _log("Step 1/4: Pruning...")
    pruned = auto_prune_strategies(dry_run=dry_run)
    result["pruned"] = pruned

    # Step 2: Load current state
    _log("Step 2/4: Loading ranking & ledger insights...")
    ranking = _load_ranking()
    ledger_insights = _extract_ledger_insights()

    if ledger_insights.get("available"):
        _log(f"  Ledger: {ledger_insights['total_trades']} trades, "
             f"loss_rate={ledger_insights['loss_rate']}%, "
             f"worst_session={ledger_insights['worst_session']}")

    # Step 3: Generate new strategy
    _log("Step 3/4: Generating new strategy...")
    strategy_code = generate_strategy(ranking, ledger_insights)
    if not strategy_code:
        _log("[evolve] Strategy generation failed — cycle aborted")
        return result

    # Step 4: Critique and improve
    _log("Step 4/4: Critiquing and refining...")
    final_code = critique_and_regenerate(strategy_code, ranking, ledger_insights, max_iterations=3)
    if not final_code:
        _log("[evolve] Strategy rejected after critique — no new strategy saved")
        return result

    # Save the evolved strategy
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    strategy_file = STRATEGY_POOL_DIR / f"evolved_{timestamp}.py"
    strategy_file.write_text(final_code)
    _log(f"[evolve] Saved evolved strategy → {strategy_file}")

    result["generated"] = True
    result["strategy_file"] = str(strategy_file)
    _log("=== Evolution cycle complete ===")
    return result


# ═══════════════════════════════════════════════════════════════════
# CLI
# ═══════════════════════════════════════════════════════════════════

def main() -> None:
    ap = argparse.ArgumentParser(description="Strategy Evolution Pipeline")
    ap.add_argument("--prune", action="store_true", help="Run auto-prune check")
    ap.add_argument("--prune-dry-run", action="store_true", help="Prune dry run (report only)")
    ap.add_argument("--evolve", action="store_true", help="Run one evolution cycle")
    ap.add_argument("--auto", action="store_true", help="Continuous auto-evolution loop")
    ap.add_argument("--interval", type=int, default=86400,
                    help="Auto-evolution interval in seconds (default: 86400 = 24h)")
    args = ap.parse_args()

    if args.prune:
        pruned = auto_prune_strategies(dry_run=False)
        print(f"Pruned: {pruned}" if pruned else "No strategies pruned.")
    elif args.prune_dry_run:
        candidates = auto_prune_strategies(dry_run=True)
        print(f"Would prune: {candidates}" if candidates else "No candidates.")
    elif args.evolve:
        result = run_evolution_cycle()
        print(json.dumps(result, indent=2, default=str))
    elif args.auto:
        _log("Starting auto-evolution loop...")
        while True:
            try:
                result = run_evolution_cycle()
                _log(f"Cycle result: {json.dumps(result, default=str)}")
            except Exception as e:
                _log(f"Auto-evolution error: {e}")
            _log(f"Sleeping {args.interval}s until next cycle...")
            time.sleep(args.interval)
    else:
        # Default: show status
        ranking = _load_ranking()
        insights = _extract_ledger_insights()
        print(f"Strategy Pool: {STRATEGY_POOL_DIR}")
        print(f"  Strategies: {len(list(STRATEGY_POOL_DIR.glob('*.py'))) if STRATEGY_POOL_DIR.exists() else 0}")
        print(f"  Graveyard: {len(list(GRAVEYARD_DIR.glob('*'))) if GRAVEYARD_DIR.exists() else 0}")
        print(f"Ranking entries: {len(ranking)}")
        print(f"Ledger insights: {json.dumps(insights, indent=2, default=str)}")


if __name__ == "__main__":
    main()
