#!/usr/bin/env python3
"""
从 MT5 导出的 HTML 报告文件中解析交易历史，
提取真实成交利润，生成标准 trading_ledger 格式。

用法：
    python3 parse_mt5_report.py /path/to/ReportHistory-XXXXX.html
    python3 parse_mt5_report.py --fix-ledger /path/to/ReportHistory-XXXXX.html
"""

import sys
import re
import json
from datetime import datetime, timezone, timedelta
from pathlib import Path
from bs4 import BeautifulSoup

MT5_TZ = timezone(timedelta(hours=3))  # MT5 broker UTC+3
SH_TZ = timezone(timedelta(hours=8))

LEDGER_PATH = Path(__file__).parent.parent / "logs" / "trading_ledger.ndjson"


def parse_mt5_html_report(html_path: str):
    """从 MT5 HTML 报告解析所有交易记录"""
    # Try UTF-16 first (common for MT5 exports), then UTF-8, then fallback
    for enc in ["utf-16", "utf-8", "latin-1"]:
        try:
            with open(html_path, "r", encoding=enc) as f:
                content = f.read()
            soup = BeautifulSoup(content, "html.parser")
            print(f"Successfully parsed with encoding: {enc}")
            break
        except UnicodeDecodeError:
            continue

    table = soup.find("table", {"id": "deal_table"})
    if not table:
        # Try finding by class or other attributes
        tables = soup.find_all("table")
        for t in tables:
            headers = t.find_all("th")
            header_texts = [h.get_text(strip=True) for h in headers]
            if "Ticket" in header_texts or "Deal" in header_texts:
                table = t
                break

    if not table:
        print("ERROR: Could not find deal table in HTML report")
        sys.exit(1)

    # Get headers
    headers = [th.get_text(strip=True) for th in table.find_all("th")]
    print(f"Table headers: {headers}")

    # Find column indices
    time_col = next((i for i, h in enumerate(headers) if "Time" in h or "Date" in h), None)
    ticket_col = next((i for i, h in enumerate(headers) if "Ticket" in h or "Deal" in h), None)
    symbol_col = next((i for i, h in enumerate(headers) if "Symbol" in h), None)
    side_col = next((i for i, h in enumerate(headers) if "Side" in h or "Action" in h or "Type" in h), None)
    volume_col = next((i for i, h in enumerate(headers) if "Volume" in h or "Lots" in h), None)
    price_col = next((i for i, h in enumerate(headers) if "Price" in h and "Action" not in h), None)
    profit_col = next((i for i, h in enumerate(headers) if "Profit" in h or "P/L" in h or "PL" in h), None)
    comment_col = next((i for i, h in enumerate(headers) if "Comment" in h), None)
    pos_id_col = next((i for i, h in enumerate(headers) if "Position" in h), None)

    print(f"Columns: time={time_col}, ticket={ticket_col}, symbol={symbol_col}, "
          f"side={side_col}, volume={volume_col}, price={price_col}, "
          f"profit={profit_col}, comment={comment_col}, pos_id={pos_id_col}")

    deals = []
    for row in table.find_all("tr")[1:]:  # skip header
        cells = row.find_all(["td", "th"])
        if len(cells) < len(headers):
            continue

        def get_col(idx):
            if idx is None or idx >= len(cells):
                return ""
            return cells[idx].get_text(strip=True)

        time_str = get_col(time_col)
        ticket_str = get_col(ticket_col)
        symbol = get_col(symbol_col)
        side = get_col(side_col)
        volume_str = get_col(volume_col)
        price_str = get_col(price_col)
        profit_str = get_col(profit_col)
        comment = get_col(comment_col)
        pos_id_str = get_col(pos_id_col)

        if not time_str:
            continue

        # Parse profit
        try:
            profit = float(profit_str.replace(",", "").replace("$", ""))
        except:
            profit = 0.0

        # Parse time (MT5 format: YYYY.MM.DD HH:MM:SS)
        # In UTC+3 MT5 broker time
        try:
            time_str_clean = time_str.replace(".", "-").replace(" ", "T")
            # Replace dots in date part only
            parts = time_str_clean.split("T")
            date_part = parts[0].replace(".", "-")
            time_str_iso = f"{date_part}T{parts[1]}"
            dt_mt5 = datetime.fromisoformat(time_str_iso).replace(tzinfo=MT5_TZ)
            dt_utc = dt_mt5.astimezone(timezone.utc)
            ts = dt_utc.isoformat()
        except Exception as e:
            print(f"Warning: could not parse time '{time_str}': {e}")
            continue

        # Parse ticket
        try:
            ticket = int(ticket_str) if ticket_str else 0
        except:
            ticket = 0

        # Parse side/action
        # MT5 deal types: 0=Buy, 1=Sell, 2=Buy Limit, 3=Sell Limit, etc.
        # We only care about actual executions (not pending orders)
        # Map to: BUY / SELL / CLOSE
        side_map = {
            "buy": "BUY", "sell": "SELL",
            "0": "BUY", "1": "SELL",
            "buy limit": "BUY_LIMIT", "sell limit": "SELL_LIMIT",
        }
        side_normalized = side_map.get(side.lower(), side) if side else "UNKNOWN"

        deals.append({
            "ts": ts,
            "ticket": ticket,
            "pos_id": pos_id_str,
            "symbol": symbol,
            "side": side_normalized,
            "action": "CLOSE",  # all deals in this report are closes
            "status": "FILLED",
            "profit": profit,
            "pnl": profit,  # use MT5 real profit
            "volume": volume_str,
            "price": price_str,
            "comment": comment,
            "_source": "mt5_html_report",
            "_recovered_from_ea": True,  # mark as verified real data
        })

    return deals


def fix_ledger_with_report(html_path: str):
    """用 MT5 报告数据替换/修复 ledger 中对应时间段的记录"""
    print(f"\n=== Parsing MT5 HTML report: {html_path} ===")
    deals = parse_mt5_html_report(html_path)
    print(f"Found {len(deals)} deal records in report")

    # Group deals by position_id to compute position-level PnL
    by_pos = {}
    for d in deals:
        pid = d.get("pos_id", d.get("ticket"))
        if pid not in by_pos:
            by_pos[pid] = []
        by_pos[pid].append(d)

    print(f"Unique positions in report: {len(by_pos)}")

    # For each position, sum all deal profits to get position PnL
    # Then create one ledger entry per position
    ledger_entries = []
    for pos_id, pos_deals in by_pos.items():
        total_pnl = sum(d["pnl"] for d in pos_deals)
        first_deal = pos_deals[0]
        last_deal = pos_deals[-1]

        # Use the CLOSE time as the position close time
        close_ts = last_deal["ts"]

        # Determine side from the opening deal
        # IN deals have entry=0, OUT deals have entry=1
        open_deal = next((d for d in pos_deals if "IN" in str(d.get("side", "")).upper()), None)
        close_deal = next((d for d in pos_deals if "OUT" in str(d.get("side", "")).upper()), None)
        
        side = open_deal["side"] if open_deal else (close_deal["side"] if close_deal else "BUY")
        entry_price = open_deal["price"] if open_deal else ""
        exit_price = close_deal["price"] if close_deal else last_deal.get("price", "")

        # Parse volume
        vol_str = first_deal.get("volume", "0.01")
        try:
            volume = float(vol_str)
        except:
            volume = 0.01

        ledger_entries.append({
            "ticket": int(pos_id) if pos_id and str(pos_id).isdigit() else 0,
            "pos_id": pos_id,
            "ts": close_ts,
            "side": side,
            "entry": entry_price,
            "exit": exit_price,
            "pnl": round(total_pnl, 2),
            "volume": volume,
            "action": "CLOSE",
            "status": "FILLED",
            "_recovered_from_ea": True,
            "_deal_count": len(pos_deals),
            "source": "mt5_html_report",
        })

    # Sort by close time
    ledger_entries.sort(key=lambda x: x["ts"])

    # Read existing ledger
    existing = []
    if LEDGER_PATH.exists():
        with open(LEDGER_PATH, "r", encoding="utf-8") as f:
            for line in f:
                if line.strip():
                    try:
                        existing.append(json.loads(line))
                    except:
                        pass

    print(f"Existing ledger records: {len(existing)}")

    # For each new entry, check if we already have it (by pos_id)
    existing_pos_ids = {e.get("pos_id") for e in existing if e.get("pos_id")}
    existing_tickets = {e.get("ticket") for e in existing if e.get("ticket")}

    added = 0
    for entry in ledger_entries:
        pid = entry.get("pos_id")
        tk = entry.get("ticket")
        # Add if not already present
        if pid not in existing_pos_ids and tk not in existing_tickets:
            existing.append(entry)
            added += 1

    print(f"Added {added} new records from MT5 report")
    print(f"Total ledger after fix: {len(existing)} records")

    # Show summary of new entries
    if ledger_entries:
        print("\nNew entries from report:")
        for e in ledger_entries:
            print(f"  {e['ts']} ticket={e['ticket']} pos_id={e['pos_id']} pnl={e['pnl']:.2f} side={e['side']}")

    # Rewrite ledger
    existing.sort(key=lambda x: x.get("ts", ""))
    with open(LEDGER_PATH, "w", encoding="utf-8") as f:
        for rec in existing:
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")

    print(f"\nLedger saved to {LEDGER_PATH}")
    
    # Summary stats
    total_pnl = sum(e.get("pnl", 0) for e in existing)
    wins = [e for e in existing if e.get("pnl", 0) > 0]
    losses = [e for e in existing if e.get("pnl", 0) < 0]
    print(f"Total: {len(existing)} trades, PnL={total_pnl:.2f}, Wins={len(wins)}, Losses={len(losses)}")
    print(f"Win rate: {len(wins)/len(existing)*100:.1f}%" if existing else "N/A")


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print(__doc__)
        sys.exit(1)

    html_path = sys.argv[-1]

    if "--fix-ledger" in sys.argv or len(sys.argv) == 2:
        fix_ledger_with_report(html_path)
    else:
        # Just parse and display
        deals = parse_mt5_html_report(html_path)
        print(f"\nTotal deals: {len(deals)}")
        by_pos = {}
        for d in deals:
            pid = d.get("pos_id", d.get("ticket"))
            if pid not in by_pos:
                by_pos[pid] = []
            by_pos[pid].append(d)

        print(f"Unique positions: {len(by_pos)}")
        for pos_id, pos_deals in sorted(by_pos.items(), key=lambda x: x[1][0]["ts"])[:10]:
            total = sum(d["pnl"] for d in pos_deals)
            print(f"  pos_id={pos_id} deals={len(pos_deals)} pnl={total:.2f} first={pos_deals[0]['ts']} last={pos_deals[-1]['ts']}")
