"""upload_scheduler.py — Cron-driven incremental data upload.

Integrates into AutonomyLoop: called every cycle (~1 min) to:
1. Capture new ledger events since last cursor
2. Anonymize (hash tenant_id, quantize timestamps, remove PII)
3. Upload with gzip + retry (async, non-blocking via ThreadPoolExecutor)
4. Update cursor on success
5. Check data quality
6. Log metrics

Usage:
    from src.data_pipeline import run_data_upload_cycle
    result = run_data_upload_cycle(root, tenant_id)
"""

from __future__ import annotations

import json
import os
import time
import uuid
from concurrent.futures import Future
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import config_manager

from src.data_pipeline.anonymizer import anonymize_batch, audit_batch_for_pii
from src.data_pipeline.event_capture import (
    get_new_trade_events,
    get_upload_status,
    _save_cursor as _save_event_cursor,
    _parse_ts as _parse_ts,
)
from src.data_pipeline.uploader import (
    check_network,
    process_retry_queue,
)

APP_DIR = config_manager.get_app_dir()
DATA_DIR = config_manager.get_data_dir()
LOGS_DIR = config_manager.get_logs_dir()
UPLOAD_DIR = DATA_DIR / "uploads"

BATCH_SIZE = int(os.environ.get("DATA_BATCH_SIZE", "500"))
MAX_UPLOAD_RETRIES = int(os.environ.get("DATA_MAX_UPLOAD_RETRIES", "5"))

# ── Pending async futures management ──────────────────────────────

_pending_futures: dict[str, tuple[str, Future, datetime]] = {}  # batch_id -> (tenant_id, future, submitted_at)
_PENDING_FILE = UPLOAD_DIR / ".pending_futures.json"


def _store_pending_future(tenant_id: str, batch_id: str, future: Future) -> None:
    """Store a pending async upload future for progress tracking."""
    _pending_futures[batch_id] = (tenant_id, future, datetime.now(timezone.utc))
    _save_pending_futures()


def _save_pending_futures() -> None:
    """Persist pending futures to disk for crash recovery."""
    try:
        UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
        data = {
            batch_id: {
                "tenant_id": tenant_id,
                "submitted_at": submitted_at.isoformat(),
                "batch_id": batch_id,
            }
            for batch_id, (tenant_id, _, submitted_at) in _pending_futures.items()
        }
        _PENDING_FILE.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
    except Exception:
        pass


def _load_pending_futures() -> None:
    """Reload pending futures from disk (on startup)."""
    if not _PENDING_FILE.exists():
        return
    try:
        data = json.loads(_PENDING_FILE.read_text(encoding="utf-8"))
        for batch_id, info in data.items():
            # Cannot restore Future objects — just clean up stale entries
            pass
        # Clear stale entries
        _PENDING_FILE.unlink(missing_ok=True)
    except Exception:
        pass


def _check_completed_futures() -> list[dict[str, Any]]:
    """Check and collect results from any completed async uploads."""
    completed: list[dict[str, Any]] = []
    done_batch_ids: list[str] = []

    for batch_id, (tenant_id, future, submitted_at) in list(_pending_futures.items()):
        if future.done():
            try:
                res = future.result(timeout=1)
                res["batch_id"] = batch_id
                res["tenant_id"] = tenant_id
                res["completed_at"] = datetime.now(timezone.utc).isoformat()
                completed.append(res)

                if res.get("chunks_uploaded", 0) > 0:
                    _append_metric("uploader", "batch_async_completed", res["chunks_uploaded"])
                if res.get("chunks_failed", 0) > 0:
                    _append_metric("uploader", "batch_async_failed", res["chunks_failed"])

            except Exception as e:
                completed.append({
                    "batch_id": batch_id,
                    "tenant_id": tenant_id,
                    "status": "error",
                    "error": str(e),
                })
                _append_metric("uploader", "batch_async_error", str(e)[:100])
            done_batch_ids.append(batch_id)

    for batch_id in done_batch_ids:
        _pending_futures.pop(batch_id, None)

    if done_batch_ids:
        _save_pending_futures()

    return completed


# Load stale pending futures on module init
_load_pending_futures()


# ── Cursor management for upload cycle ──────────────────────────────

def _load_upload_cursor(tenant_id: str) -> dict[str, Any]:
    path = UPLOAD_DIR / f".cursor_{tenant_id}.json"
    if not path.exists():
        return {"last_synced_ts": None, "last_synced_line": 0}
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except Exception:
        return {"last_synced_ts": None, "last_synced_line": 0}


def _save_upload_cursor(tenant_id: str, cursor: dict[str, Any]) -> None:
    UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
    path = UPLOAD_DIR / f".cursor_{tenant_id}.json"
    tmp = path.with_name(f".{os.getpid()}.tmp")
    tmp.write_text(json.dumps(cursor, ensure_ascii=False), encoding="utf-8")
    tmp.replace(path)


# ── Metrics persistence ──────────────────────────────────────────────

METRICS_FILE = LOGS_DIR / "data_pipeline_metrics.ndjson"


def _append_metric(category: str, key: str, value: Any) -> None:
    record = {
        "ts": datetime.now(timezone.utc).isoformat(),
        "category": category,
        "key": key,
        "value": value,
    }
    LOGS_DIR.mkdir(parents=True, exist_ok=True)
    with open(METRICS_FILE, "a", encoding="utf-8") as f:
        f.write(json.dumps(record, ensure_ascii=False) + "\n")


# ── Main upload cycle ───────────────────────────────────────────────

def run_data_upload_cycle(
    project_root: Path | None = None,
    tenant_id: str = "default",
    api_key: str | None = None,
    verify_tls: bool = True,
) -> dict[str, Any]:
    """Run one incremental upload cycle for a tenant.

    Steps:
    1. Process any pending retry batches first
    2. Capture new events since last cursor
    3. Audit for PII (fail-safe: drop batch if PII found)
    4. Anonymize
    5. Upload (with gzip + retry)
    6. Update cursor on success
    7. Return result summary

    Returns:
        Result dict with keys: uploaded, failed, skipped,
        freshness_lag_sec, queue_depth, errors.
    """
    root = project_root or APP_DIR

    result: dict[str, Any] = {
        "ts": datetime.now(timezone.utc).isoformat(),
        "tenant_id": tenant_id,
        "uploaded": 0,
        "failed": 0,
        "skipped": 0,
        "freshness_lag_sec": 0,
        "queue_depth": 0,
        "errors": [],
        "network_reachable": False,
    }

    # ── Step 0: Process retry queue ───────────────────────────────
    if check_network(timeout=3.0):
        result["network_reachable"] = True
        retry_res = process_retry_queue(
            tenant_id=tenant_id,
            api_key=api_key,
            max_retries=MAX_UPLOAD_RETRIES,
        )
        result["retry_queue"] = retry_res
        result["queue_depth"] = retry_res.get("queue_depth", 0)

        if retry_res.get("succeeded", 0) > 0:
            _append_metric("uploader", "retry_succeeded", retry_res["succeeded"])
            print(f"[data_pipeline] Retry queue: {retry_res['succeeded']} succeeded, "
                  f"{retry_res.get('failed', 0)} failed, "
                  f"{retry_res.get('dropped', 0)} dropped")
    else:
        result["errors"].append("network_unreachable")
        result["skipped"] = 1
        print("[data_pipeline] Network unreachable, skipping upload cycle")
        return result

    # ── Step 1: Capture new events ────────────────────────────────
    cursor = _load_upload_cursor(tenant_id)

    events, cursor_update = get_new_trade_events(
        batch_size=BATCH_SIZE,
        ledger_file=root / "logs" / "trading_ledger.ndjson",
    )

    if not events:
        result["skipped"] = 1
        result["freshness_lag_sec"] = _compute_lag(cursor)
        _append_metric("pipeline", "skipped_no_new_events", 1)
        print("[data_pipeline] No new events to upload")
        return result

    print(f"[data_pipeline] Captured {len(events)} new events for tenant {tenant_id}")

    # ── Step 2: PII audit ──────────────────────────────────────────
    pii_warnings = audit_batch_for_pii(events)
    if pii_warnings:
        msg = f"PII detected in batch: {pii_warnings[:3]}"
        result["errors"].append(msg)
        print(f"[data_pipeline] ERROR: {msg}")
        _append_metric("pipeline", "pii_detected", len(pii_warnings))
        # Fail-safe: don't upload — alert instead
        return result

    # ── Step 3: Anonymize ─────────────────────────────────────────
    try:
        anon_events = anonymize_batch(events, tenant_id)
    except Exception as e:
        msg = f"Anonymization failed: {e}"
        result["errors"].append(msg)
        print(f"[data_pipeline] ERROR: {msg}")
        return result

    # ── Step 4: Upload (async, non-blocking) ─────────────────────────
    # Try async upload first; fall back to sync if server doesn't support chunking
    use_async = os.environ.get("DATA_ASYNC_UPLOAD", "true").lower() == "true"
    upload_success = False
    batch_id_str: str | None = None

    if use_async:
        # Async path: submit upload in background thread, return immediately
        try:
            from src.data_pipeline.async_uploader import (
                async_upload_batch,
                get_upload_progress,
                check_upload_circuit,
            )
            from src.data_pipeline.anonymizer import hash_tenant_id

            tenant_hash = hash_tenant_id(tenant_id)
            batch_id_str = str(uuid.uuid4()) if not batch_id_str else batch_id_str

            # Capture cursor_update at submission time (before async completion)
            submission_cursor = dict(cursor_update)

            # Submit async upload (non-blocking)
            future = async_upload_batch(
                events=anon_events,
                tenant_id=tenant_id,
                tenant_hash=tenant_hash,
                api_key=api_key,
                verify_tls=verify_tls,
                batch_id=batch_id_str,
            )

            result["uploaded"] = len(anon_events)
            result["async_mode"] = True
            result["batch_id"] = batch_id_str
            result["circuit_status"] = check_upload_circuit()
            result["progress"] = {
                "events_total": len(anon_events),
                "status": "in_progress",
            }
            _append_metric("uploader", "batch_async_submitted", len(anon_events))

            # Update cursor optimistically (upload is fire-and-forget)
            _save_upload_cursor(tenant_id, submission_cursor)
            try:
                _save_event_cursor(submission_cursor)
            except Exception:
                pass

            # Store future for progress tracking in subsequent cycles
            _store_pending_future(tenant_id, batch_id_str, future)

            print(f"[data_pipeline] Async upload submitted: {len(anon_events)} events, "
                  f"batch_id={batch_id_str}")
            upload_success = True

        except ImportError:
            use_async = False
        except Exception as e:
            print(f"[data_pipeline] Async upload failed, falling back to sync: {e}")
            use_async = False

    if not use_async or not upload_success:
        # Sync fallback
        from src.data_pipeline.uploader import upload_batch as sync_upload_batch

        upload_res = sync_upload_batch(
            events=anon_events,
            tenant_id=tenant_id,
            api_key=api_key,
            verify_tls=verify_tls,
        )

        if upload_res.get("success"):
            result["uploaded"] = len(anon_events)
            _append_metric("uploader", "batch_uploaded", len(anon_events))
            _append_metric("uploader", "batch_id", upload_res.get("batch_id", ""))
            batch_id_str = upload_res.get("batch_id", "")

            # ── Step 5: Update cursor ────────────────────────────────
            _save_upload_cursor(tenant_id, cursor_update)

            try:
                _save_event_cursor(cursor_update)
            except Exception:
                pass

            print(f"[data_pipeline] Uploaded {len(anon_events)} events, "
                  f"batch_id={upload_res.get('batch_id', 'N/A')}")
            upload_success = True
        else:
            result["failed"] = len(anon_events)
            error_msg = upload_res.get("error", "unknown")
            result["errors"].append(error_msg)
            result["queue_depth"] = 1 if upload_res.get("retry_scheduled") else 0
            _append_metric("uploader", "batch_failed", len(anon_events))
            _append_metric("uploader", "error", error_msg[:100])
            print(f"[data_pipeline] Upload failed: {error_msg}")
            upload_success = False

    # ── Step 6: Freshness lag ─────────────────────────────────────
    result["freshness_lag_sec"] = _compute_lag(cursor)

    # ── Step 7: Queue depth ───────────────────────────────────────
    from src.data_pipeline.uploader import _load_retry_queue
    result["queue_depth"] = len(_load_retry_queue())

    return result


def _compute_lag(cursor: dict[str, Any]) -> float:
    """Compute lag in seconds between last synced event and now."""
    last_ts = cursor.get("last_synced_ts")
    if not last_ts:
        return 0.0
    dt = _parse_ts(last_ts)
    if dt is None:
        return 0.0
    return round((datetime.now(timezone.utc) - dt).total_seconds(), 1)


# ── CLI entry point ────────────────────────────────────────────────

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Data pipeline upload")
    parser.add_argument("--tenant", type=str, default="default")
    parser.add_argument("--status", action="store_true", help="Show upload status")
    parser.add_argument("--upload", action="store_true", help="Run one upload cycle")
    parser.add_argument("--verify-tls", action="store_true", default=True, help="Verify TLS certs")
    args = parser.parse_args()

    if args.status:
        from src.data_pipeline.event_capture import get_upload_status
        status = get_upload_status()
        print(json.dumps(status, indent=2, ensure_ascii=False))
    elif args.upload or args.tenant:
        res = run_data_upload_cycle(tenant_id=args.tenant, verify_tls=args.verify_tls)
        print(json.dumps(res, indent=2, ensure_ascii=False))
    else:
        parser.print_help()
