"""Atlas Brain - the collector behind Atlas Operations.

Pulls every signal into one atlas-state.json:
  mail      : 7 IMAP mailboxes (unread + recent headers), strictly read-only
  calendar  : Google secret-iCal feeds, recurrences expanded, next 14 days
  command   : atlas-01 today.json (attention / mesh)
  business  : atlas-01 atlas_data.json (Odoo CRM)
Then synthesizes a proactive brief via the local model-router.

Supersedes jarvis-brain-hourly.py / jarvis-brain-hourly-v2.py.

Run:
  python atlas_brain.py            full collect + synth + write state
  python atlas_brain.py --quick    collect only, skip the LLM synth (fast 15-min tick)
"""
from __future__ import annotations
import base64
import email
import imaplib
import json
import os
import re
import sys
import time
import traceback
from datetime import datetime, timezone, timedelta
from email.header import decode_header, make_header
from pathlib import Path

import httpx

HOME = Path.home()
ATLAS = HOME / ".config" / "atlas"
OPS_DIR = ATLAS / "operations"
CACHE = ATLAS / "cache"
OPS_DIR.mkdir(parents=True, exist_ok=True)
CACHE.mkdir(parents=True, exist_ok=True)

STATE_JSON = OPS_DIR / "atlas-state.json"
SPOKEN_JSON = ATLAS / "atlas-brief-spoken.json"
MAIL_CFG = ATLAS / "mail-accounts.json"
CAL_CFG = ATLAS / "calendar-feeds.json"
CAL_CACHE = CACHE / "calendar-last.json"
LOG = ATLAS / "atlas-brain.log"

ATLAS01 = os.environ.get("ATLAS01_URL", "http://atlas-01.tailab40ed.ts.net")
A01_USER = "atlas"
A01_PASS = os.environ.get("ATLAS_BASIC_PASS", "")

SERVICES = [
    {"name": "Command Center", "url": "https://atlascorporation.nl/command"},
    {"name": "Nextcloud", "url": "http://atlas-01:8443"},
    {"name": "MeshCentral", "url": "https://mesh.atlascorporation.nl/"},
    {"name": "Odoo", "url": "http://<TAILSCALE_IP>:8069"},
    {"name": "Atlas Portal", "url": "http://atlas-01/portal.html"},
    {"name": "n8n", "url": "http://<TAILSCALE_IP>:5678"},
    {"name": "Stalwart", "url": "http://<TAILSCALE_IP>:8080"},
    {"name": "Authentik", "url": "http://<TAILSCALE_IP>:9000"},
    {"name": "Ollama", "url": "http://localhost:11434"},
]

sys.path.insert(0, str(ATLAS))
try:
    import model_router
except Exception:
    model_router = None


def log(msg: str) -> None:
    try:
        with LOG.open("a", encoding="utf-8") as f:
            f.write(f"[{datetime.now().isoformat(timespec='seconds')}] {msg}\n")
    except Exception:
        pass
    print(f"[brain] {msg}")


def _safe(fn, name: str):
    try:
        return fn()
    except Exception as e:
        log(f"{name} FAILED: {e}")
        log(traceback.format_exc())
        return {"error": f"{name}: {e}"}


def _decode(s) -> str:
    if not s:
        return ""
    try:
        return str(make_header(decode_header(s)))
    except Exception:
        return str(s)


def _iso(d) -> str:
    if isinstance(d, datetime):
        if d.tzinfo is None:
            d = d.replace(tzinfo=timezone.utc)
        return d.astimezone().isoformat(timespec="minutes")
    return str(d)


def _strip_md(text: str) -> str:
    t = re.sub(r"```.*?```", "", text or "", flags=re.S)
    t = re.sub(r"[`*_#>]", "", t)
    t = re.sub(r"^\s*[-+]\s+", "", t, flags=re.M)
    t = re.sub(r"^\s*\d+\.\s+", "", t, flags=re.M)
    t = re.sub(r"\n{2,}", ". ", t)
    t = re.sub(r"\n", " ", t)
    t = re.sub(r"\s{2,}", " ", t)
    return t.strip()


def _extract_section(md: str, name: str) -> str:
    m = re.search(rf"##\s*{re.escape(name)}\s*\n(.*?)(?:\n##\s|\Z)", md or "", re.S)
    return m.group(1).strip() if m else ""


# ---------------- MAIL ----------------
def collect_mail() -> dict:
    if not MAIL_CFG.exists():
        return {"error": "mail-accounts.json not found", "accounts": [], "total_unread": 0}
    accounts = json.loads(MAIL_CFG.read_text(encoding="utf-8"))
    out, total_unread = [], 0
    for acc in accounts:
        rec = {"label": acc["label"], "group": acc.get("group", ""),
               "total": 0, "unread": 0, "recent": [], "error": None}
        _last_err = None
        for _attempt in range(2):
            try:
                M = imaplib.IMAP4_SSL(acc["host"], acc.get("port", 993), timeout=25)
                M.login(acc["user"], acc["password"])
                M.select("INBOX", readonly=True)
                ids = M.search(None, "ALL")[1][0].split()
                uids = M.search(None, "UNSEEN")[1][0].split()
                rec["total"] = len(ids)
                rec["unread"] = len(uids)
                total_unread += len(uids)
                for mid in reversed((uids or ids)[-6:]):
                    try:
                        md = M.fetch(mid.decode(), "(BODY.PEEK[HEADER.FIELDS (SUBJECT FROM DATE)])")[1]
                        raw = next((p[1] for p in md if isinstance(p, tuple)), b"")
                        msg = email.message_from_bytes(raw)
                        rec["recent"].append({
                            "subject": _decode(msg.get("Subject"))[:160],
                            "from": _decode(msg.get("From"))[:120],
                            "date": (msg.get("Date") or "")[:40],
                            "unread": mid in uids,
                        })
                    except Exception as fe:
                        log(f"  fetch glitch {acc['label']}: {fe}")
                M.logout()
                _last_err = None
                break  # success — exit retry loop
            except Exception as e:
                _last_err = e
                if _attempt == 0:
                    log(f"  mail retry {acc['label']}: {e}")
                    time.sleep(1)
        if _last_err is not None:
            rec["error"] = str(_last_err)[:200]
        out.append(rec)
        log(f"mail {acc['label']}: {rec['unread']} unread / {rec['total']}"
            + (f"  ERR {rec['error']}" if rec["error"] else ""))
    return {"error": None, "accounts": out, "total_unread": total_unread,
            "total_accounts": len(out)}


# ---------------- CALENDAR ----------------
def _split_events(events: list, errmsg=None) -> dict:
    """Sort events and split into today vs upcoming, by the current local date."""
    events = sorted(events, key=lambda e: e.get("start", ""))
    today_str = datetime.now().strftime("%Y-%m-%d")
    return {
        "error": errmsg,
        "today": [e for e in events if e.get("start", "")[:10] == today_str],
        "upcoming": [e for e in events if e.get("start", "")[:10] > today_str][:40],
        "count": len(events),
    }


def collect_calendar(days: int = 14) -> dict:
    cfg = json.loads(CAL_CFG.read_text(encoding="utf-8")) if CAL_CFG.exists() else {}
    feeds = cfg.get("feeds", [])
    if not feeds:
        # No live feed: re-split the cached events by today's date so a seed
        # stays valid as days pass.
        if CAL_CACHE.exists():
            try:
                cached = json.loads(CAL_CACHE.read_text(encoding="utf-8"))
                res = _split_events(cached.get("events", []),
                                    "geen live feed - getoond uit cache")
                res["feeds"] = 0
                return res
            except Exception as e:
                return {"error": f"cache unreadable: {e}", "today": [], "upcoming": [],
                        "feeds": 0, "count": 0}
        return {"error": "no calendar feeds configured", "today": [], "upcoming": [],
                "feeds": 0, "count": 0}
    try:
        import recurring_ical_events
        from icalendar import Calendar
    except Exception as e:
        return {"error": f"ical libs missing ({e}) - run: pip install recurring-ical-events icalendar",
                "today": [], "upcoming": [], "feeds": len(feeds), "count": 0}

    now = datetime.now().astimezone()
    end = now + timedelta(days=days)
    events, errors = [], []
    for feed in feeds:
        try:
            r = httpx.get(feed["url"], timeout=25, follow_redirects=True)
            r.raise_for_status()
            cal = Calendar.from_ical(r.content)
            for ev in recurring_ical_events.of(cal).between(now, end):
                start = ev.get("DTSTART").dt
                dtend = ev.get("DTEND")
                endt = dtend.dt if dtend else start
                events.append({
                    "summary": str(ev.get("SUMMARY", "(no title)")),
                    "start": _iso(start),
                    "end": _iso(endt),
                    "location": str(ev.get("LOCATION", "")),
                    "calendar": feed.get("label", ""),
                    "kind": feed.get("kind", ""),
                    "all_day": not isinstance(start, datetime),
                })
        except Exception as e:
            errors.append(f"{feed.get('label', '?')}: {str(e)[:120]}")
    if events:
        try:
            CAL_CACHE.write_text(json.dumps(
                {"events": events, "saved_at": now.isoformat(timespec="seconds")},
                indent=2, ensure_ascii=False), encoding="utf-8")
        except Exception:
            pass
    res = _split_events(events, "; ".join(errors) if errors else None)
    res["feeds"] = len(feeds)
    return res


# ---------------- COMMAND / BUSINESS ----------------
def _fetch_basic(url: str, timeout: int = 8):
    try:
        creds = base64.b64encode(f"{A01_USER}:{A01_PASS}".encode()).decode()
        r = httpx.get(url, headers={"Authorization": f"Basic {creds}"},
                      timeout=timeout, follow_redirects=True)
        r.raise_for_status()
        return r.json()
    except Exception as e:
        return {"_error": str(e)[:160]}


def collect_command() -> dict:
    today = _fetch_basic(f"{ATLAS01}/today.json")
    biz = _fetch_basic(f"{ATLAS01}/atlas_data.json")
    online = not (isinstance(today, dict) and today.get("_error"))
    return {"today": today, "business": biz, "atlas01_online": online}


# ---------------- SERVICE HEALTH ----------------
LOCAL_SERVICES = [
    {"name": "Ollama",          "url": "http://localhost:11434/api/tags"},
    {"name": "Operations",      "url": "http://localhost:7799/"},
    {"name": "Atlas CFO",       "url": "http://localhost:7902/health"},
    {"name": "Learning Agent",  "url": "http://localhost:7901/health"},
    {"name": "Atlas Portal",    "url": "http://<TAILSCALE_IP>/portal.html"},
]

def collect_service_health() -> list:
    results = []
    for svc in LOCAL_SERVICES:
        try:
            r = httpx.get(svc["url"], timeout=4, follow_redirects=True)
            results.append({"name": svc["name"], "url": svc["url"],
                            "ok": r.status_code < 400, "status": r.status_code})
        except Exception as e:
            results.append({"name": svc["name"], "url": svc["url"],
                            "ok": False, "status": 0, "error": str(e)[:80]})
    return results


# ---------------- OLLAMA ----------------
def collect_ollama() -> dict:
    """Get Ollama model list and count."""
    try:
        r = httpx.get("http://localhost:11434/api/tags", timeout=5)
        if r.status_code != 200:
            return {"ok": False, "status": r.status_code}
        models = r.json().get("models", [])
        return {"ok": True, "model_count": len(models),
                "models": [m.get("name", "") for m in models]}
    except Exception as e:
        return {"ok": False, "error": str(e)[:120]}


# ---------------- SYNCTHING ----------------
def collect_syncthing() -> dict:
    """Check Syncthing sync health via local REST API."""
    import xml.etree.ElementTree as ET
    import subprocess
    key_path = HOME / "AppData" / "Local" / "Syncthing" / "config.xml"
    try:
        tree = ET.parse(key_path)
        api_key = (tree.find(".//apikey") or tree.find(".//{*}apikey"))
        api_key = api_key.text if api_key is not None else ""
    except Exception as e:
        return {"ok": False, "error": f"key_read: {e}"}
    if not api_key:
        return {"ok": False, "error": "apikey not found in config.xml"}
    try:
        r = httpx.get("http://localhost:8384/rest/system/status",
                      headers={"X-API-Key": api_key}, timeout=5)
        d = r.json()
        uptime_h = round(d.get("uptime", 0) / 3600, 1)
        # Get folder completion summary
        fr = httpx.get("http://localhost:8384/rest/db/completion",
                       headers={"X-API-Key": api_key}, timeout=5)
        completion = round(fr.json().get("completion", 0), 1) if fr.status_code == 200 else None
        return {"ok": True, "uptime_h": uptime_h,
                "completion_pct": completion, "myID_short": d.get("myID", "")[:8]}
    except Exception as e:
        return {"ok": False, "error": str(e)[:120]}


# ---------------- TAILSCALE ----------------
def collect_tailscale() -> dict:
    """Get Tailscale mesh node status via CLI."""
    import subprocess
    exe = r"C:\Program Files\Tailscale\tailscale.exe"
    try:
        r = subprocess.run([exe, "status", "--json"],
                           capture_output=True, text=True, timeout=8)
        if r.returncode != 0:
            return {"ok": False, "error": r.stderr[:120]}
        d = json.loads(r.stdout)
        peers = d.get("Peer", {})
        online = sum(1 for p in peers.values() if p.get("Online", False))
        my_ips = d.get("TailscaleIPs") or []
        my_ip = my_ips[0] if my_ips else "?"
        return {"ok": True, "nodes_total": len(peers) + 1,
                "nodes_online": online + 1, "peers_online": online,
                "my_ip": my_ip}
    except Exception as e:
        return {"ok": False, "error": str(e)[:120]}


# ---------------- SYNTHESIS ----------------
def synthesize(state: dict) -> dict:
    if model_router is None:
        return {"markdown": "_model-router niet beschikbaar_", "ok": False, "model": ""}
    mail = state.get("mail", {})
    cal = state.get("calendar", {})
    cmd = state.get("command", {})
    ctx = {
        "now": state["generated_at"],
        "mail_unread_total": mail.get("total_unread"),
        "mail_by_account": [
            {"account": a["label"], "unread": a["unread"],
             "recent_unread": [f"{r['from']} | {r['subject']}" for r in a["recent"] if r["unread"]][:4]}
            for a in mail.get("accounts", []) if not a.get("error")
        ],
        "today_events": [
            f"{e['start'][11:16]}-{e['end'][11:16]} {e['summary']}"
            + (f" @ {e['location']}" if e["location"] else "")
            for e in cal.get("today", [])
        ],
        "upcoming_events": [
            f"{e['start'][:16]} {e['summary']}" + (f" @ {e['location']}" if e["location"] else "")
            for e in cal.get("upcoming", [])[:18]
        ],
        "command_center": json.dumps(cmd.get("today", {}), default=str, ensure_ascii=False)[:1400],
        "business_crm": json.dumps(cmd.get("business", {}), default=str, ensure_ascii=False)[:1400],
        "odoo_summary": {
            "open_tasks": state.get("odoo", {}).get("open_tasks"),
            "overdue_invoices": state.get("odoo", {}).get("overdue_count"),
            "outstanding_eur": state.get("odoo", {}).get("total_residual"),
        },
        "infrastructure": {
            "atlas01_ok": state.get("atlas01", {}).get("ok"),
            "atlas01_ram_pct": state.get("atlas01", {}).get("used_ram_pct"),
            "atlas01_disk_free_gb": state.get("atlas01", {}).get("disk_free_gb"),
            "backup_hourly_age_h": state.get("backup", {}).get("hourly_age_h"),
            "backup_critical": state.get("backup", {}).get("critical"),
            "pos_fleet_active": state.get("fleet", {}).get("pos_active"),
            "dmarc_ok": state.get("dmarc", {}).get("ok"),
        },
    }
    system = (
        "You are Atlas Brain — the operational staff for Chaib, solo entrepreneur "
        "(Atlas Agency, Atlas POS, and collaborations). You receive his mailboxes, calendar, "
        "and command-center status. Write a SHARP briefing. Be concrete: name real senders, "
        "real appointments, real times. Spot calendar conflicts — can he be in two places at once? "
        "Flag explicitly what he risks forgetting: unread mail from important senders, appointments "
        "still needing confirmation. Give 3-5 concrete actions. English only. No inventions — "
        "only what is in the data."
    )
    user = (
        "Context (JSON):\n```json\n"
        + json.dumps(ctx, indent=1, default=str, ensure_ascii=False)[:7000]
        + "\n```\n\nRespond in EXACTLY this structure (markdown):\n"
        "## Headline\n<one sentence: the most important reality of today>\n\n"
        "## Today's agenda\n<the appointments that actually matter; name conflicts explicitly>\n\n"
        "## At risk of forgetting\n- ...\n- ...\n\n"
        "## Actions today\n1. ...\n2. ...\n3. ...\n\n"
        "## Mail that needs attention\n<1 to 3 lines: from whom, about what>\n\n"
        "## Spoken\n<a flowing spoken paragraph of max 80 words, "
        "complete sentences, no bullet points, no markdown>\n"
    )
    out = model_router.ask(user, task="chat", system=system, budget_s=240, num_predict=780)
    return {"markdown": (out.get("text") or "").strip() or "_lege synthese_",
            "ok": out.get("ok", False), "model": out.get("model", "")}


# ---------------- SPOKEN ----------------
def build_spoken(state: dict) -> dict:
    mail = state.get("mail", {})
    cal = state.get("calendar", {})
    md = state.get("brief", {}).get("markdown", "")
    now_hm = datetime.now().strftime("%H:%M")
    future = [e for e in cal.get("today", []) if e.get("start", "")[11:16] >= now_hm]
    nxt = future[0] if future else None
    unread = mail.get("total_unread", 0)

    spoken_para = _strip_md(_extract_section(md, "Spoken"))
    if spoken_para and len(spoken_para) > 25:
        full = spoken_para
    else:
        parts = [state.get("headline") or "Here is your briefing."]
        if nxt:
            parts.append(f"Your next appointment is {nxt['summary']} at {nxt['start'][11:16]}.")
        parts.append(f"You have {unread} unread messages across your mailboxes.")
        full = " ".join(parts)

    sp = [f"It is {now_hm}."]
    if nxt:
        sp.append(f"Next appointment: {nxt['summary']} at {nxt['start'][11:16]}.")
    else:
        sp.append("No more appointments today.")
    sp.append(f"{unread} unread emails.")
    return {"full": full, "short": " ".join(sp)}


# ---------------- ODOO ----------------
def _odoo_connect():
    """Return (models_proxy, uid, DB, PASS) or raise."""
    import xmlrpc.client
    ODOO = os.environ.get("ATLAS_ODOO_URL", "http://<TAILSCALE_IP>:8069")
    DB   = os.environ.get("ATLAS_ODOO_DB", "atlas")
    USER = os.environ.get("ATLAS_ODOO_USER", "chaib@atlascorporation.nl")
    PASS = os.environ.get("ATLAS_ODOO_PW", "")
    common = xmlrpc.client.ServerProxy(f"{ODOO}/xmlrpc/2/common")
    uid = common.authenticate(DB, USER, PASS, {})
    if not uid:
        raise RuntimeError("Odoo auth_failed")
    models = xmlrpc.client.ServerProxy(f"{ODOO}/xmlrpc/2/object")
    return models, uid, DB, PASS


def collect_odoo_overdue() -> dict:
    """Fetch overdue invoices: posted, unpaid/partial, past invoice_date_due."""
    try:
        models, uid, DB, PASS = _odoo_connect()
        today_str = datetime.now().strftime("%Y-%m-%d")
        domain = [
            ["move_type", "in", ["out_invoice"]],
            ["state", "=", "posted"],
            ["payment_state", "in", ["not_paid", "partial"]],
            ["invoice_date_due", "<", today_str],
        ]
        inv_ids = models.execute_kw(DB, uid, PASS, "account.move", "search",
                                    [domain], {"limit": 50})
        if not inv_ids:
            return {"ok": True, "count": 0, "total_residual": 0.0, "invoices": []}
        invoices = models.execute_kw(DB, uid, PASS, "account.move", "read", [inv_ids],
            {"fields": ["name", "partner_id", "amount_residual",
                        "amount_total", "invoice_date_due", "payment_state"]})
        result = []
        total_residual = 0.0
        for i in invoices:
            due_str = i.get("invoice_date_due") or today_str
            try:
                due_dt = datetime.strptime(due_str, "%Y-%m-%d")
                days_overdue = (datetime.now() - due_dt).days
            except Exception:
                days_overdue = 0
            residual = float(i.get("amount_residual") or i.get("amount_total") or 0)
            total_residual += residual
            result.append({
                "name": i["name"],
                "partner": (i.get("partner_id") or [None, "?"])[1],
                "amount_residual": round(residual, 2),
                "amount_total": round(float(i.get("amount_total") or 0), 2),
                "due": due_str,
                "days_overdue": days_overdue,
                "payment_state": i.get("payment_state", ""),
            })
        result.sort(key=lambda x: x["days_overdue"], reverse=True)
        return {
            "ok": True,
            "count": len(result),
            "total_residual": round(total_residual, 2),
            "invoices": result,
        }
    except Exception as e:
        return {"ok": False, "error": str(e)[:200]}


def collect_odoo() -> dict:
    """Pull task + invoice counts from self-hosted Odoo on atlas-01."""
    try:
        models, uid, DB, PASS = _odoo_connect()
        # Open tasks
        task_ids = models.execute_kw(DB, uid, PASS, "project.task", "search",
            [[["stage_id.fold", "=", False]]], {"limit": 100})
        tasks_count = len(task_ids)
        # All unpaid invoices (posted)
        today_str = datetime.now().strftime("%Y-%m-%d")
        inv_ids = models.execute_kw(DB, uid, PASS, "account.move", "search",
            [[["move_type", "in", ["out_invoice"]], ["state", "=", "posted"],
              ["payment_state", "in", ["not_paid", "partial"]]]],
            {"limit": 50})
        invoices = models.execute_kw(DB, uid, PASS, "account.move", "read", [inv_ids],
            {"fields": ["name", "partner_id", "amount_total", "amount_residual",
                        "invoice_date_due", "payment_state"]})
        overdue = [i for i in invoices if i.get("invoice_date_due") and
                   i["invoice_date_due"] < today_str]
        # Detailed overdue block (reuse collect_odoo_overdue logic inline)
        overdue_detail = []
        total_residual = 0.0
        for i in overdue:
            due_str = i.get("invoice_date_due") or today_str
            try:
                due_dt = datetime.strptime(due_str, "%Y-%m-%d")
                days_overdue = (datetime.now() - due_dt).days
            except Exception:
                days_overdue = 0
            residual = float(i.get("amount_residual") or i.get("amount_total") or 0)
            total_residual += residual
            overdue_detail.append({
                "name": i["name"],
                "partner": (i.get("partner_id") or [None, "?"])[1],
                "amount_residual": round(residual, 2),
                "amount_total": round(float(i.get("amount_total") or 0), 2),
                "due": due_str,
                "days_overdue": days_overdue,
                "payment_state": i.get("payment_state", ""),
            })
        overdue_detail.sort(key=lambda x: x["days_overdue"], reverse=True)
        return {
            "ok": True,
            "open_tasks": tasks_count,
            "unpaid_invoices": len(invoices),
            "overdue_count": len(overdue),
            "total_outstanding": round(sum(float(i.get("amount_total") or 0) for i in invoices), 2),
            "total_residual": round(total_residual, 2),
            "invoices": [{"name": i["name"], "partner": (i.get("partner_id") or [None, "?"])[1],
                          "amount": float(i.get("amount_total") or 0),
                          "residual": float(i.get("amount_residual") or 0),
                          "due": i.get("invoice_date_due"),
                          "state": i.get("payment_state")} for i in invoices[:8]],
            "overdue_invoices": overdue_detail,
        }
    except Exception as e:
        return {"ok": False, "error": str(e)[:120]}


# ---------------- DISK USAGE ----------------
def collect_disk() -> dict:
    import shutil
    drives = {}
    for drive in ["C:\\", "D:\\"]:
        try:
            u = shutil.disk_usage(drive)
            drives[drive[0]] = {
                "total_gb": round(u.total / 1024**3, 1),
                "used_gb": round(u.used / 1024**3, 1),
                "free_gb": round(u.free / 1024**3, 1),
                "used_pct": round(u.used / u.total * 100, 1),
                "critical": (u.free / 1024**3) < 20 or (u.used / u.total) > 0.95,
                "warn": (u.free / 1024**3) < 50 or (u.used / u.total) > 0.85,
            }
        except Exception:
            pass
    c = drives.get("C", {})
    return {
        "drives": drives,
        "free_gb": c.get("free_gb"),
        "used_pct": c.get("used_pct"),
        "critical": c.get("critical", False),
        "warn": c.get("warn", False),
    }


# ---------------- SCHEDULED TASKS ----------------
def collect_scheduled_tasks() -> dict:
    """Count Windows scheduled tasks via schtasks."""
    import subprocess
    try:
        r = subprocess.run(
            ["schtasks", "/query", "/fo", "CSV", "/nh"],
            capture_output=True, text=True, timeout=15
        )
        lines = [l for l in r.stdout.splitlines() if l.strip() and not l.startswith('"TaskName"')]
        atlas_lines = [l for l in lines if 'atlas' in l.lower() or 'jarvis' in l.lower()]
        return {"ok": True, "total": len(lines), "atlas_count": len(atlas_lines)}
    except Exception as e:
        return {"ok": False, "error": str(e)[:120]}


# ---------------- VAULT STATS ----------------
def collect_vault_stats() -> dict:
    """Count vault notes + size from the MEGA Obsidian vault."""
    import time as _time
    vault = HOME / "MEGA" / "Atlas Corporation"
    if not vault.exists():
        return {"ok": False, "error": "vault not found"}
    try:
        count = 0
        total_size = 0
        newest_mtime = 0.0
        newest_name = None
        for f in vault.rglob("*.md"):
            try:
                st = f.stat()
                count += 1
                total_size += st.st_size
                if st.st_mtime > newest_mtime:
                    newest_mtime = st.st_mtime
                    newest_name = f.name
            except Exception:
                pass
        newest_age_h = round((_time.time() - newest_mtime) / 3600, 1) if newest_mtime else None
        return {
            "ok": True,
            "note_count": count,
            "size_mb": round(total_size / 1024 / 1024, 1),
            "newest_note": newest_name,
            "newest_age_h": newest_age_h,
        }
    except Exception as e:
        return {"ok": False, "error": str(e)[:100]}


# ---------------- BACKUP STATUS ----------------
def collect_backup_status() -> dict:
    """Check Odoo backup freshness from the MEGA-synced backup folder."""
    import time as _time
    BACKUP_ROOT = HOME / "MEGA" / "Atlas Corporation" / "07_Finance" / "_atlas-odoo-backups"
    result: dict = {"ok": True, "hourly_age_h": None, "daily_age_h": None, "warn": False, "critical": False}
    for sub in ("hourly", "daily"):
        d = BACKUP_ROOT / sub
        if not d.exists():
            result[f"{sub}_age_h"] = None
            continue
        files = sorted(d.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True) if d.exists() else []
        if files:
            age_h = round((_time.time() - files[0].stat().st_mtime) / 3600, 1)
            result[f"{sub}_age_h"] = age_h
            result[f"{sub}_name"] = files[0].name
        else:
            result[f"{sub}_age_h"] = None
    hourly = result.get("hourly_age_h")
    daily = result.get("daily_age_h")
    if hourly is None or hourly > 3:
        result["warn"] = True
    if hourly is None or hourly > 12:
        result["critical"] = True
    if daily is None or daily > 48:
        result["warn"] = True
    return result


# ---------------- ATLAS-01 RESOURCES ----------------
def collect_atlas01_resources() -> dict:
    """SSH to atlas-01 to get free RAM, CPU load, uptime, and container count."""
    import subprocess
    cmd = [
        "ssh", "-i", str(HOME / ".ssh" / "id_ed25519_atlas"),
        "-o", "BatchMode=yes", "-o", "ConnectTimeout=10",
        "-o", "StrictHostKeyChecking=accept-new",
        "root@<TAILSCALE_IP>",
        "python3 -c \""
        "import os,subprocess;"
        "mem=open('/proc/meminfo').read();"
        "free_kb=int([l for l in mem.splitlines() if 'MemAvailable' in l][0].split()[1]);"
        "total_kb=int([l for l in mem.splitlines() if 'MemTotal' in l][0].split()[1]);"
        "load=open('/proc/loadavg').read().split()[:3];"
        "uptm=float(open('/proc/uptime').read().split()[0]);"
        "ctrs=len([l for l in subprocess.run(['docker','ps','-q'],capture_output=True,text=True).stdout.strip().splitlines() if l]);"
        "df_o=subprocess.run(['df','-BG','/'],capture_output=True,text=True).stdout.splitlines();"
        "dp=df_o[1].split() if len(df_o)>1 else ['?','0G','0G','0G'];"
        "d_tot=int(dp[1].rstrip('G'));d_free=int(dp[3].rstrip('G'));"
        "print(f'{free_kb},{total_kb},{load[0]},{load[1]},{load[2]},{int(uptm//3600)},{ctrs},{d_tot},{d_free}')\""
    ]
    try:
        r = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
        if r.returncode != 0:
            return {"ok": False, "error": r.stderr.strip()[:100]}
        parts = r.stdout.strip().split(",")
        if len(parts) < 9:
            return {"ok": False, "error": "unexpected output"}
        free_kb, total_kb = int(parts[0]), int(parts[1])
        load1, load5, load15 = float(parts[2]), float(parts[3]), float(parts[4])
        uptime_h = int(parts[5])
        containers = int(parts[6])
        disk_total_gb, disk_free_gb = int(parts[7]), int(parts[8])
        disk_used_pct = round((disk_total_gb - disk_free_gb) / disk_total_gb * 100, 1) if disk_total_gb > 0 else 0
        free_gb = round(free_kb / 1024 / 1024, 2)
        used_pct = round((total_kb - free_kb) / total_kb * 100, 1)
        return {
            "ok": True,
            "free_ram_gb": free_gb,
            "total_ram_gb": round(total_kb / 1024 / 1024, 2),
            "used_ram_pct": used_pct,
            "load_1m": load1,
            "load_5m": load5,
            "load_15m": load15,
            "uptime_h": uptime_h,
            "containers": containers,
            "disk_total_gb": disk_total_gb,
            "disk_free_gb": disk_free_gb,
            "disk_used_pct": disk_used_pct,
            "disk_critical": disk_free_gb < 10,
            "warn": used_pct > 85 or load1 > 4.0 or disk_free_gb < 20,
        }
    except Exception as e:
        return {"ok": False, "error": str(e)[:120]}


# ---------------- AUCTION INTEL ----------------
def collect_auction_intel() -> dict:
    """Pull live stats from the auction intel dashboard API (localhost:7781)."""
    import sqlite3
    intel_db = HOME / "twa-scrape" / "intel.db"
    if not intel_db.exists():
        return {"ok": False, "reason": "db_not_found"}
    try:
        conn = sqlite3.connect(str(intel_db))
        conn.row_factory = sqlite3.Row
        active  = conn.execute("SELECT COUNT(*) FROM lots WHERE status='active'").fetchone()[0]
        strong  = conn.execute("SELECT COUNT(*) FROM lots WHERE status='active' AND stars>=4").fetchone()[0]
        avg_m   = conn.execute("SELECT AVG(margin_pct) FROM lots WHERE status='active' AND margin_pct IS NOT NULL").fetchone()[0]
        try:
            urgent = conn.execute("SELECT COUNT(*) FROM lots WHERE status='active' AND urgency=1").fetchone()[0]
        except Exception:
            urgent = 0
        top5    = [dict(r) for r in conn.execute(
            "SELECT id, title, model_key, hammer, margin_pct, stars, close_time "
            "FROM lots WHERE status='active' AND stars>=4 "
            "ORDER BY stars DESC, margin_pct DESC LIMIT 5"
        ).fetchall()]
        try:
            last_run = dict(conn.execute(
                "SELECT started_at, run_type, lots_found FROM run_log ORDER BY id DESC LIMIT 1"
            ).fetchone() or {})
        except Exception:
            last_run = {}
        conn.close()
        return {
            "ok": True,
            "lots_active": active,
            "strong_buys": strong,
            "urgent_count": urgent,
            "avg_margin": round(avg_m or 0, 1),
            "top_lots": top5,
            "last_run": last_run,
            "dashboard_url": "http://localhost:7781/",
        }
    except Exception as e:
        return {"ok": False, "reason": str(e)}


# ---------------- POS FLEET ----------------
def collect_pos_fleet() -> dict:
    """Read POS fleet config — static count from pos_targets.json."""
    pts = ATLAS / "pos_targets.json"
    if not pts.exists():
        return {"ok": False, "reason": "pos_targets.json not found"}
    try:
        cfg = json.loads(pts.read_text(encoding="utf-8"))
        pos = cfg.get("pos_agents", [])
        internal = cfg.get("atlas_internal", [])
        active = [p for p in pos if not p.get("skip_autopilot")]
        skipped = [p for p in pos if p.get("skip_autopilot")]
        return {
            "ok": True,
            "pos_count": len(pos),
            "pos_active": len(active),
            "pos_skipped": len(skipped),
            "internal_count": len(internal),
            "skip_reasons": [{"name": p["dws_name"], "reason": p.get("skip_reason", "?")[:60]}
                             for p in skipped],
        }
    except Exception as e:
        return {"ok": False, "error": str(e)[:120]}


# ---------------- DMARC CHECK ----------------
def collect_dmarc() -> dict:
    """DNS check for DMARC policy on Atlas domains. p=none = not enforced."""
    import socket
    domains = ["atlascorporation.nl", "atlasworks.nl", "atlasworkspace.eu", "atlasoperations.nl"]
    results = {}
    for d in domains:
        try:
            txt = socket.getaddrinfo(f"_dmarc.{d}", None)
        except Exception:
            txt = None
        try:
            import subprocess
            r = subprocess.run(
                ["nslookup", "-type=TXT", f"_dmarc.{d}", "8.8.8.8"],
                capture_output=True, text=True, timeout=8
            )
            raw = r.stdout
            if "v=DMARC1" in raw:
                policy = "quarantine" if "p=quarantine" in raw else (
                    "reject" if "p=reject" in raw else "none")
                results[d] = {"ok": policy != "none", "policy": policy}
            else:
                results[d] = {"ok": False, "policy": "missing"}
        except Exception as e:
            results[d] = {"ok": None, "error": str(e)[:80]}
    enforced = [d for d, v in results.items() if v.get("ok")]
    missing  = [d for d, v in results.items() if not v.get("ok")]
    return {"ok": len(missing) == 0, "domains": results,
            "enforced": enforced, "missing": missing}


# ---------------- MAIN ----------------
def main() -> None:
    quick = "--quick" in sys.argv
    send_report = "--report" in sys.argv
    t0 = time.time()
    state = {
        "generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
        "generated_unix": int(time.time()),
        "operator": "Chaib",
        "mode": "quick" if quick else "full",
    }
    state["mail"] = _safe(collect_mail, "mail")
    state["calendar"] = _safe(collect_calendar, "calendar")
    state["command"] = _safe(collect_command, "command")
    state["service_health"] = _safe(collect_service_health, "service_health")
    state["syncthing"] = _safe(collect_syncthing, "syncthing")
    state["tailscale"] = _safe(collect_tailscale, "tailscale")
    state["ollama"] = _safe(collect_ollama, "ollama")
    state["scheduled_tasks"] = _safe(collect_scheduled_tasks, "scheduled_tasks")
    state["auction_intel"] = _safe(collect_auction_intel, "auction_intel")
    state["disk"] = _safe(collect_disk, "disk")
    state["atlas01"] = _safe(collect_atlas01_resources, "atlas01")
    state["backup"] = _safe(collect_backup_status, "backup")
    state["fleet"] = _safe(collect_pos_fleet, "fleet")
    state["dmarc"] = _safe(collect_dmarc, "dmarc")
    # Slow collectors — only on full runs; quick ticks carry forward the last state
    if not quick:
        state["vault"] = _safe(collect_vault_stats, "vault")  # 5s rglob over 10k files
        state["odoo"] = _safe(collect_odoo, "odoo")
        # Store overdue invoices as its own top-level key for check_overdue.py / portal
        odoo_data = state["odoo"]
        if odoo_data.get("ok"):
            state["overdue_invoices"] = {
                "ok": True,
                "count": odoo_data.get("overdue_count", 0),
                "total_residual": odoo_data.get("total_residual", 0.0),
                "invoices": odoo_data.get("overdue_invoices", []),
                "updated_at": state["generated_at"],
            }
        else:
            state["overdue_invoices"] = {"ok": False, "error": odoo_data.get("error", "odoo_error"),
                                         "updated_at": state["generated_at"]}
    else:
        try:
            prev = json.loads(STATE_JSON.read_text(encoding="utf-8")) if STATE_JSON.exists() else {}
            state["odoo"] = prev.get("odoo", {"ok": False, "error": "quick_tick_no_refresh"})
            state["overdue_invoices"] = prev.get("overdue_invoices",
                                                 {"ok": False, "error": "quick_tick_no_refresh"})
            state["vault"] = prev.get("vault", {"ok": False, "error": "quick_tick_no_refresh"})
        except Exception:
            state["odoo"] = {"ok": False, "error": "quick_tick_no_refresh"}
            state["overdue_invoices"] = {"ok": False, "error": "quick_tick_no_refresh"}
            state["vault"] = {"ok": False, "error": "quick_tick_no_refresh"}

    if quick:
        state["brief"] = {"markdown": f"_quick tick {state['generated_at'][11:16]} — no synthesis_",
                          "ok": False, "model": ""}
    else:
        state["brief"] = _safe(lambda: synthesize(state), "synth")

    state["headline"] = (_extract_section(state["brief"].get("markdown", ""), "Headline").splitlines()
                         or _extract_section(state["brief"].get("markdown", ""), "Kop").splitlines()
                         or ["Atlas briefing."])[0].strip().lstrip("#").strip() or "Atlas briefing."
    state["spoken"] = build_spoken(state)
    state["services"] = SERVICES  # static link list for portal
    state["build_seconds"] = round(time.time() - t0, 1)

    state_bytes = json.dumps(state, indent=2, default=str, ensure_ascii=False)
    STATE_JSON.write_text(state_bytes, encoding="utf-8")
    # Mirror to state/ sub-dir (served by the 7799 HTTP server)
    (OPS_DIR / "state" / "atlas-state.json").write_text(state_bytes, encoding="utf-8")
    SPOKEN_JSON.write_text(json.dumps(state["spoken"], indent=2, ensure_ascii=False), encoding="utf-8")
    try:
        cmd_dir = HOME / "MEGA" / "Atlas Corporation" / "vault" / "_command"
        if cmd_dir.exists():
            (cmd_dir / "jarvis-brief.md").write_text(state["brief"].get("markdown", ""), encoding="utf-8")
    except Exception:
        pass

    log(f"state written -> {STATE_JSON}  ({state['build_seconds']}s, "
        f"mail_unread={state['mail'].get('total_unread')}, "
        f"today_events={len(state['calendar'].get('today', []))})")

    if send_report and not quick:
        try:
            import subprocess
            py = sys.executable
            rpt = Path(__file__).parent / "atlas_morning_report.py"
            subprocess.Popen([py, str(rpt)], creationflags=0x08000000)
            log("Morning report spawned")
        except Exception as e:
            log(f"Morning report failed: {e}")

    print(json.dumps({
        "ok": True, "mode": state["mode"], "state_file": str(STATE_JSON),
        "mail_unread": state["mail"].get("total_unread"),
        "today_events": len(state["calendar"].get("today", [])),
        "headline": state["headline"], "seconds": state["build_seconds"],
    }, indent=2, ensure_ascii=False))


if __name__ == "__main__":
    try:
        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    except Exception:
        pass
    main()