8c1efe6e30
- assets/header.svg: animated SVG header (twinkling stars, orbiting service dots, Atlas A logo with glow, float/pulse animations, corner accents) - README.md: animated banner, full 39-app service catalog, Mermaid architecture, AI stack table, tech stack, quick-deploy, repo structure, security model - ARCHITECTURE.md: one-box philosophy, 5-tier service layers, data flow diagram, Caddy SSO pattern, AI mesh topology, Docker Compose topology - SERVICES.md: all 55 containers + 20 systemd services documented - CHANGELOG.md: full build history from v0.5 to v2.0 - assets/logo.svg: Atlas A logo with gradient - deploy/install.sh: one-command installer (Docker + Caddy + Tailscale + stacks) - deploy/compose/atlas-core.yml: Authentik + Nextcloud + Stalwart + Forgejo + Vaultwarden - deploy/caddy/Caddyfile.template: Authentik SSO pattern, sanitized - deploy/README.md: full deployment guide - server/: sanitized atlas_brain.py, cockpit_shim.py, intelligence.py, meridian.py, atlas_router_api.py, cli-atlas.py (all secrets stripped, env var patterns) - server/systemd/: 6 atlas-* service unit files - server/README.md: deployment and configuration guide Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
938 lines
39 KiB
Python
938 lines
39 KiB
Python
"""Atlas Brain - the collector behind Atlas Operations.
|
|
|
|
Pulls every signal into one atlas-state.json:
|
|
mail : 7 IMAP mailboxes (unread + recent headers), strictly read-only
|
|
calendar : Google secret-iCal feeds, recurrences expanded, next 14 days
|
|
command : atlas-01 today.json (attention / mesh)
|
|
business : atlas-01 atlas_data.json (Odoo CRM)
|
|
Then synthesizes a proactive brief via the local model-router.
|
|
|
|
Supersedes jarvis-brain-hourly.py / jarvis-brain-hourly-v2.py.
|
|
|
|
Run:
|
|
python atlas_brain.py full collect + synth + write state
|
|
python atlas_brain.py --quick collect only, skip the LLM synth (fast 15-min tick)
|
|
"""
|
|
from __future__ import annotations
|
|
import base64
|
|
import email
|
|
import imaplib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from datetime import datetime, timezone, timedelta
|
|
from email.header import decode_header, make_header
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
|
|
HOME = Path.home()
|
|
ATLAS = HOME / ".config" / "atlas"
|
|
OPS_DIR = ATLAS / "operations"
|
|
CACHE = ATLAS / "cache"
|
|
OPS_DIR.mkdir(parents=True, exist_ok=True)
|
|
CACHE.mkdir(parents=True, exist_ok=True)
|
|
|
|
STATE_JSON = OPS_DIR / "atlas-state.json"
|
|
SPOKEN_JSON = ATLAS / "atlas-brief-spoken.json"
|
|
MAIL_CFG = ATLAS / "mail-accounts.json"
|
|
CAL_CFG = ATLAS / "calendar-feeds.json"
|
|
CAL_CACHE = CACHE / "calendar-last.json"
|
|
LOG = ATLAS / "atlas-brain.log"
|
|
|
|
ATLAS01 = os.environ.get("ATLAS01_URL", "http://atlas-01.tailab40ed.ts.net")
|
|
A01_USER = "atlas"
|
|
A01_PASS = os.environ.get("ATLAS_BASIC_PASS", "")
|
|
|
|
SERVICES = [
|
|
{"name": "Command Center", "url": "https://atlascorporation.nl/command"},
|
|
{"name": "Nextcloud", "url": "http://atlas-01:8443"},
|
|
{"name": "MeshCentral", "url": "https://mesh.atlascorporation.nl/"},
|
|
{"name": "Odoo", "url": "http://<TAILSCALE_IP>:8069"},
|
|
{"name": "Atlas Portal", "url": "http://atlas-01/portal.html"},
|
|
{"name": "n8n", "url": "http://<TAILSCALE_IP>:5678"},
|
|
{"name": "Stalwart", "url": "http://<TAILSCALE_IP>:8080"},
|
|
{"name": "Authentik", "url": "http://<TAILSCALE_IP>:9000"},
|
|
{"name": "Ollama", "url": "http://localhost:11434"},
|
|
]
|
|
|
|
sys.path.insert(0, str(ATLAS))
|
|
try:
|
|
import model_router
|
|
except Exception:
|
|
model_router = None
|
|
|
|
|
|
def log(msg: str) -> None:
|
|
try:
|
|
with LOG.open("a", encoding="utf-8") as f:
|
|
f.write(f"[{datetime.now().isoformat(timespec='seconds')}] {msg}\n")
|
|
except Exception:
|
|
pass
|
|
print(f"[brain] {msg}")
|
|
|
|
|
|
def _safe(fn, name: str):
|
|
try:
|
|
return fn()
|
|
except Exception as e:
|
|
log(f"{name} FAILED: {e}")
|
|
log(traceback.format_exc())
|
|
return {"error": f"{name}: {e}"}
|
|
|
|
|
|
def _decode(s) -> str:
|
|
if not s:
|
|
return ""
|
|
try:
|
|
return str(make_header(decode_header(s)))
|
|
except Exception:
|
|
return str(s)
|
|
|
|
|
|
def _iso(d) -> str:
|
|
if isinstance(d, datetime):
|
|
if d.tzinfo is None:
|
|
d = d.replace(tzinfo=timezone.utc)
|
|
return d.astimezone().isoformat(timespec="minutes")
|
|
return str(d)
|
|
|
|
|
|
def _strip_md(text: str) -> str:
|
|
t = re.sub(r"```.*?```", "", text or "", flags=re.S)
|
|
t = re.sub(r"[`*_#>]", "", t)
|
|
t = re.sub(r"^\s*[-+]\s+", "", t, flags=re.M)
|
|
t = re.sub(r"^\s*\d+\.\s+", "", t, flags=re.M)
|
|
t = re.sub(r"\n{2,}", ". ", t)
|
|
t = re.sub(r"\n", " ", t)
|
|
t = re.sub(r"\s{2,}", " ", t)
|
|
return t.strip()
|
|
|
|
|
|
def _extract_section(md: str, name: str) -> str:
|
|
m = re.search(rf"##\s*{re.escape(name)}\s*\n(.*?)(?:\n##\s|\Z)", md or "", re.S)
|
|
return m.group(1).strip() if m else ""
|
|
|
|
|
|
# ---------------- MAIL ----------------
|
|
def collect_mail() -> dict:
|
|
if not MAIL_CFG.exists():
|
|
return {"error": "mail-accounts.json not found", "accounts": [], "total_unread": 0}
|
|
accounts = json.loads(MAIL_CFG.read_text(encoding="utf-8"))
|
|
out, total_unread = [], 0
|
|
for acc in accounts:
|
|
rec = {"label": acc["label"], "group": acc.get("group", ""),
|
|
"total": 0, "unread": 0, "recent": [], "error": None}
|
|
_last_err = None
|
|
for _attempt in range(2):
|
|
try:
|
|
M = imaplib.IMAP4_SSL(acc["host"], acc.get("port", 993), timeout=25)
|
|
M.login(acc["user"], acc["password"])
|
|
M.select("INBOX", readonly=True)
|
|
ids = M.search(None, "ALL")[1][0].split()
|
|
uids = M.search(None, "UNSEEN")[1][0].split()
|
|
rec["total"] = len(ids)
|
|
rec["unread"] = len(uids)
|
|
total_unread += len(uids)
|
|
for mid in reversed((uids or ids)[-6:]):
|
|
try:
|
|
md = M.fetch(mid.decode(), "(BODY.PEEK[HEADER.FIELDS (SUBJECT FROM DATE)])")[1]
|
|
raw = next((p[1] for p in md if isinstance(p, tuple)), b"")
|
|
msg = email.message_from_bytes(raw)
|
|
rec["recent"].append({
|
|
"subject": _decode(msg.get("Subject"))[:160],
|
|
"from": _decode(msg.get("From"))[:120],
|
|
"date": (msg.get("Date") or "")[:40],
|
|
"unread": mid in uids,
|
|
})
|
|
except Exception as fe:
|
|
log(f" fetch glitch {acc['label']}: {fe}")
|
|
M.logout()
|
|
_last_err = None
|
|
break # success — exit retry loop
|
|
except Exception as e:
|
|
_last_err = e
|
|
if _attempt == 0:
|
|
log(f" mail retry {acc['label']}: {e}")
|
|
time.sleep(1)
|
|
if _last_err is not None:
|
|
rec["error"] = str(_last_err)[:200]
|
|
out.append(rec)
|
|
log(f"mail {acc['label']}: {rec['unread']} unread / {rec['total']}"
|
|
+ (f" ERR {rec['error']}" if rec["error"] else ""))
|
|
return {"error": None, "accounts": out, "total_unread": total_unread,
|
|
"total_accounts": len(out)}
|
|
|
|
|
|
# ---------------- CALENDAR ----------------
|
|
def _split_events(events: list, errmsg=None) -> dict:
|
|
"""Sort events and split into today vs upcoming, by the current local date."""
|
|
events = sorted(events, key=lambda e: e.get("start", ""))
|
|
today_str = datetime.now().strftime("%Y-%m-%d")
|
|
return {
|
|
"error": errmsg,
|
|
"today": [e for e in events if e.get("start", "")[:10] == today_str],
|
|
"upcoming": [e for e in events if e.get("start", "")[:10] > today_str][:40],
|
|
"count": len(events),
|
|
}
|
|
|
|
|
|
def collect_calendar(days: int = 14) -> dict:
|
|
cfg = json.loads(CAL_CFG.read_text(encoding="utf-8")) if CAL_CFG.exists() else {}
|
|
feeds = cfg.get("feeds", [])
|
|
if not feeds:
|
|
# No live feed: re-split the cached events by today's date so a seed
|
|
# stays valid as days pass.
|
|
if CAL_CACHE.exists():
|
|
try:
|
|
cached = json.loads(CAL_CACHE.read_text(encoding="utf-8"))
|
|
res = _split_events(cached.get("events", []),
|
|
"geen live feed - getoond uit cache")
|
|
res["feeds"] = 0
|
|
return res
|
|
except Exception as e:
|
|
return {"error": f"cache unreadable: {e}", "today": [], "upcoming": [],
|
|
"feeds": 0, "count": 0}
|
|
return {"error": "no calendar feeds configured", "today": [], "upcoming": [],
|
|
"feeds": 0, "count": 0}
|
|
try:
|
|
import recurring_ical_events
|
|
from icalendar import Calendar
|
|
except Exception as e:
|
|
return {"error": f"ical libs missing ({e}) - run: pip install recurring-ical-events icalendar",
|
|
"today": [], "upcoming": [], "feeds": len(feeds), "count": 0}
|
|
|
|
now = datetime.now().astimezone()
|
|
end = now + timedelta(days=days)
|
|
events, errors = [], []
|
|
for feed in feeds:
|
|
try:
|
|
r = httpx.get(feed["url"], timeout=25, follow_redirects=True)
|
|
r.raise_for_status()
|
|
cal = Calendar.from_ical(r.content)
|
|
for ev in recurring_ical_events.of(cal).between(now, end):
|
|
start = ev.get("DTSTART").dt
|
|
dtend = ev.get("DTEND")
|
|
endt = dtend.dt if dtend else start
|
|
events.append({
|
|
"summary": str(ev.get("SUMMARY", "(no title)")),
|
|
"start": _iso(start),
|
|
"end": _iso(endt),
|
|
"location": str(ev.get("LOCATION", "")),
|
|
"calendar": feed.get("label", ""),
|
|
"kind": feed.get("kind", ""),
|
|
"all_day": not isinstance(start, datetime),
|
|
})
|
|
except Exception as e:
|
|
errors.append(f"{feed.get('label', '?')}: {str(e)[:120]}")
|
|
if events:
|
|
try:
|
|
CAL_CACHE.write_text(json.dumps(
|
|
{"events": events, "saved_at": now.isoformat(timespec="seconds")},
|
|
indent=2, ensure_ascii=False), encoding="utf-8")
|
|
except Exception:
|
|
pass
|
|
res = _split_events(events, "; ".join(errors) if errors else None)
|
|
res["feeds"] = len(feeds)
|
|
return res
|
|
|
|
|
|
# ---------------- COMMAND / BUSINESS ----------------
|
|
def _fetch_basic(url: str, timeout: int = 8):
|
|
try:
|
|
creds = base64.b64encode(f"{A01_USER}:{A01_PASS}".encode()).decode()
|
|
r = httpx.get(url, headers={"Authorization": f"Basic {creds}"},
|
|
timeout=timeout, follow_redirects=True)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
except Exception as e:
|
|
return {"_error": str(e)[:160]}
|
|
|
|
|
|
def collect_command() -> dict:
|
|
today = _fetch_basic(f"{ATLAS01}/today.json")
|
|
biz = _fetch_basic(f"{ATLAS01}/atlas_data.json")
|
|
online = not (isinstance(today, dict) and today.get("_error"))
|
|
return {"today": today, "business": biz, "atlas01_online": online}
|
|
|
|
|
|
# ---------------- SERVICE HEALTH ----------------
|
|
LOCAL_SERVICES = [
|
|
{"name": "Ollama", "url": "http://localhost:11434/api/tags"},
|
|
{"name": "Operations", "url": "http://localhost:7799/"},
|
|
{"name": "Atlas CFO", "url": "http://localhost:7902/health"},
|
|
{"name": "Learning Agent", "url": "http://localhost:7901/health"},
|
|
{"name": "Atlas Portal", "url": "http://<TAILSCALE_IP>/portal.html"},
|
|
]
|
|
|
|
def collect_service_health() -> list:
|
|
results = []
|
|
for svc in LOCAL_SERVICES:
|
|
try:
|
|
r = httpx.get(svc["url"], timeout=4, follow_redirects=True)
|
|
results.append({"name": svc["name"], "url": svc["url"],
|
|
"ok": r.status_code < 400, "status": r.status_code})
|
|
except Exception as e:
|
|
results.append({"name": svc["name"], "url": svc["url"],
|
|
"ok": False, "status": 0, "error": str(e)[:80]})
|
|
return results
|
|
|
|
|
|
# ---------------- OLLAMA ----------------
|
|
def collect_ollama() -> dict:
|
|
"""Get Ollama model list and count."""
|
|
try:
|
|
r = httpx.get("http://localhost:11434/api/tags", timeout=5)
|
|
if r.status_code != 200:
|
|
return {"ok": False, "status": r.status_code}
|
|
models = r.json().get("models", [])
|
|
return {"ok": True, "model_count": len(models),
|
|
"models": [m.get("name", "") for m in models]}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:120]}
|
|
|
|
|
|
# ---------------- SYNCTHING ----------------
|
|
def collect_syncthing() -> dict:
|
|
"""Check Syncthing sync health via local REST API."""
|
|
import xml.etree.ElementTree as ET
|
|
import subprocess
|
|
key_path = HOME / "AppData" / "Local" / "Syncthing" / "config.xml"
|
|
try:
|
|
tree = ET.parse(key_path)
|
|
api_key = (tree.find(".//apikey") or tree.find(".//{*}apikey"))
|
|
api_key = api_key.text if api_key is not None else ""
|
|
except Exception as e:
|
|
return {"ok": False, "error": f"key_read: {e}"}
|
|
if not api_key:
|
|
return {"ok": False, "error": "apikey not found in config.xml"}
|
|
try:
|
|
r = httpx.get("http://localhost:8384/rest/system/status",
|
|
headers={"X-API-Key": api_key}, timeout=5)
|
|
d = r.json()
|
|
uptime_h = round(d.get("uptime", 0) / 3600, 1)
|
|
# Get folder completion summary
|
|
fr = httpx.get("http://localhost:8384/rest/db/completion",
|
|
headers={"X-API-Key": api_key}, timeout=5)
|
|
completion = round(fr.json().get("completion", 0), 1) if fr.status_code == 200 else None
|
|
return {"ok": True, "uptime_h": uptime_h,
|
|
"completion_pct": completion, "myID_short": d.get("myID", "")[:8]}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:120]}
|
|
|
|
|
|
# ---------------- TAILSCALE ----------------
|
|
def collect_tailscale() -> dict:
|
|
"""Get Tailscale mesh node status via CLI."""
|
|
import subprocess
|
|
exe = r"C:\Program Files\Tailscale\tailscale.exe"
|
|
try:
|
|
r = subprocess.run([exe, "status", "--json"],
|
|
capture_output=True, text=True, timeout=8)
|
|
if r.returncode != 0:
|
|
return {"ok": False, "error": r.stderr[:120]}
|
|
d = json.loads(r.stdout)
|
|
peers = d.get("Peer", {})
|
|
online = sum(1 for p in peers.values() if p.get("Online", False))
|
|
my_ips = d.get("TailscaleIPs") or []
|
|
my_ip = my_ips[0] if my_ips else "?"
|
|
return {"ok": True, "nodes_total": len(peers) + 1,
|
|
"nodes_online": online + 1, "peers_online": online,
|
|
"my_ip": my_ip}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:120]}
|
|
|
|
|
|
# ---------------- SYNTHESIS ----------------
|
|
def synthesize(state: dict) -> dict:
|
|
if model_router is None:
|
|
return {"markdown": "_model-router niet beschikbaar_", "ok": False, "model": ""}
|
|
mail = state.get("mail", {})
|
|
cal = state.get("calendar", {})
|
|
cmd = state.get("command", {})
|
|
ctx = {
|
|
"now": state["generated_at"],
|
|
"mail_unread_total": mail.get("total_unread"),
|
|
"mail_by_account": [
|
|
{"account": a["label"], "unread": a["unread"],
|
|
"recent_unread": [f"{r['from']} | {r['subject']}" for r in a["recent"] if r["unread"]][:4]}
|
|
for a in mail.get("accounts", []) if not a.get("error")
|
|
],
|
|
"today_events": [
|
|
f"{e['start'][11:16]}-{e['end'][11:16]} {e['summary']}"
|
|
+ (f" @ {e['location']}" if e["location"] else "")
|
|
for e in cal.get("today", [])
|
|
],
|
|
"upcoming_events": [
|
|
f"{e['start'][:16]} {e['summary']}" + (f" @ {e['location']}" if e["location"] else "")
|
|
for e in cal.get("upcoming", [])[:18]
|
|
],
|
|
"command_center": json.dumps(cmd.get("today", {}), default=str, ensure_ascii=False)[:1400],
|
|
"business_crm": json.dumps(cmd.get("business", {}), default=str, ensure_ascii=False)[:1400],
|
|
"odoo_summary": {
|
|
"open_tasks": state.get("odoo", {}).get("open_tasks"),
|
|
"overdue_invoices": state.get("odoo", {}).get("overdue_count"),
|
|
"outstanding_eur": state.get("odoo", {}).get("total_residual"),
|
|
},
|
|
"infrastructure": {
|
|
"atlas01_ok": state.get("atlas01", {}).get("ok"),
|
|
"atlas01_ram_pct": state.get("atlas01", {}).get("used_ram_pct"),
|
|
"atlas01_disk_free_gb": state.get("atlas01", {}).get("disk_free_gb"),
|
|
"backup_hourly_age_h": state.get("backup", {}).get("hourly_age_h"),
|
|
"backup_critical": state.get("backup", {}).get("critical"),
|
|
"pos_fleet_active": state.get("fleet", {}).get("pos_active"),
|
|
"dmarc_ok": state.get("dmarc", {}).get("ok"),
|
|
},
|
|
}
|
|
system = (
|
|
"You are Atlas Brain — the operational staff for Chaib, solo entrepreneur "
|
|
"(Atlas Agency, Atlas POS, and collaborations). You receive his mailboxes, calendar, "
|
|
"and command-center status. Write a SHARP briefing. Be concrete: name real senders, "
|
|
"real appointments, real times. Spot calendar conflicts — can he be in two places at once? "
|
|
"Flag explicitly what he risks forgetting: unread mail from important senders, appointments "
|
|
"still needing confirmation. Give 3-5 concrete actions. English only. No inventions — "
|
|
"only what is in the data."
|
|
)
|
|
user = (
|
|
"Context (JSON):\n```json\n"
|
|
+ json.dumps(ctx, indent=1, default=str, ensure_ascii=False)[:7000]
|
|
+ "\n```\n\nRespond in EXACTLY this structure (markdown):\n"
|
|
"## Headline\n<one sentence: the most important reality of today>\n\n"
|
|
"## Today's agenda\n<the appointments that actually matter; name conflicts explicitly>\n\n"
|
|
"## At risk of forgetting\n- ...\n- ...\n\n"
|
|
"## Actions today\n1. ...\n2. ...\n3. ...\n\n"
|
|
"## Mail that needs attention\n<1 to 3 lines: from whom, about what>\n\n"
|
|
"## Spoken\n<a flowing spoken paragraph of max 80 words, "
|
|
"complete sentences, no bullet points, no markdown>\n"
|
|
)
|
|
out = model_router.ask(user, task="chat", system=system, budget_s=240, num_predict=780)
|
|
return {"markdown": (out.get("text") or "").strip() or "_lege synthese_",
|
|
"ok": out.get("ok", False), "model": out.get("model", "")}
|
|
|
|
|
|
# ---------------- SPOKEN ----------------
|
|
def build_spoken(state: dict) -> dict:
|
|
mail = state.get("mail", {})
|
|
cal = state.get("calendar", {})
|
|
md = state.get("brief", {}).get("markdown", "")
|
|
now_hm = datetime.now().strftime("%H:%M")
|
|
future = [e for e in cal.get("today", []) if e.get("start", "")[11:16] >= now_hm]
|
|
nxt = future[0] if future else None
|
|
unread = mail.get("total_unread", 0)
|
|
|
|
spoken_para = _strip_md(_extract_section(md, "Spoken"))
|
|
if spoken_para and len(spoken_para) > 25:
|
|
full = spoken_para
|
|
else:
|
|
parts = [state.get("headline") or "Here is your briefing."]
|
|
if nxt:
|
|
parts.append(f"Your next appointment is {nxt['summary']} at {nxt['start'][11:16]}.")
|
|
parts.append(f"You have {unread} unread messages across your mailboxes.")
|
|
full = " ".join(parts)
|
|
|
|
sp = [f"It is {now_hm}."]
|
|
if nxt:
|
|
sp.append(f"Next appointment: {nxt['summary']} at {nxt['start'][11:16]}.")
|
|
else:
|
|
sp.append("No more appointments today.")
|
|
sp.append(f"{unread} unread emails.")
|
|
return {"full": full, "short": " ".join(sp)}
|
|
|
|
|
|
# ---------------- ODOO ----------------
|
|
def _odoo_connect():
|
|
"""Return (models_proxy, uid, DB, PASS) or raise."""
|
|
import xmlrpc.client
|
|
ODOO = os.environ.get("ATLAS_ODOO_URL", "http://<TAILSCALE_IP>:8069")
|
|
DB = os.environ.get("ATLAS_ODOO_DB", "atlas")
|
|
USER = os.environ.get("ATLAS_ODOO_USER", "chaib@atlascorporation.nl")
|
|
PASS = os.environ.get("ATLAS_ODOO_PW", "")
|
|
common = xmlrpc.client.ServerProxy(f"{ODOO}/xmlrpc/2/common")
|
|
uid = common.authenticate(DB, USER, PASS, {})
|
|
if not uid:
|
|
raise RuntimeError("Odoo auth_failed")
|
|
models = xmlrpc.client.ServerProxy(f"{ODOO}/xmlrpc/2/object")
|
|
return models, uid, DB, PASS
|
|
|
|
|
|
def collect_odoo_overdue() -> dict:
|
|
"""Fetch overdue invoices: posted, unpaid/partial, past invoice_date_due."""
|
|
try:
|
|
models, uid, DB, PASS = _odoo_connect()
|
|
today_str = datetime.now().strftime("%Y-%m-%d")
|
|
domain = [
|
|
["move_type", "in", ["out_invoice"]],
|
|
["state", "=", "posted"],
|
|
["payment_state", "in", ["not_paid", "partial"]],
|
|
["invoice_date_due", "<", today_str],
|
|
]
|
|
inv_ids = models.execute_kw(DB, uid, PASS, "account.move", "search",
|
|
[domain], {"limit": 50})
|
|
if not inv_ids:
|
|
return {"ok": True, "count": 0, "total_residual": 0.0, "invoices": []}
|
|
invoices = models.execute_kw(DB, uid, PASS, "account.move", "read", [inv_ids],
|
|
{"fields": ["name", "partner_id", "amount_residual",
|
|
"amount_total", "invoice_date_due", "payment_state"]})
|
|
result = []
|
|
total_residual = 0.0
|
|
for i in invoices:
|
|
due_str = i.get("invoice_date_due") or today_str
|
|
try:
|
|
due_dt = datetime.strptime(due_str, "%Y-%m-%d")
|
|
days_overdue = (datetime.now() - due_dt).days
|
|
except Exception:
|
|
days_overdue = 0
|
|
residual = float(i.get("amount_residual") or i.get("amount_total") or 0)
|
|
total_residual += residual
|
|
result.append({
|
|
"name": i["name"],
|
|
"partner": (i.get("partner_id") or [None, "?"])[1],
|
|
"amount_residual": round(residual, 2),
|
|
"amount_total": round(float(i.get("amount_total") or 0), 2),
|
|
"due": due_str,
|
|
"days_overdue": days_overdue,
|
|
"payment_state": i.get("payment_state", ""),
|
|
})
|
|
result.sort(key=lambda x: x["days_overdue"], reverse=True)
|
|
return {
|
|
"ok": True,
|
|
"count": len(result),
|
|
"total_residual": round(total_residual, 2),
|
|
"invoices": result,
|
|
}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:200]}
|
|
|
|
|
|
def collect_odoo() -> dict:
|
|
"""Pull task + invoice counts from self-hosted Odoo on atlas-01."""
|
|
try:
|
|
models, uid, DB, PASS = _odoo_connect()
|
|
# Open tasks
|
|
task_ids = models.execute_kw(DB, uid, PASS, "project.task", "search",
|
|
[[["stage_id.fold", "=", False]]], {"limit": 100})
|
|
tasks_count = len(task_ids)
|
|
# All unpaid invoices (posted)
|
|
today_str = datetime.now().strftime("%Y-%m-%d")
|
|
inv_ids = models.execute_kw(DB, uid, PASS, "account.move", "search",
|
|
[[["move_type", "in", ["out_invoice"]], ["state", "=", "posted"],
|
|
["payment_state", "in", ["not_paid", "partial"]]]],
|
|
{"limit": 50})
|
|
invoices = models.execute_kw(DB, uid, PASS, "account.move", "read", [inv_ids],
|
|
{"fields": ["name", "partner_id", "amount_total", "amount_residual",
|
|
"invoice_date_due", "payment_state"]})
|
|
overdue = [i for i in invoices if i.get("invoice_date_due") and
|
|
i["invoice_date_due"] < today_str]
|
|
# Detailed overdue block (reuse collect_odoo_overdue logic inline)
|
|
overdue_detail = []
|
|
total_residual = 0.0
|
|
for i in overdue:
|
|
due_str = i.get("invoice_date_due") or today_str
|
|
try:
|
|
due_dt = datetime.strptime(due_str, "%Y-%m-%d")
|
|
days_overdue = (datetime.now() - due_dt).days
|
|
except Exception:
|
|
days_overdue = 0
|
|
residual = float(i.get("amount_residual") or i.get("amount_total") or 0)
|
|
total_residual += residual
|
|
overdue_detail.append({
|
|
"name": i["name"],
|
|
"partner": (i.get("partner_id") or [None, "?"])[1],
|
|
"amount_residual": round(residual, 2),
|
|
"amount_total": round(float(i.get("amount_total") or 0), 2),
|
|
"due": due_str,
|
|
"days_overdue": days_overdue,
|
|
"payment_state": i.get("payment_state", ""),
|
|
})
|
|
overdue_detail.sort(key=lambda x: x["days_overdue"], reverse=True)
|
|
return {
|
|
"ok": True,
|
|
"open_tasks": tasks_count,
|
|
"unpaid_invoices": len(invoices),
|
|
"overdue_count": len(overdue),
|
|
"total_outstanding": round(sum(float(i.get("amount_total") or 0) for i in invoices), 2),
|
|
"total_residual": round(total_residual, 2),
|
|
"invoices": [{"name": i["name"], "partner": (i.get("partner_id") or [None, "?"])[1],
|
|
"amount": float(i.get("amount_total") or 0),
|
|
"residual": float(i.get("amount_residual") or 0),
|
|
"due": i.get("invoice_date_due"),
|
|
"state": i.get("payment_state")} for i in invoices[:8]],
|
|
"overdue_invoices": overdue_detail,
|
|
}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:120]}
|
|
|
|
|
|
# ---------------- DISK USAGE ----------------
|
|
def collect_disk() -> dict:
|
|
import shutil
|
|
drives = {}
|
|
for drive in ["C:\\", "D:\\"]:
|
|
try:
|
|
u = shutil.disk_usage(drive)
|
|
drives[drive[0]] = {
|
|
"total_gb": round(u.total / 1024**3, 1),
|
|
"used_gb": round(u.used / 1024**3, 1),
|
|
"free_gb": round(u.free / 1024**3, 1),
|
|
"used_pct": round(u.used / u.total * 100, 1),
|
|
"critical": (u.free / 1024**3) < 20 or (u.used / u.total) > 0.95,
|
|
"warn": (u.free / 1024**3) < 50 or (u.used / u.total) > 0.85,
|
|
}
|
|
except Exception:
|
|
pass
|
|
c = drives.get("C", {})
|
|
return {
|
|
"drives": drives,
|
|
"free_gb": c.get("free_gb"),
|
|
"used_pct": c.get("used_pct"),
|
|
"critical": c.get("critical", False),
|
|
"warn": c.get("warn", False),
|
|
}
|
|
|
|
|
|
# ---------------- SCHEDULED TASKS ----------------
|
|
def collect_scheduled_tasks() -> dict:
|
|
"""Count Windows scheduled tasks via schtasks."""
|
|
import subprocess
|
|
try:
|
|
r = subprocess.run(
|
|
["schtasks", "/query", "/fo", "CSV", "/nh"],
|
|
capture_output=True, text=True, timeout=15
|
|
)
|
|
lines = [l for l in r.stdout.splitlines() if l.strip() and not l.startswith('"TaskName"')]
|
|
atlas_lines = [l for l in lines if 'atlas' in l.lower() or 'jarvis' in l.lower()]
|
|
return {"ok": True, "total": len(lines), "atlas_count": len(atlas_lines)}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:120]}
|
|
|
|
|
|
# ---------------- VAULT STATS ----------------
|
|
def collect_vault_stats() -> dict:
|
|
"""Count vault notes + size from the MEGA Obsidian vault."""
|
|
import time as _time
|
|
vault = HOME / "MEGA" / "Atlas Corporation"
|
|
if not vault.exists():
|
|
return {"ok": False, "error": "vault not found"}
|
|
try:
|
|
count = 0
|
|
total_size = 0
|
|
newest_mtime = 0.0
|
|
newest_name = None
|
|
for f in vault.rglob("*.md"):
|
|
try:
|
|
st = f.stat()
|
|
count += 1
|
|
total_size += st.st_size
|
|
if st.st_mtime > newest_mtime:
|
|
newest_mtime = st.st_mtime
|
|
newest_name = f.name
|
|
except Exception:
|
|
pass
|
|
newest_age_h = round((_time.time() - newest_mtime) / 3600, 1) if newest_mtime else None
|
|
return {
|
|
"ok": True,
|
|
"note_count": count,
|
|
"size_mb": round(total_size / 1024 / 1024, 1),
|
|
"newest_note": newest_name,
|
|
"newest_age_h": newest_age_h,
|
|
}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:100]}
|
|
|
|
|
|
# ---------------- BACKUP STATUS ----------------
|
|
def collect_backup_status() -> dict:
|
|
"""Check Odoo backup freshness from the MEGA-synced backup folder."""
|
|
import time as _time
|
|
BACKUP_ROOT = HOME / "MEGA" / "Atlas Corporation" / "07_Finance" / "_atlas-odoo-backups"
|
|
result: dict = {"ok": True, "hourly_age_h": None, "daily_age_h": None, "warn": False, "critical": False}
|
|
for sub in ("hourly", "daily"):
|
|
d = BACKUP_ROOT / sub
|
|
if not d.exists():
|
|
result[f"{sub}_age_h"] = None
|
|
continue
|
|
files = sorted(d.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True) if d.exists() else []
|
|
if files:
|
|
age_h = round((_time.time() - files[0].stat().st_mtime) / 3600, 1)
|
|
result[f"{sub}_age_h"] = age_h
|
|
result[f"{sub}_name"] = files[0].name
|
|
else:
|
|
result[f"{sub}_age_h"] = None
|
|
hourly = result.get("hourly_age_h")
|
|
daily = result.get("daily_age_h")
|
|
if hourly is None or hourly > 3:
|
|
result["warn"] = True
|
|
if hourly is None or hourly > 12:
|
|
result["critical"] = True
|
|
if daily is None or daily > 48:
|
|
result["warn"] = True
|
|
return result
|
|
|
|
|
|
# ---------------- ATLAS-01 RESOURCES ----------------
|
|
def collect_atlas01_resources() -> dict:
|
|
"""SSH to atlas-01 to get free RAM, CPU load, uptime, and container count."""
|
|
import subprocess
|
|
cmd = [
|
|
"ssh", "-i", str(HOME / ".ssh" / "id_ed25519_atlas"),
|
|
"-o", "BatchMode=yes", "-o", "ConnectTimeout=10",
|
|
"-o", "StrictHostKeyChecking=accept-new",
|
|
"root@<TAILSCALE_IP>",
|
|
"python3 -c \""
|
|
"import os,subprocess;"
|
|
"mem=open('/proc/meminfo').read();"
|
|
"free_kb=int([l for l in mem.splitlines() if 'MemAvailable' in l][0].split()[1]);"
|
|
"total_kb=int([l for l in mem.splitlines() if 'MemTotal' in l][0].split()[1]);"
|
|
"load=open('/proc/loadavg').read().split()[:3];"
|
|
"uptm=float(open('/proc/uptime').read().split()[0]);"
|
|
"ctrs=len([l for l in subprocess.run(['docker','ps','-q'],capture_output=True,text=True).stdout.strip().splitlines() if l]);"
|
|
"df_o=subprocess.run(['df','-BG','/'],capture_output=True,text=True).stdout.splitlines();"
|
|
"dp=df_o[1].split() if len(df_o)>1 else ['?','0G','0G','0G'];"
|
|
"d_tot=int(dp[1].rstrip('G'));d_free=int(dp[3].rstrip('G'));"
|
|
"print(f'{free_kb},{total_kb},{load[0]},{load[1]},{load[2]},{int(uptm//3600)},{ctrs},{d_tot},{d_free}')\""
|
|
]
|
|
try:
|
|
r = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
|
|
if r.returncode != 0:
|
|
return {"ok": False, "error": r.stderr.strip()[:100]}
|
|
parts = r.stdout.strip().split(",")
|
|
if len(parts) < 9:
|
|
return {"ok": False, "error": "unexpected output"}
|
|
free_kb, total_kb = int(parts[0]), int(parts[1])
|
|
load1, load5, load15 = float(parts[2]), float(parts[3]), float(parts[4])
|
|
uptime_h = int(parts[5])
|
|
containers = int(parts[6])
|
|
disk_total_gb, disk_free_gb = int(parts[7]), int(parts[8])
|
|
disk_used_pct = round((disk_total_gb - disk_free_gb) / disk_total_gb * 100, 1) if disk_total_gb > 0 else 0
|
|
free_gb = round(free_kb / 1024 / 1024, 2)
|
|
used_pct = round((total_kb - free_kb) / total_kb * 100, 1)
|
|
return {
|
|
"ok": True,
|
|
"free_ram_gb": free_gb,
|
|
"total_ram_gb": round(total_kb / 1024 / 1024, 2),
|
|
"used_ram_pct": used_pct,
|
|
"load_1m": load1,
|
|
"load_5m": load5,
|
|
"load_15m": load15,
|
|
"uptime_h": uptime_h,
|
|
"containers": containers,
|
|
"disk_total_gb": disk_total_gb,
|
|
"disk_free_gb": disk_free_gb,
|
|
"disk_used_pct": disk_used_pct,
|
|
"disk_critical": disk_free_gb < 10,
|
|
"warn": used_pct > 85 or load1 > 4.0 or disk_free_gb < 20,
|
|
}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:120]}
|
|
|
|
|
|
# ---------------- AUCTION INTEL ----------------
|
|
def collect_auction_intel() -> dict:
|
|
"""Pull live stats from the auction intel dashboard API (localhost:7781)."""
|
|
import sqlite3
|
|
intel_db = HOME / "twa-scrape" / "intel.db"
|
|
if not intel_db.exists():
|
|
return {"ok": False, "reason": "db_not_found"}
|
|
try:
|
|
conn = sqlite3.connect(str(intel_db))
|
|
conn.row_factory = sqlite3.Row
|
|
active = conn.execute("SELECT COUNT(*) FROM lots WHERE status='active'").fetchone()[0]
|
|
strong = conn.execute("SELECT COUNT(*) FROM lots WHERE status='active' AND stars>=4").fetchone()[0]
|
|
avg_m = conn.execute("SELECT AVG(margin_pct) FROM lots WHERE status='active' AND margin_pct IS NOT NULL").fetchone()[0]
|
|
try:
|
|
urgent = conn.execute("SELECT COUNT(*) FROM lots WHERE status='active' AND urgency=1").fetchone()[0]
|
|
except Exception:
|
|
urgent = 0
|
|
top5 = [dict(r) for r in conn.execute(
|
|
"SELECT id, title, model_key, hammer, margin_pct, stars, close_time "
|
|
"FROM lots WHERE status='active' AND stars>=4 "
|
|
"ORDER BY stars DESC, margin_pct DESC LIMIT 5"
|
|
).fetchall()]
|
|
try:
|
|
last_run = dict(conn.execute(
|
|
"SELECT started_at, run_type, lots_found FROM run_log ORDER BY id DESC LIMIT 1"
|
|
).fetchone() or {})
|
|
except Exception:
|
|
last_run = {}
|
|
conn.close()
|
|
return {
|
|
"ok": True,
|
|
"lots_active": active,
|
|
"strong_buys": strong,
|
|
"urgent_count": urgent,
|
|
"avg_margin": round(avg_m or 0, 1),
|
|
"top_lots": top5,
|
|
"last_run": last_run,
|
|
"dashboard_url": "http://localhost:7781/",
|
|
}
|
|
except Exception as e:
|
|
return {"ok": False, "reason": str(e)}
|
|
|
|
|
|
# ---------------- POS FLEET ----------------
|
|
def collect_pos_fleet() -> dict:
|
|
"""Read POS fleet config — static count from pos_targets.json."""
|
|
pts = ATLAS / "pos_targets.json"
|
|
if not pts.exists():
|
|
return {"ok": False, "reason": "pos_targets.json not found"}
|
|
try:
|
|
cfg = json.loads(pts.read_text(encoding="utf-8"))
|
|
pos = cfg.get("pos_agents", [])
|
|
internal = cfg.get("atlas_internal", [])
|
|
active = [p for p in pos if not p.get("skip_autopilot")]
|
|
skipped = [p for p in pos if p.get("skip_autopilot")]
|
|
return {
|
|
"ok": True,
|
|
"pos_count": len(pos),
|
|
"pos_active": len(active),
|
|
"pos_skipped": len(skipped),
|
|
"internal_count": len(internal),
|
|
"skip_reasons": [{"name": p["dws_name"], "reason": p.get("skip_reason", "?")[:60]}
|
|
for p in skipped],
|
|
}
|
|
except Exception as e:
|
|
return {"ok": False, "error": str(e)[:120]}
|
|
|
|
|
|
# ---------------- DMARC CHECK ----------------
|
|
def collect_dmarc() -> dict:
|
|
"""DNS check for DMARC policy on Atlas domains. p=none = not enforced."""
|
|
import socket
|
|
domains = ["atlascorporation.nl", "atlasworks.nl", "atlasworkspace.eu", "atlasoperations.nl"]
|
|
results = {}
|
|
for d in domains:
|
|
try:
|
|
txt = socket.getaddrinfo(f"_dmarc.{d}", None)
|
|
except Exception:
|
|
txt = None
|
|
try:
|
|
import subprocess
|
|
r = subprocess.run(
|
|
["nslookup", "-type=TXT", f"_dmarc.{d}", "8.8.8.8"],
|
|
capture_output=True, text=True, timeout=8
|
|
)
|
|
raw = r.stdout
|
|
if "v=DMARC1" in raw:
|
|
policy = "quarantine" if "p=quarantine" in raw else (
|
|
"reject" if "p=reject" in raw else "none")
|
|
results[d] = {"ok": policy != "none", "policy": policy}
|
|
else:
|
|
results[d] = {"ok": False, "policy": "missing"}
|
|
except Exception as e:
|
|
results[d] = {"ok": None, "error": str(e)[:80]}
|
|
enforced = [d for d, v in results.items() if v.get("ok")]
|
|
missing = [d for d, v in results.items() if not v.get("ok")]
|
|
return {"ok": len(missing) == 0, "domains": results,
|
|
"enforced": enforced, "missing": missing}
|
|
|
|
|
|
# ---------------- MAIN ----------------
|
|
def main() -> None:
|
|
quick = "--quick" in sys.argv
|
|
send_report = "--report" in sys.argv
|
|
t0 = time.time()
|
|
state = {
|
|
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
|
"generated_unix": int(time.time()),
|
|
"operator": "Chaib",
|
|
"mode": "quick" if quick else "full",
|
|
}
|
|
state["mail"] = _safe(collect_mail, "mail")
|
|
state["calendar"] = _safe(collect_calendar, "calendar")
|
|
state["command"] = _safe(collect_command, "command")
|
|
state["service_health"] = _safe(collect_service_health, "service_health")
|
|
state["syncthing"] = _safe(collect_syncthing, "syncthing")
|
|
state["tailscale"] = _safe(collect_tailscale, "tailscale")
|
|
state["ollama"] = _safe(collect_ollama, "ollama")
|
|
state["scheduled_tasks"] = _safe(collect_scheduled_tasks, "scheduled_tasks")
|
|
state["auction_intel"] = _safe(collect_auction_intel, "auction_intel")
|
|
state["disk"] = _safe(collect_disk, "disk")
|
|
state["atlas01"] = _safe(collect_atlas01_resources, "atlas01")
|
|
state["backup"] = _safe(collect_backup_status, "backup")
|
|
state["fleet"] = _safe(collect_pos_fleet, "fleet")
|
|
state["dmarc"] = _safe(collect_dmarc, "dmarc")
|
|
# Slow collectors — only on full runs; quick ticks carry forward the last state
|
|
if not quick:
|
|
state["vault"] = _safe(collect_vault_stats, "vault") # 5s rglob over 10k files
|
|
state["odoo"] = _safe(collect_odoo, "odoo")
|
|
# Store overdue invoices as its own top-level key for check_overdue.py / portal
|
|
odoo_data = state["odoo"]
|
|
if odoo_data.get("ok"):
|
|
state["overdue_invoices"] = {
|
|
"ok": True,
|
|
"count": odoo_data.get("overdue_count", 0),
|
|
"total_residual": odoo_data.get("total_residual", 0.0),
|
|
"invoices": odoo_data.get("overdue_invoices", []),
|
|
"updated_at": state["generated_at"],
|
|
}
|
|
else:
|
|
state["overdue_invoices"] = {"ok": False, "error": odoo_data.get("error", "odoo_error"),
|
|
"updated_at": state["generated_at"]}
|
|
else:
|
|
try:
|
|
prev = json.loads(STATE_JSON.read_text(encoding="utf-8")) if STATE_JSON.exists() else {}
|
|
state["odoo"] = prev.get("odoo", {"ok": False, "error": "quick_tick_no_refresh"})
|
|
state["overdue_invoices"] = prev.get("overdue_invoices",
|
|
{"ok": False, "error": "quick_tick_no_refresh"})
|
|
state["vault"] = prev.get("vault", {"ok": False, "error": "quick_tick_no_refresh"})
|
|
except Exception:
|
|
state["odoo"] = {"ok": False, "error": "quick_tick_no_refresh"}
|
|
state["overdue_invoices"] = {"ok": False, "error": "quick_tick_no_refresh"}
|
|
state["vault"] = {"ok": False, "error": "quick_tick_no_refresh"}
|
|
|
|
if quick:
|
|
state["brief"] = {"markdown": f"_quick tick {state['generated_at'][11:16]} — no synthesis_",
|
|
"ok": False, "model": ""}
|
|
else:
|
|
state["brief"] = _safe(lambda: synthesize(state), "synth")
|
|
|
|
state["headline"] = (_extract_section(state["brief"].get("markdown", ""), "Headline").splitlines()
|
|
or _extract_section(state["brief"].get("markdown", ""), "Kop").splitlines()
|
|
or ["Atlas briefing."])[0].strip().lstrip("#").strip() or "Atlas briefing."
|
|
state["spoken"] = build_spoken(state)
|
|
state["services"] = SERVICES # static link list for portal
|
|
state["build_seconds"] = round(time.time() - t0, 1)
|
|
|
|
state_bytes = json.dumps(state, indent=2, default=str, ensure_ascii=False)
|
|
STATE_JSON.write_text(state_bytes, encoding="utf-8")
|
|
# Mirror to state/ sub-dir (served by the 7799 HTTP server)
|
|
(OPS_DIR / "state" / "atlas-state.json").write_text(state_bytes, encoding="utf-8")
|
|
SPOKEN_JSON.write_text(json.dumps(state["spoken"], indent=2, ensure_ascii=False), encoding="utf-8")
|
|
try:
|
|
cmd_dir = HOME / "MEGA" / "Atlas Corporation" / "vault" / "_command"
|
|
if cmd_dir.exists():
|
|
(cmd_dir / "jarvis-brief.md").write_text(state["brief"].get("markdown", ""), encoding="utf-8")
|
|
except Exception:
|
|
pass
|
|
|
|
log(f"state written -> {STATE_JSON} ({state['build_seconds']}s, "
|
|
f"mail_unread={state['mail'].get('total_unread')}, "
|
|
f"today_events={len(state['calendar'].get('today', []))})")
|
|
|
|
if send_report and not quick:
|
|
try:
|
|
import subprocess
|
|
py = sys.executable
|
|
rpt = Path(__file__).parent / "atlas_morning_report.py"
|
|
subprocess.Popen([py, str(rpt)], creationflags=0x08000000)
|
|
log("Morning report spawned")
|
|
except Exception as e:
|
|
log(f"Morning report failed: {e}")
|
|
|
|
print(json.dumps({
|
|
"ok": True, "mode": state["mode"], "state_file": str(STATE_JSON),
|
|
"mail_unread": state["mail"].get("total_unread"),
|
|
"today_events": len(state["calendar"].get("today", [])),
|
|
"headline": state["headline"], "seconds": state["build_seconds"],
|
|
}, indent=2, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
except Exception:
|
|
pass
|
|
main()
|