From eb5b67dee267488829072898b744cdf5c5055434 Mon Sep 17 00:00:00 2001 From: Atlas Date: Wed, 1 Jul 2026 21:50:20 +0200 Subject: [PATCH] Add connectors + setup wizard (files / IMAP email / Nextcloud-WebDAV) + encrypted vault Product layer on top of the pure engine: a wizard to connect the apps that feed the brain, an encrypted credential vault (Fernet/PBKDF2; app logins never enter brain.db), and 3 working connectors. Connector data flows through the same credential-scrub path as the engine. Atlas Corporation proprietary. --- README.md | 24 +++++++++++ connectors/__init__.py | 1 + connectors/base.py | 60 ++++++++++++++++++++++++++++ connectors/filesystem.py | 25 ++++++++++++ connectors/imap_email.py | 57 ++++++++++++++++++++++++++ connectors/vault.py | 51 ++++++++++++++++++++++++ connectors/webdav.py | 49 +++++++++++++++++++++++ requirements.txt | 1 + setup.py | 86 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 354 insertions(+) create mode 100644 connectors/__init__.py create mode 100644 connectors/base.py create mode 100644 connectors/filesystem.py create mode 100644 connectors/imap_email.py create mode 100644 connectors/vault.py create mode 100644 connectors/webdav.py create mode 100755 setup.py diff --git a/README.md b/README.md index 1c17a40..86698fb 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,30 @@ Output: def login(user, pw): # verifies against argon2 hash, issues a signed … ``` +## Setup wizard — connect your apps + +`brain.py` is the engine; the **setup wizard** is the product experience. Instead of manually +pointing at directories, run: + +```bash +python3 setup.py # pick apps → sign in → test → sync +python3 setup.py sync # re-sync everything (put on a cron/timer) +python3 setup.py list # available connectors +``` + +The wizard walks you through connecting the apps that hold your life, stores each app's login in +an **encrypted vault** (`connectors/vault.py` — Fernet/PBKDF2, `~/.secondbrain/vault.enc`, chmod +600), tests the connection, and syncs. **App credentials never enter `brain.db`** — connectors +pull data and only *scrubbed* text reaches the brain. + +**Connectors today:** Files & folders · Email (IMAP) · Nextcloud / WebDAV. +**On the roadmap:** Google & Microsoft (OAuth) · WhatsApp (Evolution) · Calendar (CalDAV) · +Paperless-ngx · Odoo · Obsidian · Git/GitHub. A connector is ~40 lines — subclass `Connector`, +declare its `fields`, implement `test()` + `sync()`; `feed()` runs it through the engine's scrub. + +> **Architecture note:** the engine stays pure and connector-agnostic. Connectors + vault + wizard +> are an optional layer on top — SecondBrain still works as a bare `brain.py` library/CLI without them. + ## Configuration All via environment (or a `.env` next to `brain.py`). Copy `.env.example` → `.env`: diff --git a/connectors/__init__.py b/connectors/__init__.py new file mode 100644 index 0000000..be7d79d --- /dev/null +++ b/connectors/__init__.py @@ -0,0 +1 @@ +# SecondBrain connectors — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE. diff --git a/connectors/base.py b/connectors/base.py new file mode 100644 index 0000000..0d12c31 --- /dev/null +++ b/connectors/base.py @@ -0,0 +1,60 @@ +# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE. +"""Connector framework — the "senses" made concrete. + +Each connector authenticates to an app, pulls data, and feeds it through the SAME +scrub + dedup path as the core engine (`brain._stage`). Consequences: + * connector data is credential-scrubbed too (secrets never land in brain.db); + * app logins live only in the encrypted vault (see vault.py), never in brain.db; + * the engine (brain.py) stays completely connector-agnostic. +""" +import importlib, os, sys, pkgutil + +REGISTRY = {} +def register(cls): + REGISTRY[cls.id] = cls + return cls + +class Field: + def __init__(self, key, label, secret=False, default="", required=True): + self.key, self.label, self.secret = key, label, secret + self.default, self.required = default, required + +class Connector: + id = "" # short stable id (vault key) + name = "" # display name + blurb = "" # one-line description + fields = [] # list[Field] the wizard collects + def test(self, cfg): # verify creds; raise on failure; return status str + return "ok" + def sync(self, cfg): # pull -> feed(); return new-chunk count + raise NotImplementedError + +def _brain(): + """Import the sibling engine regardless of CWD.""" + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + if here not in sys.path: + sys.path.insert(0, here) + import brain + return brain + +def feed(items, source): + """Stage connector items through the engine's scrub+dedup. + items: iterable of dict(text[, project, path, ref, ts, role]). + Returns new-chunk count. Does NOT embed (run brain.embed() after a sync batch).""" + brain = _brain() + conn = brain.db(); n = 0 + for it in items: + if not it.get("text"): + continue + n += brain._stage(conn, source, it.get("project", ""), it.get("path", ""), + it.get("ref", ""), it.get("ts", ""), it.get("role", ""), it["text"]) + conn.commit(); conn.close() + return n + +def load_all(): + """Import every connector module so it self-registers.""" + pkgdir = os.path.dirname(os.path.abspath(__file__)) + for m in pkgutil.iter_modules([pkgdir]): + if m.name != "base" and not m.name.startswith("_") and m.name != "vault": + importlib.import_module(f"connectors.{m.name}") + return REGISTRY diff --git a/connectors/filesystem.py b/connectors/filesystem.py new file mode 100644 index 0000000..fdf873c --- /dev/null +++ b/connectors/filesystem.py @@ -0,0 +1,25 @@ +# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE. +"""Connector: local files & folders (notes, code, docs).""" +import os +from .base import Connector, Field, register, _brain + +@register +class FilesystemConnector(Connector): + id = "files" + name = "Files & folders" + blurb = "Index a local directory of notes / code / docs (60+ text types)." + fields = [Field("path", "Directory to index", default="~/notes")] + + def test(self, cfg): + p = os.path.expanduser(cfg["path"]) + if not os.path.isdir(p): + raise RuntimeError(f"not a directory: {p}") + return f"dir ok: {p}" + + def sync(self, cfg): + brain = _brain() + p = os.path.expanduser(cfg["path"]) + conn = brain.db(); before = conn.execute("select count(*) from chunks").fetchone()[0]; conn.close() + brain.ingest_files(p) # stages directly through scrub + conn = brain.db(); after = conn.execute("select count(*) from chunks").fetchone()[0]; conn.close() + return after - before diff --git a/connectors/imap_email.py b/connectors/imap_email.py new file mode 100644 index 0000000..4f26da3 --- /dev/null +++ b/connectors/imap_email.py @@ -0,0 +1,57 @@ +# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE. +"""Connector: Email over IMAP. Use an app password (not your main password). +Message bodies pass through the engine scrub, so any secrets inside emails are redacted.""" +import imaplib, email +from email.header import decode_header, make_header +from .base import Connector, Field, register, feed + +@register +class ImapEmailConnector(Connector): + id = "email" + name = "Email (IMAP)" + blurb = "Pull recent messages from any IMAP mailbox (Gmail, Outlook, self-hosted…)." + fields = [ + Field("host", "IMAP host", default="imap.gmail.com"), + Field("port", "IMAP port", default="993"), + Field("user", "Username / email address"), + Field("password", "App password", secret=True), + Field("folder", "Folder", default="INBOX"), + Field("limit", "Max recent messages", default="200"), + ] + + def _open(self, cfg): + M = imaplib.IMAP4_SSL(cfg["host"], int(cfg.get("port") or 993)) + M.login(cfg["user"], cfg["password"]) + return M + + def test(self, cfg): + M = self._open(cfg); M.select(cfg.get("folder", "INBOX"), readonly=True); M.logout() + return "login ok" + + def sync(self, cfg): + M = self._open(cfg) + M.select(cfg.get("folder", "INBOX"), readonly=True) + _, data = M.search(None, "ALL") + ids = data[0].split()[-int(cfg.get("limit") or 200):] + items = [] + for i in ids: + _, d = M.fetch(i, "(RFC822)") + if not d or not isinstance(d[0], tuple): + continue + msg = email.message_from_bytes(d[0][1]) + def H(h): + try: return str(make_header(decode_header(msg.get(h, "") or ""))) + except Exception: return msg.get(h, "") or "" + body = "" + if msg.is_multipart(): + for p in msg.walk(): + if p.get_content_type() == "text/plain": + try: body = (p.get_payload(decode=True) or b"").decode("utf-8", "ignore"); break + except Exception: pass + else: + try: body = (msg.get_payload(decode=True) or b"").decode("utf-8", "ignore") + except Exception: body = str(msg.get_payload()) + items.append({"text": f"[email from {H('From')}] {H('Subject')}\n{body[:1500]}", + "project": "email", "ref": H('From')[:80], "ts": H('Date'), "role": "email"}) + M.logout() + return feed(items, "events") diff --git a/connectors/vault.py b/connectors/vault.py new file mode 100644 index 0000000..691ac52 --- /dev/null +++ b/connectors/vault.py @@ -0,0 +1,51 @@ +# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE. +"""Encrypted credential vault for connectors. + +App logins (IMAP passwords, WebDAV app-passwords, OAuth tokens) are stored here, +encrypted at rest with a master passphrase (Fernet, PBKDF2-HMAC-SHA256, 390k iters). +brain.db NEVER sees these — connectors read creds from the vault, pull data, and only +scrubbed text reaches the brain. Set BRAIN_VAULT_PASS to run non-interactively. +""" +import os, json, base64, getpass +from cryptography.fernet import Fernet +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC +from cryptography.hazmat.primitives import hashes + +_HOME = os.environ.get("BRAIN_HOME", os.path.expanduser("~/.secondbrain")) +VAULT = os.path.join(_HOME, "vault.enc") +SALT_F = os.path.join(_HOME, "vault.salt") + +def _salt(): + os.makedirs(_HOME, exist_ok=True) + if not os.path.exists(SALT_F): + s = os.urandom(16) + with open(SALT_F, "wb") as f: + f.write(s) + os.chmod(SALT_F, 0o600) + return s + with open(SALT_F, "rb") as f: + return f.read() + +def _key(passphrase, salt): + kdf = PBKDF2HMAC(algorithm=hashes.SHA256(), length=32, salt=salt, iterations=390000) + return base64.urlsafe_b64encode(kdf.derive(passphrase.encode())) + +def _pass(prompt): + return os.environ.get("BRAIN_VAULT_PASS") or getpass.getpass(prompt) + +def load(passphrase=None): + if not os.path.exists(VAULT): + return {} + passphrase = passphrase or _pass("Vault passphrase: ") + f = Fernet(_key(passphrase, _salt())) + with open(VAULT, "rb") as fh: + return json.loads(f.decrypt(fh.read()).decode()) + +def save(data, passphrase=None): + passphrase = passphrase or _pass("Vault passphrase (set/confirm): ") + os.makedirs(_HOME, exist_ok=True) + f = Fernet(_key(passphrase, _salt())) + tok = f.encrypt(json.dumps(data).encode()) + with open(VAULT, "wb") as fh: + fh.write(tok) + os.chmod(VAULT, 0o600) diff --git a/connectors/webdav.py b/connectors/webdav.py new file mode 100644 index 0000000..de39a95 --- /dev/null +++ b/connectors/webdav.py @@ -0,0 +1,49 @@ +# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE. +"""Connector: Nextcloud / any WebDAV — index text files from a cloud folder. +For Nextcloud the base URL looks like: https://cloud.example.com/remote.php/dav/files/USERNAME/ +Use a Nextcloud *app password*, not your login password.""" +import os +import requests +from xml.etree import ElementTree as ET +from .base import Connector, Field, register, feed + +TEXT_EXT = (".md", ".txt", ".rst", ".py", ".sh", ".json", ".csv", ".tsv", ".yml", ".yaml", + ".ini", ".cfg", ".conf", ".html", ".js", ".ts", ".sql", ".xml") + +@register +class WebDavConnector(Connector): + id = "webdav" + name = "Nextcloud / WebDAV files" + blurb = "Index text files from a Nextcloud (or any WebDAV) folder." + fields = [ + Field("base_url", "WebDAV base URL (…/remote.php/dav/files/USER/)"), + Field("user", "Username"), + Field("password", "App password", secret=True), + Field("folder", "Folder path under base", default="/"), + ] + + def _list(self, cfg): + url = cfg["base_url"].rstrip("/") + "/" + cfg.get("folder", "/").strip("/") + r = requests.request("PROPFIND", url, auth=(cfg["user"], cfg["password"]), + headers={"Depth": "infinity"}, timeout=30) + r.raise_for_status() + hrefs = [e.text for e in ET.fromstring(r.content).iter("{DAV:}href")] + return [h for h in hrefs if h and h.lower().endswith(TEXT_EXT)] + + def test(self, cfg): + return f"{len(self._list(cfg))} text files visible" + + def sync(self, cfg): + host = cfg["base_url"].split("/remote.php")[0] + items = [] + for href in self._list(cfg): + u = href if href.startswith("http") else host + href + try: + rr = requests.get(u, auth=(cfg["user"], cfg["password"]), timeout=30) + if rr.status_code != 200 or len(rr.content) > 400000: + continue + items.append({"text": rr.text, "project": "nextcloud", "path": href, + "ref": os.path.basename(href.rstrip("/")), "role": "file"}) + except Exception: + continue + return feed(items, "file") diff --git a/requirements.txt b/requirements.txt index 6dcf501..ef38998 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ sqlite-vec>=0.1.6 requests>=2.31 numpy>=1.24 +cryptography>=41 # connectors credential vault (setup wizard); engine works without it diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..0e94d10 --- /dev/null +++ b/setup.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE. +"""SecondBrain setup wizard — connect the apps that feed your brain. + + python3 setup.py interactive: pick apps, sign in, test, sync + python3 setup.py sync re-sync every already-connected app (for cron/timer) + python3 setup.py list show available connectors + +App credentials go into the encrypted vault (connectors/vault.py), never into brain.db. +""" +import sys, os +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from connectors import base, vault + + +def _ask(field): + import getpass + label = f" {field.label}" + (f" [{field.default}]" if field.default else "") + ": " + val = getpass.getpass(label) if field.secret else input(label) + return (val or "").strip() or field.default + + +def cmd_list(): + base.load_all() + print("\nAvailable connectors:") + for c in base.REGISTRY.values(): + print(f" • {c.name:26} {c.blurb}") + + +def wizard(): + base.load_all() + conns = list(base.REGISTRY.values()) + print("\n🧠 SecondBrain — connect your apps\n") + for i, c in enumerate(conns, 1): + print(f" {i}. {c.name} — {c.blurb}") + print(" q. done") + cfgs = vault.load() if os.path.exists(vault.VAULT) else {} + while True: + ch = input("\nConnect which? [number/q] ").strip().lower() + if ch in ("q", ""): + break + try: + c = conns[int(ch) - 1] + except Exception: + print(" ?"); continue + inst = c() + cfg = {f.key: _ask(f) for f in inst.fields} + try: + print(" testing…", inst.test(cfg)) + except Exception as e: + print(f" ✗ could not connect: {e}"); continue + cfgs[c.id] = cfg + vault.save(cfgs) + print(f" ✓ {c.name} connected + saved to encrypted vault") + if input(" sync now? [Y/n] ").strip().lower() in ("", "y"): + try: + n = inst.sync(cfg); print(f" … ingested {n} new chunks") + except Exception as e: + print(f" sync error: {e}") + print("\nEmbedding new content (local)…") + try: + base._brain().embed(limit=20000) + except Exception as e: + print(f" embed skipped: {e}") + print('Done. Ask your brain: python3 brain.py recall "…" --hybrid\n') + + +def cmd_sync(): + base.load_all() + cfgs = vault.load() + if not cfgs: + print("no connectors configured — run `python3 setup.py` first"); return + for cid, cfg in cfgs.items(): + c = base.REGISTRY.get(cid) + if not c: + print(f"[{cid}] unknown connector, skipped"); continue + try: + n = c().sync(cfg); print(f"[{cid}] +{n} chunks") + except Exception as e: + print(f"[{cid}] error: {e}") + base._brain().embed(limit=50000) + + +if __name__ == "__main__": + arg = sys.argv[1] if len(sys.argv) > 1 else "" + {"sync": cmd_sync, "list": cmd_list}.get(arg, wizard)()