Add connectors + setup wizard (files / IMAP email / Nextcloud-WebDAV) + encrypted vault

Product layer on top of the pure engine: a wizard to connect the apps that feed
the brain, an encrypted credential vault (Fernet/PBKDF2; app logins never enter
brain.db), and 3 working connectors. Connector data flows through the same
credential-scrub path as the engine. Atlas Corporation proprietary.

README.md

@@ -79,6 +79,30 @@ Output:
    def login(user, pw):  # verifies against argon2 hash, issues a signed …
 ```
 
+## Setup wizard — connect your apps
+
+`brain.py` is the engine; the **setup wizard** is the product experience. Instead of manually
+pointing at directories, run:
+
+```bash
+python3 setup.py            # pick apps → sign in → test → sync
+python3 setup.py sync       # re-sync everything (put on a cron/timer)
+python3 setup.py list       # available connectors
+```
+
+The wizard walks you through connecting the apps that hold your life, stores each app's login in
+an **encrypted vault** (`connectors/vault.py` — Fernet/PBKDF2, `~/.secondbrain/vault.enc`, chmod
+600), tests the connection, and syncs. **App credentials never enter `brain.db`** — connectors
+pull data and only *scrubbed* text reaches the brain.
+
+**Connectors today:** Files & folders · Email (IMAP) · Nextcloud / WebDAV.
+**On the roadmap:** Google & Microsoft (OAuth) · WhatsApp (Evolution) · Calendar (CalDAV) ·
+Paperless-ngx · Odoo · Obsidian · Git/GitHub. A connector is ~40 lines — subclass `Connector`,
+declare its `fields`, implement `test()` + `sync()`; `feed()` runs it through the engine's scrub.
+
+> **Architecture note:** the engine stays pure and connector-agnostic. Connectors + vault + wizard
+> are an optional layer on top — SecondBrain still works as a bare `brain.py` library/CLI without them.
+
 ## Configuration
 
 All via environment (or a `.env` next to `brain.py`). Copy `.env.example` → `.env`:

connectors/__init__.py

@@ -0,0 +1 @@
+# SecondBrain connectors — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE.

connectors/base.py

@@ -0,0 +1,60 @@
+# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE.
+"""Connector framework — the "senses" made concrete.
+
+Each connector authenticates to an app, pulls data, and feeds it through the SAME
+scrub + dedup path as the core engine (`brain._stage`). Consequences:
+  * connector data is credential-scrubbed too (secrets never land in brain.db);
+  * app logins live only in the encrypted vault (see vault.py), never in brain.db;
+  * the engine (brain.py) stays completely connector-agnostic.
+"""
+import importlib, os, sys, pkgutil
+
+REGISTRY = {}
+def register(cls):
+    REGISTRY[cls.id] = cls
+    return cls
+
+class Field:
+    def __init__(self, key, label, secret=False, default="", required=True):
+        self.key, self.label, self.secret = key, label, secret
+        self.default, self.required = default, required
+
+class Connector:
+    id = ""       # short stable id (vault key)
+    name = ""     # display name
+    blurb = ""    # one-line description
+    fields = []   # list[Field] the wizard collects
+    def test(self, cfg):        # verify creds; raise on failure; return status str
+        return "ok"
+    def sync(self, cfg):        # pull -> feed(); return new-chunk count
+        raise NotImplementedError
+
+def _brain():
+    """Import the sibling engine regardless of CWD."""
+    here = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    if here not in sys.path:
+        sys.path.insert(0, here)
+    import brain
+    return brain
+
+def feed(items, source):
+    """Stage connector items through the engine's scrub+dedup.
+    items: iterable of dict(text[, project, path, ref, ts, role]).
+    Returns new-chunk count. Does NOT embed (run brain.embed() after a sync batch)."""
+    brain = _brain()
+    conn = brain.db(); n = 0
+    for it in items:
+        if not it.get("text"):
+            continue
+        n += brain._stage(conn, source, it.get("project", ""), it.get("path", ""),
+                          it.get("ref", ""), it.get("ts", ""), it.get("role", ""), it["text"])
+    conn.commit(); conn.close()
+    return n
+
+def load_all():
+    """Import every connector module so it self-registers."""
+    pkgdir = os.path.dirname(os.path.abspath(__file__))
+    for m in pkgutil.iter_modules([pkgdir]):
+        if m.name != "base" and not m.name.startswith("_") and m.name != "vault":
+            importlib.import_module(f"connectors.{m.name}")
+    return REGISTRY

connectors/filesystem.py

@@ -0,0 +1,25 @@
+# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE.
+"""Connector: local files & folders (notes, code, docs)."""
+import os
+from .base import Connector, Field, register, _brain
+
+@register
+class FilesystemConnector(Connector):
+    id = "files"
+    name = "Files & folders"
+    blurb = "Index a local directory of notes / code / docs (60+ text types)."
+    fields = [Field("path", "Directory to index", default="~/notes")]
+
+    def test(self, cfg):
+        p = os.path.expanduser(cfg["path"])
+        if not os.path.isdir(p):
+            raise RuntimeError(f"not a directory: {p}")
+        return f"dir ok: {p}"
+
+    def sync(self, cfg):
+        brain = _brain()
+        p = os.path.expanduser(cfg["path"])
+        conn = brain.db(); before = conn.execute("select count(*) from chunks").fetchone()[0]; conn.close()
+        brain.ingest_files(p)      # stages directly through scrub
+        conn = brain.db(); after = conn.execute("select count(*) from chunks").fetchone()[0]; conn.close()
+        return after - before

connectors/imap_email.py

@@ -0,0 +1,57 @@
+# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE.
+"""Connector: Email over IMAP. Use an app password (not your main password).
+Message bodies pass through the engine scrub, so any secrets inside emails are redacted."""
+import imaplib, email
+from email.header import decode_header, make_header
+from .base import Connector, Field, register, feed
+
+@register
+class ImapEmailConnector(Connector):
+    id = "email"
+    name = "Email (IMAP)"
+    blurb = "Pull recent messages from any IMAP mailbox (Gmail, Outlook, self-hosted…)."
+    fields = [
+        Field("host", "IMAP host", default="imap.gmail.com"),
+        Field("port", "IMAP port", default="993"),
+        Field("user", "Username / email address"),
+        Field("password", "App password", secret=True),
+        Field("folder", "Folder", default="INBOX"),
+        Field("limit", "Max recent messages", default="200"),
+    ]
+
+    def _open(self, cfg):
+        M = imaplib.IMAP4_SSL(cfg["host"], int(cfg.get("port") or 993))
+        M.login(cfg["user"], cfg["password"])
+        return M
+
+    def test(self, cfg):
+        M = self._open(cfg); M.select(cfg.get("folder", "INBOX"), readonly=True); M.logout()
+        return "login ok"
+
+    def sync(self, cfg):
+        M = self._open(cfg)
+        M.select(cfg.get("folder", "INBOX"), readonly=True)
+        _, data = M.search(None, "ALL")
+        ids = data[0].split()[-int(cfg.get("limit") or 200):]
+        items = []
+        for i in ids:
+            _, d = M.fetch(i, "(RFC822)")
+            if not d or not isinstance(d[0], tuple):
+                continue
+            msg = email.message_from_bytes(d[0][1])
+            def H(h):
+                try: return str(make_header(decode_header(msg.get(h, "") or "")))
+                except Exception: return msg.get(h, "") or ""
+            body = ""
+            if msg.is_multipart():
+                for p in msg.walk():
+                    if p.get_content_type() == "text/plain":
+                        try: body = (p.get_payload(decode=True) or b"").decode("utf-8", "ignore"); break
+                        except Exception: pass
+            else:
+                try: body = (msg.get_payload(decode=True) or b"").decode("utf-8", "ignore")
+                except Exception: body = str(msg.get_payload())
+            items.append({"text": f"[email from {H('From')}] {H('Subject')}\n{body[:1500]}",
+                          "project": "email", "ref": H('From')[:80], "ts": H('Date'), "role": "email"})
+        M.logout()
+        return feed(items, "events")

connectors/vault.py

@@ -0,0 +1,51 @@
+# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE.
+"""Encrypted credential vault for connectors.
+
+App logins (IMAP passwords, WebDAV app-passwords, OAuth tokens) are stored here,
+encrypted at rest with a master passphrase (Fernet, PBKDF2-HMAC-SHA256, 390k iters).
+brain.db NEVER sees these — connectors read creds from the vault, pull data, and only
+scrubbed text reaches the brain. Set BRAIN_VAULT_PASS to run non-interactively.
+"""
+import os, json, base64, getpass
+from cryptography.fernet import Fernet
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+from cryptography.hazmat.primitives import hashes
+
+_HOME = os.environ.get("BRAIN_HOME", os.path.expanduser("~/.secondbrain"))
+VAULT = os.path.join(_HOME, "vault.enc")
+SALT_F = os.path.join(_HOME, "vault.salt")
+
+def _salt():
+    os.makedirs(_HOME, exist_ok=True)
+    if not os.path.exists(SALT_F):
+        s = os.urandom(16)
+        with open(SALT_F, "wb") as f:
+            f.write(s)
+        os.chmod(SALT_F, 0o600)
+        return s
+    with open(SALT_F, "rb") as f:
+        return f.read()
+
+def _key(passphrase, salt):
+    kdf = PBKDF2HMAC(algorithm=hashes.SHA256(), length=32, salt=salt, iterations=390000)
+    return base64.urlsafe_b64encode(kdf.derive(passphrase.encode()))
+
+def _pass(prompt):
+    return os.environ.get("BRAIN_VAULT_PASS") or getpass.getpass(prompt)
+
+def load(passphrase=None):
+    if not os.path.exists(VAULT):
+        return {}
+    passphrase = passphrase or _pass("Vault passphrase: ")
+    f = Fernet(_key(passphrase, _salt()))
+    with open(VAULT, "rb") as fh:
+        return json.loads(f.decrypt(fh.read()).decode())
+
+def save(data, passphrase=None):
+    passphrase = passphrase or _pass("Vault passphrase (set/confirm): ")
+    os.makedirs(_HOME, exist_ok=True)
+    f = Fernet(_key(passphrase, _salt()))
+    tok = f.encrypt(json.dumps(data).encode())
+    with open(VAULT, "wb") as fh:
+        fh.write(tok)
+    os.chmod(VAULT, 0o600)

connectors/webdav.py

@@ -0,0 +1,49 @@
+# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE.
+"""Connector: Nextcloud / any WebDAV — index text files from a cloud folder.
+For Nextcloud the base URL looks like:  https://cloud.example.com/remote.php/dav/files/USERNAME/
+Use a Nextcloud *app password*, not your login password."""
+import os
+import requests
+from xml.etree import ElementTree as ET
+from .base import Connector, Field, register, feed
+
+TEXT_EXT = (".md", ".txt", ".rst", ".py", ".sh", ".json", ".csv", ".tsv", ".yml", ".yaml",
+            ".ini", ".cfg", ".conf", ".html", ".js", ".ts", ".sql", ".xml")
+
+@register
+class WebDavConnector(Connector):
+    id = "webdav"
+    name = "Nextcloud / WebDAV files"
+    blurb = "Index text files from a Nextcloud (or any WebDAV) folder."
+    fields = [
+        Field("base_url", "WebDAV base URL (…/remote.php/dav/files/USER/)"),
+        Field("user", "Username"),
+        Field("password", "App password", secret=True),
+        Field("folder", "Folder path under base", default="/"),
+    ]
+
+    def _list(self, cfg):
+        url = cfg["base_url"].rstrip("/") + "/" + cfg.get("folder", "/").strip("/")
+        r = requests.request("PROPFIND", url, auth=(cfg["user"], cfg["password"]),
+                             headers={"Depth": "infinity"}, timeout=30)
+        r.raise_for_status()
+        hrefs = [e.text for e in ET.fromstring(r.content).iter("{DAV:}href")]
+        return [h for h in hrefs if h and h.lower().endswith(TEXT_EXT)]
+
+    def test(self, cfg):
+        return f"{len(self._list(cfg))} text files visible"
+
+    def sync(self, cfg):
+        host = cfg["base_url"].split("/remote.php")[0]
+        items = []
+        for href in self._list(cfg):
+            u = href if href.startswith("http") else host + href
+            try:
+                rr = requests.get(u, auth=(cfg["user"], cfg["password"]), timeout=30)
+                if rr.status_code != 200 or len(rr.content) > 400000:
+                    continue
+                items.append({"text": rr.text, "project": "nextcloud", "path": href,
+                              "ref": os.path.basename(href.rstrip("/")), "role": "file"})
+            except Exception:
+                continue
+        return feed(items, "file")

requirements.txt

@@ -1,3 +1,4 @@
 sqlite-vec>=0.1.6
 requests>=2.31
 numpy>=1.24
+cryptography>=41    # connectors credential vault (setup wizard); engine works without it

setup.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# SecondBrain — (c) 2026 Atlas Corporation. All Rights Reserved. Proprietary; see LICENSE.
+"""SecondBrain setup wizard — connect the apps that feed your brain.
+
+  python3 setup.py          interactive: pick apps, sign in, test, sync
+  python3 setup.py sync     re-sync every already-connected app (for cron/timer)
+  python3 setup.py list     show available connectors
+
+App credentials go into the encrypted vault (connectors/vault.py), never into brain.db.
+"""
+import sys, os
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from connectors import base, vault
+
+
+def _ask(field):
+    import getpass
+    label = f"  {field.label}" + (f" [{field.default}]" if field.default else "") + ": "
+    val = getpass.getpass(label) if field.secret else input(label)
+    return (val or "").strip() or field.default
+
+
+def cmd_list():
+    base.load_all()
+    print("\nAvailable connectors:")
+    for c in base.REGISTRY.values():
+        print(f"  • {c.name:26} {c.blurb}")
+
+
+def wizard():
+    base.load_all()
+    conns = list(base.REGISTRY.values())
+    print("\n🧠  SecondBrain — connect your apps\n")
+    for i, c in enumerate(conns, 1):
+        print(f"  {i}. {c.name} — {c.blurb}")
+    print("  q. done")
+    cfgs = vault.load() if os.path.exists(vault.VAULT) else {}
+    while True:
+        ch = input("\nConnect which? [number/q] ").strip().lower()
+        if ch in ("q", ""):
+            break
+        try:
+            c = conns[int(ch) - 1]
+        except Exception:
+            print("  ?"); continue
+        inst = c()
+        cfg = {f.key: _ask(f) for f in inst.fields}
+        try:
+            print("  testing…", inst.test(cfg))
+        except Exception as e:
+            print(f"  ✗ could not connect: {e}"); continue
+        cfgs[c.id] = cfg
+        vault.save(cfgs)
+        print(f"  ✓ {c.name} connected + saved to encrypted vault")
+        if input("  sync now? [Y/n] ").strip().lower() in ("", "y"):
+            try:
+                n = inst.sync(cfg); print(f"  … ingested {n} new chunks")
+            except Exception as e:
+                print(f"  sync error: {e}")
+    print("\nEmbedding new content (local)…")
+    try:
+        base._brain().embed(limit=20000)
+    except Exception as e:
+        print(f"  embed skipped: {e}")
+    print('Done.  Ask your brain:  python3 brain.py recall "…" --hybrid\n')
+
+
+def cmd_sync():
+    base.load_all()
+    cfgs = vault.load()
+    if not cfgs:
+        print("no connectors configured — run `python3 setup.py` first"); return
+    for cid, cfg in cfgs.items():
+        c = base.REGISTRY.get(cid)
+        if not c:
+            print(f"[{cid}] unknown connector, skipped"); continue
+        try:
+            n = c().sync(cfg); print(f"[{cid}] +{n} chunks")
+        except Exception as e:
+            print(f"[{cid}] error: {e}")
+    base._brain().embed(limit=50000)
+
+
+if __name__ == "__main__":
+    arg = sys.argv[1] if len(sys.argv) > 1 else ""
+    {"sync": cmd_sync, "list": cmd_list}.get(arg, wizard)()