From 8d3b49192d029342a3a0f815269f6803a78a3449 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 10 Apr 2026 21:48:08 +0800 Subject: [PATCH] =?UTF-8?q?feat(plugins):=20add=20mempalace-indexer=20?= =?UTF-8?q?=E2=80=94=20local=20semantic=20memory=20for=20agents=20(#43)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugins/mempalace-indexer/.gitignore | 1 + plugins/mempalace-indexer/README.md | 87 +++++ plugins/mempalace-indexer/hooks/after_turn.py | 333 ++++++++++++++++++ plugins/mempalace-indexer/hooks/ingest.py | 129 +++++++ plugins/mempalace-indexer/hooks/prune.py | 126 +++++++ plugins/mempalace-indexer/plugin.toml | 121 +++++++ plugins/mempalace-indexer/requirements.txt | 2 + .../tests/test_after_turn.py | 276 +++++++++++++++ .../mempalace-indexer/tests/test_ingest.py | 114 ++++++ plugins/mempalace-indexer/tests/test_prune.py | 116 ++++++ 10 files changed, 1305 insertions(+) create mode 100644 plugins/mempalace-indexer/.gitignore create mode 100644 plugins/mempalace-indexer/README.md create mode 100644 plugins/mempalace-indexer/hooks/after_turn.py create mode 100644 plugins/mempalace-indexer/hooks/ingest.py create mode 100644 plugins/mempalace-indexer/hooks/prune.py create mode 100644 plugins/mempalace-indexer/plugin.toml create mode 100644 plugins/mempalace-indexer/requirements.txt create mode 100644 plugins/mempalace-indexer/tests/test_after_turn.py create mode 100644 plugins/mempalace-indexer/tests/test_ingest.py create mode 100644 plugins/mempalace-indexer/tests/test_prune.py diff --git a/plugins/mempalace-indexer/.gitignore b/plugins/mempalace-indexer/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/plugins/mempalace-indexer/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/plugins/mempalace-indexer/README.md b/plugins/mempalace-indexer/README.md new file mode 100644 index 0000000..3af26cf --- /dev/null +++ b/plugins/mempalace-indexer/README.md @@ -0,0 +1,87 @@ +# mempalace-indexer + +LibreFang plugin for persistent, local semantic memory via [MemPalace](https://github.com/milla-jovovich/mempalace). No API keys, no cloud. + +## Quick start + +```bash +librefang plugin install mempalace-indexer +librefang plugin requirements mempalace-indexer + +mempalace init /path/to/workspace --yes +mempalace mine /path/to/workspace +``` + +Restart the daemon. Done. + +## Hooks + +| Hook | When | What | +|------|------|------| +| `ingest` | Message arrives | Searches palace for relevant memories, injects into context | +| `after_turn` | After LLM responds | Auto-saves memorable turns with dedup + classification | +| `prune` | On demand / scheduled | Deletes drawers older than `MEMPALACE_MAX_AGE_DAYS` | + +## How after_turn saves + +Five filters run before writing to the palace: + +1. **MCP dedup** — skip if the agent already called `mcp_mempalace_add_drawer` this turn +2. **Length** — skip exchanges under `MEMPALACE_MIN_CHARS` (default 80) +3. **Relevance** — English text must match keywords (decisions, appointments, contacts, etc.); non-English passes on length alone +4. **Content dedup** — skip if a near-identical turn was already saved (SHA-256 hash store) +5. **Noise** — code blocks are stripped; residual tool/error output is discarded + +Matched turns are classified and written to the appropriate room: + +| Content type | Wing | Room | +|---|---|---| +| Contacts, email addresses, family | `people` | `contacts` | +| Appointments, meetings, reminders | `time` | `calendar` | +| Payments, invoices, expenses | `finance` | `transactions` | +| Packages, shipments | `logistics` | `orders` | +| Decisions, preferences | `knowledge` | `decisions` | +| Everything else | `default` | `sessions` | + +A single turn can match multiple rooms and will be written to all of them. + +## Configuration + +All settings are optional environment variables: + +| Variable | Default | Description | +|---|---|---| +| `MEMPALACE_PALACE_PATH` | `~/.mempalace/palace` | Palace directory | +| `MEMPALACE_MIN_CHARS` | `80` | Minimum text length to save | +| `MEMPALACE_WINDOW_SIZE` | `6` | Recent messages to consider | +| `MEMPALACE_DEDUP_MAX` | `500` | Hash store rolling cap | +| `MEMPALACE_LANG_DETECT` | `1` | Set to `0` to disable language detection | +| `MEMPALACE_MAX_CHARS` | `300` | Max characters per injected memory snippet | +| `MEMPALACE_MIN_SIMILARITY` | `0.3` | Min similarity score for ingest results (0 = disabled) | +| `MEMPALACE_N_RESULTS` | `5` | Number of memories to inject per turn | +| `MEMPALACE_MAX_AGE_DAYS` | `90` | Prune drawers older than this (0 = disabled) | + +## MCP server (optional) + +Add 19 explicit memory tools to all agents: + +```toml +[[mcp_servers]] +name = "mempalace" +timeout_secs = 60 +[mcp_servers.transport] +type = "stdio" +command = "python3" +args = ["-m", "mempalace.mcp_server"] +``` + +## Pruning + +Run manually: + +```bash +python3 ~/.librefang/plugins/mempalace-indexer/hooks/prune.py --dry-run +python3 ~/.librefang/plugins/mempalace-indexer/hooks/prune.py +``` + +Or trigger via the LibreFang hook system on a schedule. diff --git a/plugins/mempalace-indexer/hooks/after_turn.py b/plugins/mempalace-indexer/hooks/after_turn.py new file mode 100644 index 0000000..99b52fe --- /dev/null +++ b/plugins/mempalace-indexer/hooks/after_turn.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +"""MemPalace after_turn hook for LibreFang. + +Filters conversation turns for relevant memories and saves them to MemPalace. +Skips: tool calls, short exchanges, noise, and turns where the agent already +used mcp_mempalace tools explicitly (deduplication). + +Input (stdin): {"type": "after_turn", "agent_id": "...", "messages": [...]} +Output (stdout): {"status": "..."} (fire-and-forget) + +Install: librefang plugin install mempalace-indexer && librefang plugin requirements mempalace-indexer +""" +import hashlib +import json +import os +import re +import sys +from datetime import datetime +from pathlib import Path + +try: + from langdetect import detect as _langdetect, LangDetectException + _LANGDETECT_AVAILABLE = True +except ImportError: + _LANGDETECT_AVAILABLE = False + +# --------------------------------------------------------------------------- +# Configuration: read from LIBREFANG_PLUGIN_CONFIG (written by the runtime), +# fall back to individual environment variables for direct invocation. +# --------------------------------------------------------------------------- + +def _load_config(): + cfg_path = os.environ.get("LIBREFANG_PLUGIN_CONFIG") + if cfg_path: + try: + with open(cfg_path) as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + pass + return {} + +_cfg = _load_config() + + +def _cfg_str(key, env_key, default): + if key in _cfg: + return str(_cfg[key]) + return os.environ.get(env_key, default) + + +def _cfg_int(key, env_key, default): + if key in _cfg: + try: + return int(_cfg[key]) + except (TypeError, ValueError): + pass + try: + return int(os.environ.get(env_key, str(default))) + except ValueError: + return default + + +def _cfg_bool(key, env_key, default): + if key in _cfg: + v = _cfg[key] + if isinstance(v, bool): + return v + return str(v).lower() not in ("0", "false", "no", "off") + raw = os.environ.get(env_key) + if raw is None: + return default + return raw != "0" + + +PALACE_PATH = _cfg_str("palace_path", "MEMPALACE_PALACE_PATH", + os.path.expanduser("~/.mempalace/palace")) +# Minimum character length of extracted text to be worth saving. +MIN_CONTENT_LENGTH = _cfg_int("min_chars", "MEMPALACE_MIN_CHARS", 80) +# How many recent messages to consider (sliding window). +WINDOW_SIZE = _cfg_int("window_size", "MEMPALACE_WINDOW_SIZE", 6) +# Max content hashes to keep in the dedup store (rolling, oldest dropped first). +DEDUP_MAX = _cfg_int("dedup_max", "MEMPALACE_DEDUP_MAX", 500) +# When langdetect is available, RELEVANCE_RE is only applied to English text. +# Non-English text passes on length + dedup alone. Set to false/0 to disable. +LANG_DETECT_ENABLED = _cfg_bool("lang_detect", "MEMPALACE_LANG_DETECT", True) + +# Room classification: all matching rules win (multi-room). +# Falls back to ("default", "sessions") when nothing matches. +ROOM_RULES: list[tuple[re.Pattern, tuple[str, str]]] = [ + ( + re.compile( + r"\b(contact|phone|email|address|family|wife|husband" + r"|son|daughter|parent|colleague|coworker)\b" + r"|\S+@\S+\.\w+", # bare email address pattern + re.IGNORECASE, + ), + ("people", "contacts"), + ), + ( + re.compile( + r"\b(appointment|deadline|birthday|event|meeting|schedule" + r"|remind me|reminder|calendar|due date|due on)\b", + re.IGNORECASE, + ), + ("time", "calendar"), + ), + ( + re.compile( + r"\b(budget|expense|transaction|payment|bill|salary|invoice" + r"|cost|price|paid|spending|refund)\b", + re.IGNORECASE, + ), + ("finance", "transactions"), + ), + ( + re.compile( + r"\b(package|order|shipment|delivery|tracking|shipped|arrived)\b", + re.IGNORECASE, + ), + ("logistics", "orders"), + ), + ( + re.compile( + r"\b(decision|decided|prefer|from now on|going forward" + r"|we.ll use|i.ll use|switching to|chosen|agreed)\b", + re.IGNORECASE, + ), + ("knowledge", "decisions"), + ), +] + +RELEVANCE_RE = re.compile( + r"\b(decision|decided|prefer|from now on|going forward|remember that|note that" + r"|remind me|don.t forget|important|urgent|critical|keep in mind" + r"|appointment|deadline|birthday|event|meeting|schedule|due date" + r"|budget|expense|transaction|payment|bill|salary|invoice|cost|price" + r"|package|order|shipment|delivery|tracking" + r"|contact|phone|email|address" + r"|like|dislike|preference|habit|allergy" + r"|family|wife|husband|son|daughter|parent|colleague" + r"|work|client|project|we.ll use|i.ll use|switching to)\b", + re.IGNORECASE, +) + +# Matches fenced code blocks — stripped from content before relevance checks. +CODE_BLOCK_RE = re.compile(r"```.*?```", re.DOTALL) + +# Residual noise patterns after code block stripping. +# Patterns are anchored or specific to avoid matching normal prose +# ("with the exception of", "no traceback available" in casual writing). +NOISE_RE = re.compile( + r"\[tool_call\]|\[tool_result\]|\"type\":\s*\"tool" + r"|Traceback \(most recent call last\)" # Python traceback header + r"|^\s*(Exception|Error|Warning):", # exception/error line starts + re.IGNORECASE | re.MULTILINE, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def emit(obj: dict) -> None: + json.dump(obj, sys.stdout) + sys.stdout.write("\n") + + +def _detect_language(text: str) -> str: + """Return ISO 639-1 language code, or 'unknown' on failure.""" + if not _LANGDETECT_AVAILABLE or not LANG_DETECT_ENABLED: + return "unknown" + try: + return _langdetect(text[:400]) + except Exception: + return "unknown" + + +def _is_english(lang: str) -> bool: + return lang in ("en", "unknown") + + +def _classify_rooms(text: str) -> list[tuple[str, str]]: + """Return all matching (wing, room) destinations. Falls back to default.""" + matches = [dest for pattern, dest in ROOM_RULES if pattern.search(text)] + return matches if matches else [("default", "sessions")] + + +def _strip_code_blocks(text: str) -> str: + """Replace fenced code blocks with a placeholder, preserving surrounding context.""" + return CODE_BLOCK_RE.sub("[code]", text).strip() + + +def _content_hash(text: str) -> str: + """SHA-256 of the first 500 chars — stable fingerprint for near-duplicate detection.""" + return hashlib.sha256(text[:500].encode()).hexdigest() + + +def _dedup_path(agent_id: str) -> Path: + # Per-agent store: prevents one agent's memories from blocking another's + # when multiple agents share the same palace. + safe_id = re.sub(r"[^\w-]", "_", agent_id)[:64] + return Path(PALACE_PATH) / f".after_turn_seen_{safe_id}.json" + + +def _load_seen(agent_id: str) -> list: + try: + return json.loads(_dedup_path(agent_id).read_text()) + except (FileNotFoundError, json.JSONDecodeError): + return [] + + +def _save_seen(agent_id: str, hashes: list) -> None: + path = _dedup_path(agent_id) + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(hashes[-DEDUP_MAX:])) + except OSError: + pass # dedup is best-effort; don't block indexing + + +def _is_duplicate(text: str, agent_id: str) -> bool: + h = _content_hash(text) + seen = _load_seen(agent_id) + if h in seen: + return True + seen.append(h) + _save_seen(agent_id, seen) + return False + + +def extract_text(messages): # list[dict] -> tuple[str, bool] + """Extract user+assistant text; detect if agent already saved to mempalace.""" + recent = messages[-WINDOW_SIZE:] + parts = [] + agent_used_mempalace = False + + for msg in recent: + role = msg.get("role", "") + content = msg.get("content") or "" + + if role in ("tool", "assistant") and "mcp_mempalace" in str(content): + agent_used_mempalace = True + + if role not in ("user", "assistant"): + continue + + if isinstance(content, list): + content = "\n".join( + b.get("text", "") for b in content + if isinstance(b, dict) and b.get("type") == "text" + ) + + if not content: + continue + + content = _strip_code_blocks(content) + + if not content or NOISE_RE.search(content): + continue + + parts.append(f"[{role}] {content}") + + return "\n".join(parts), agent_used_mempalace + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def main() -> None: + try: + data = json.load(sys.stdin) + except (json.JSONDecodeError, EOFError): + emit({"status": "skip", "reason": "bad input"}) + return + + messages = data.get("messages", []) + agent_id = data.get("agent_id", "unknown") + + if not messages: + emit({"status": "skip", "reason": "no messages"}) + return + + text, already_saved = extract_text(messages) + + if already_saved: + emit({"status": "skip", "reason": "agent used mcp_mempalace"}) + return + + if len(text) < MIN_CONTENT_LENGTH: + emit({"status": "skip", "reason": "too short"}) + return + + lang = _detect_language(text) + # RELEVANCE_RE is English-only; skip it for non-English to avoid false negatives. + if _is_english(lang) and not RELEVANCE_RE.search(text): + emit({"status": "skip", "reason": "not relevant"}) + return + + if _is_duplicate(text, agent_id): + emit({"status": "skip", "reason": "duplicate"}) + return + + try: + from mempalace.miner import get_collection, add_drawer + + collection = get_collection(PALACE_PATH) + source = f"auto-{agent_id}-{datetime.now().strftime('%Y%m%d-%H%M%S%f')}" + rooms = _classify_rooms(text) + + for wing, room in rooms: + add_drawer( + collection=collection, + wing=wing, + room=room, + content=text, + source_file=source, + chunk_index=0, + agent="mempalace-indexer", + ) + + emit({ + "status": "indexed", + "chars": len(text), + "lang": lang, + "rooms": [{"wing": w, "room": r} for w, r in rooms], + }) + except Exception as e: + emit({"status": "error", "error": str(e)}) + + +if __name__ == "__main__": + main() diff --git a/plugins/mempalace-indexer/hooks/ingest.py b/plugins/mempalace-indexer/hooks/ingest.py new file mode 100644 index 0000000..96a6311 --- /dev/null +++ b/plugins/mempalace-indexer/hooks/ingest.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""MemPalace ingest hook for LibreFang. + +Searches MemPalace for memories relevant to the incoming user message +and injects them into the agent's context as MemoryFragments. + +Input (stdin): {"type": "ingest", "agent_id": "...", "message": "user text"} +Output (stdout): {"memories": [{"content": "..."}]} + +Install: librefang plugin install mempalace-indexer && librefang plugin requirements mempalace-indexer +""" +import sys +import json +import os + +# --------------------------------------------------------------------------- +# Configuration: read from LIBREFANG_PLUGIN_CONFIG (written by the runtime), +# fall back to individual environment variables for direct invocation. +# --------------------------------------------------------------------------- + +def _load_config(): + cfg_path = os.environ.get("LIBREFANG_PLUGIN_CONFIG") + if cfg_path: + try: + with open(cfg_path) as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + pass + return {} + +_cfg = _load_config() + + +def _cfg_str(key, env_key, default): + if key in _cfg: + return str(_cfg[key]) + return os.environ.get(env_key, default) + + +def _cfg_int(key, env_key, default): + if key in _cfg: + try: + return int(_cfg[key]) + except (TypeError, ValueError): + pass + return int(os.environ.get(env_key, default)) + + +def _cfg_float(key, env_key, default): + if key in _cfg: + try: + return float(_cfg[key]) + except (TypeError, ValueError): + pass + return float(os.environ.get(env_key, default)) + + +PALACE_PATH = _cfg_str("palace_path", "MEMPALACE_PALACE_PATH", + os.path.expanduser("~/.mempalace/palace")) +MAX_MEMORY_CHARS = _cfg_int("max_chars", "MEMPALACE_MAX_CHARS", "300") + +# MemPalace returns similarity in [0, 1] — higher means more relevant. +# Results below MIN_SIMILARITY are too dissimilar to be useful. +# Set to 0 (or MEMPALACE_MIN_SIMILARITY=0) to disable filtering. +MIN_SIMILARITY = _cfg_float("min_similarity", "MEMPALACE_MIN_SIMILARITY", "0.3") +N_RESULTS = _cfg_int("n_results", "MEMPALACE_N_RESULTS", "5") + + +def emit(obj): + """Write JSON response to stdout with trailing newline.""" + json.dump(obj, sys.stdout) + sys.stdout.write("\n") + + +def truncate_at_word(text, max_len): + """Truncate text at nearest word boundary.""" + if len(text) <= max_len: + return text + truncated = text[:max_len] + last_space = truncated.rfind(" ") + if last_space > max_len // 2: + return truncated[:last_space] + "..." + return truncated + "..." + + +def main(): + try: + data = json.load(sys.stdin) + except (json.JSONDecodeError, EOFError): + emit({"memories": []}) + return + + message = data.get("message", "") + if not message or len(message) < 5: + emit({"memories": []}) + return + + try: + from mempalace.searcher import search_memories + + results = search_memories(message, PALACE_PATH, n_results=N_RESULTS) + + memories = [] + for r in results.get("results", []): + text = r.get("text", "") + wing = r.get("wing", "") or "memory" + room = r.get("room", "") + similarity = r.get("similarity") + + if not text: + continue + + # Filter out low-relevance results when the backend provides a score. + # similarity=None means the backend didn't return one — allow through. + if similarity is not None and MIN_SIMILARITY > 0 and similarity < MIN_SIMILARITY: + continue + + snippet = truncate_at_word(text, MAX_MEMORY_CHARS) + # Use wing/room (semantically meaningful) instead of raw source filename. + label = f"{wing}/{room}" if room else wing + memories.append({"content": f"[{label}] {snippet}"}) + + emit({"memories": memories}) + except Exception as e: + emit({"memories": [], "error": str(e)}) + + +if __name__ == "__main__": + main() diff --git a/plugins/mempalace-indexer/hooks/prune.py b/plugins/mempalace-indexer/hooks/prune.py new file mode 100644 index 0000000..cc1a209 --- /dev/null +++ b/plugins/mempalace-indexer/hooks/prune.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""MemPalace pruner for LibreFang. + +Deletes drawers older than MEMPALACE_MAX_AGE_DAYS from the palace. +Intended to be run periodically (e.g. via a LibreFang scheduled hook or cron). + +NOTE: This script reads ChromaDB directly using the collection name +("mempalace_drawers") and metadata field ("filed_at") that MemPalace uses +internally. If a future MemPalace release changes these, update accordingly. + +Usage: + python3 prune.py [--dry-run] + +Input (stdin): {"type": "prune", "agent_id": "..."} (optional, for hook mode) +Output (stdout): {"status": "...", "deleted": N, "kept": N} + +Environment: + MEMPALACE_PALACE_PATH Path to the palace directory (default: ~/.mempalace/palace) + MEMPALACE_MAX_AGE_DAYS Delete drawers older than this many days (default: 90, 0 = disabled) +""" +import json +import os +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +PALACE_PATH = os.environ.get( + "MEMPALACE_PALACE_PATH", + os.path.expanduser("~/.mempalace/palace"), +) +MAX_AGE_DAYS = int(os.environ.get("MEMPALACE_MAX_AGE_DAYS", "90")) + +# MemPalace internal constants — update if upstream changes them. +_COLLECTION_NAME = "mempalace_drawers" +_FILED_AT_FIELD = "filed_at" + + +def emit(obj: dict) -> None: + json.dump(obj, sys.stdout) + sys.stdout.write("\n") + + +def _parse_filed_at(value: str): # -> datetime | None + """Parse MemPalace's ISO timestamp into a timezone-aware datetime.""" + if not value: + return None + try: + dt = datetime.fromisoformat(value) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + except ValueError: + return None + + +def prune(dry_run: bool = False) -> dict: + if MAX_AGE_DAYS <= 0: + return {"status": "skip", "reason": "MEMPALACE_MAX_AGE_DAYS=0 (disabled)"} + + cutoff = datetime.now(tz=timezone.utc) - timedelta(days=MAX_AGE_DAYS) + + try: + import chromadb + except ImportError: + return {"status": "error", "error": "chromadb not installed (install mempalace first)"} + + palace = Path(PALACE_PATH) + if not palace.exists(): + return {"status": "error", "error": f"Palace path not found: {PALACE_PATH}"} + + try: + client = chromadb.PersistentClient(path=str(palace)) + collection = client.get_collection(_COLLECTION_NAME) + except Exception as e: + return {"status": "error", "error": f"Could not open collection '{_COLLECTION_NAME}': {e}"} + + # Fetch all drawers. For large palaces this is O(n) in memory — acceptable + # for a personal-scale deployment (tens of thousands of items at most). + try: + result = collection.get(include=["metadatas"]) + except Exception as e: + return {"status": "error", "error": f"collection.get() failed: {e}"} + + ids = result.get("ids", []) + metadatas = result.get("metadatas", []) + + to_delete = [] + kept = 0 + + for doc_id, meta in zip(ids, metadatas): + filed_at = _parse_filed_at((meta or {}).get(_FILED_AT_FIELD, "")) + if filed_at is not None and filed_at < cutoff: + to_delete.append(doc_id) + else: + kept += 1 + + if to_delete and not dry_run: + try: + collection.delete(ids=to_delete) + except Exception as e: + return {"status": "error", "error": f"delete failed: {e}", "attempted": len(to_delete)} + + return { + "status": "dry-run" if dry_run else "ok", + "deleted": len(to_delete), + "kept": kept, + "cutoff": cutoff.isoformat(), + "max_age_days": MAX_AGE_DAYS, + } + + +def main() -> None: + dry_run = "--dry-run" in sys.argv + + # Also accept hook-style JSON input (stdin), ignoring the payload content. + if not sys.stdin.isatty(): + try: + json.load(sys.stdin) + except (json.JSONDecodeError, EOFError): + pass + + emit(prune(dry_run=dry_run)) + + +if __name__ == "__main__": + main() diff --git a/plugins/mempalace-indexer/plugin.toml b/plugins/mempalace-indexer/plugin.toml new file mode 100644 index 0000000..2347cc3 --- /dev/null +++ b/plugins/mempalace-indexer/plugin.toml @@ -0,0 +1,121 @@ +name = "mempalace-indexer" +version = "0.3.0" +description = "Auto-index conversations into MemPalace and recall relevant memories. No API keys, no cloud." +author = "LibreFang Community" +requirements = "requirements.txt" +librefang_min_version = "2026.4.0" + +# --------------------------------------------------------------------------- +# Hook execution settings +# +# MUST sit above [hooks] so they land on the top-level table instead of being +# folded into [hooks] by TOML table scoping rules (validate.py iterates +# data["hooks"].items() expecting every value to be a hook file path). +# --------------------------------------------------------------------------- + +# ingest is fast (vector search) — 30 s is generous. +# after_turn does classification + write — 60 s covers a slow machine or large turn. +hook_timeout_secs = 30 + +# A temporary mempalace outage should not block the agent turn. +on_hook_failure = "warn" + +# One silent retry handles transient ChromaDB lock contention. +max_retries = 1 +retry_delay_ms = 500 + +# Python interpreter startup is ~200 ms. Keep subprocesses alive between calls +# to eliminate that overhead on every user message. +persistent_subprocess = true + +# ingest results are deterministic for the same message text — cache for 60 s +# to avoid redundant embedding lookups on rapid follow-up turns. +hook_cache_ttl_secs = 60 + +# Priority 0 (default). Raise to e.g. 10 to run before other recall plugins. +priority = 0 + +# --------------------------------------------------------------------------- +# System binary requirements +# --------------------------------------------------------------------------- + +[[requires]] +binary = "python3" +install_hint = "Install Python 3.9+ from https://python.org or via your system package manager (apt install python3 / brew install python)" + +# --------------------------------------------------------------------------- +# Hook scripts +# +# Every value in this section MUST be a path relative to the plugin directory. +# Execution settings live above at the top level. +# --------------------------------------------------------------------------- + +[hooks] +ingest = "hooks/ingest.py" +after_turn = "hooks/after_turn.py" + +# --------------------------------------------------------------------------- +# [config] — user-configurable settings +# +# The runtime merges user overrides with these defaults and writes the result +# as JSON to the path in LIBREFANG_PLUGIN_CONFIG before each hook subprocess. +# Hooks should read that file (falling back to environment variables for +# backward compatibility with direct invocation). +# --------------------------------------------------------------------------- + +[config.palace_path] +type = "string" +default = "~/.mempalace/palace" +description = "Path to the MemPalace directory (passed to mempalace APIs and ChromaDB)." + +[config.min_chars] +type = "number" +default = 80 +description = "Minimum character length of extracted conversation text to be worth saving. Shorter exchanges are skipped by after_turn." + +[config.window_size] +type = "number" +default = 6 +description = "Number of recent messages the after_turn hook considers when extracting text (sliding window from the end of the conversation)." + +[config.dedup_max] +type = "number" +default = 500 +description = "Maximum number of SHA-256 content hashes kept in the per-agent deduplication store. Oldest entries are dropped first when the cap is reached." + +[config.lang_detect] +type = "boolean" +default = true +description = "Enable language detection via langdetect. When true, non-English exchanges bypass the English keyword relevance filter and are indexed on length alone." + +[config.max_chars] +type = "number" +default = 300 +description = "Maximum characters per injected memory snippet in the ingest hook. Longer texts are truncated at the nearest word boundary." + +[config.min_similarity] +type = "number" +default = 0.3 +description = "Minimum similarity score (0-1, higher = more relevant) for memories returned by the ingest hook. Results below this threshold are dropped. Set to 0 to disable filtering." + +[config.n_results] +type = "number" +default = 5 +description = "Number of memories the ingest hook retrieves from MemPalace per turn." + +[config.max_age_days] +type = "number" +default = 90 +description = "Prune drawers older than this many days when running hooks/prune.py. Set to 0 to disable TTL pruning." + +# --------------------------------------------------------------------------- +# [integrity] — SHA-256 hashes verified at load time +# +# Regenerate after editing hook files: +# sha256sum hooks/ingest.py hooks/after_turn.py hooks/prune.py +# --------------------------------------------------------------------------- + +[integrity] +"hooks/ingest.py" = "621b17f2d0afbd6142a49963fb57187a500cfeac057c13cb80961b6222bd03e2" +"hooks/after_turn.py" = "1c5bcdade4ca0d785878d60910105208fcff38f74872426ee821363ce5bf9811" +"hooks/prune.py" = "2dceeb9bb0d0b7ee224291bfce600f629f38b4a853cd437774d9b66ef226838d" diff --git a/plugins/mempalace-indexer/requirements.txt b/plugins/mempalace-indexer/requirements.txt new file mode 100644 index 0000000..8ec50bc --- /dev/null +++ b/plugins/mempalace-indexer/requirements.txt @@ -0,0 +1,2 @@ +mempalace>=3.0.0,<4 +langdetect>=1.0.9,<2 diff --git a/plugins/mempalace-indexer/tests/test_after_turn.py b/plugins/mempalace-indexer/tests/test_after_turn.py new file mode 100644 index 0000000..7b037be --- /dev/null +++ b/plugins/mempalace-indexer/tests/test_after_turn.py @@ -0,0 +1,276 @@ +"""Tests for the after_turn hook (stdin/stdout interface).""" +import importlib.util +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +HOOK = Path(__file__).parent.parent / "hooks" / "after_turn.py" + + +def run_hook(payload: dict, env=None) -> dict: + import os + run_env = os.environ.copy() + if env: + run_env.update(env) + result = subprocess.run( + [sys.executable, str(HOOK)], + input=json.dumps(payload), + capture_output=True, + text=True, + env=run_env, + ) + return json.loads(result.stdout.strip()) + + +def _load_module(env_overrides=None): + import os + saved = {} + if env_overrides: + for k, v in env_overrides.items(): + saved[k] = os.environ.get(k) + os.environ[k] = v + spec = importlib.util.spec_from_file_location("after_turn", HOOK) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + return mod + + +_mod = _load_module() + + +# --------------------------------------------------------------------------- +# Skip cases +# --------------------------------------------------------------------------- + +def test_empty_messages_skipped(): + out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": []}) + assert out["status"] == "skip" + assert out["reason"] == "no messages" + + +def test_bad_json_skipped(): + result = subprocess.run( + [sys.executable, str(HOOK)], + input="not json", + capture_output=True, + text=True, + ) + out = json.loads(result.stdout.strip()) + assert out["status"] == "skip" + assert out["reason"] == "bad input" + + +def test_short_exchange_skipped(): + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] + out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages}) + assert out["status"] == "skip" + assert out["reason"] == "too short" + + +def test_irrelevant_exchange_skipped(): + messages = [ + {"role": "user", "content": "What is the capital of France and why is it historically significant?"}, + {"role": "assistant", "content": "Paris has been the capital since the 10th century and is the cultural hub."}, + ] + out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages}) + assert out["status"] == "skip" + assert out["reason"] == "not relevant" + + +def test_agent_mempalace_tool_call_skipped(): + messages = [ + {"role": "user", "content": "Save this to my memory please."}, + {"role": "tool", "content": '{"tool": "mcp_mempalace_add_drawer", "result": "ok"}'}, + {"role": "assistant", "content": "Done, I saved it using mcp_mempalace."}, + ] + out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages}) + assert out["status"] == "skip" + assert out["reason"] == "agent used mcp_mempalace" + + +def test_user_mentioning_mempalace_not_skipped(): + """User saying 'mcp_mempalace' should NOT trigger the dedup skip.""" + messages = [ + {"role": "user", "content": "Can you use mcp_mempalace to save my dentist appointment on April 15th?"}, + {"role": "assistant", "content": "I will remember your dentist appointment on April 15th."}, + ] + out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages}) + assert out.get("reason") != "agent used mcp_mempalace" + + +# --------------------------------------------------------------------------- +# Code block stripping +# --------------------------------------------------------------------------- + +def test_noise_re_does_not_match_common_prose(): + """'exception' and 'traceback' as normal words must not trigger noise filter.""" + assert not _mod.NOISE_RE.search("There's no exception to this rule.") + assert not _mod.NOISE_RE.search("Let me traceback the history of this decision.") + + +def test_noise_re_matches_python_traceback(): + assert _mod.NOISE_RE.search("Traceback (most recent call last):\n File foo.py") + + +def test_noise_re_matches_error_line(): + assert _mod.NOISE_RE.search("Exception: something went wrong") + assert _mod.NOISE_RE.search("Error: connection refused") + + +def test_code_block_stripped_not_whole_message_skipped(): + """A message with a code block but meaningful surrounding text should not be skipped.""" + text = "My email is alice@example.com. Here is the script:\n```bash\necho hello\n```\nRun it daily." + result = _mod._strip_code_blocks(text) + assert "alice@example.com" in result + assert "```" not in result + assert "[code]" in result + + +def test_pure_code_block_skipped_after_strip(): + """A message that is only a code block becomes empty after stripping → too short → skip.""" + messages = [ + {"role": "user", "content": "here's the script"}, + {"role": "assistant", "content": "```python\nfor i in range(10):\n print(i)\n```"}, + ] + out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages}) + # Should be skipped due to length or irrelevance after stripping + assert out["status"] == "skip" + + +# --------------------------------------------------------------------------- +# Multi-room classification +# --------------------------------------------------------------------------- + +def test_classify_single_room(): + assert _mod._classify_rooms("Her email is alice@example.com") == [("people", "contacts")] + + +def test_classify_multiple_rooms(): + # "meeting" → calendar, "email" → contacts — both should match + rooms = _mod._classify_rooms("Schedule a meeting with alice@example.com next Tuesday") + assert ("people", "contacts") in rooms + assert ("time", "calendar") in rooms + + +def test_classify_finance(): + assert _mod._classify_rooms("Invoice for $500 is due next week") == [("finance", "transactions")] + + +def test_classify_logistics(): + assert _mod._classify_rooms("The shipment arrived today") == [("logistics", "orders")] + + +def test_classify_decisions(): + assert _mod._classify_rooms("We decided to use Postgres going forward") == [("knowledge", "decisions")] + + +def test_classify_default_fallback(): + assert _mod._classify_rooms("The sky is blue and the grass is green.") == [("default", "sessions")] + + +# --------------------------------------------------------------------------- +# Content deduplication +# --------------------------------------------------------------------------- + +def test_duplicate_detection(): + with tempfile.TemporaryDirectory() as tmpdir: + mod = _load_module({"MEMPALACE_PALACE_PATH": tmpdir}) + text = "I have a dentist appointment on April 15th with Dr. Smith." + assert mod._is_duplicate(text, "agent-1") is False # first time: not a duplicate + assert mod._is_duplicate(text, "agent-1") is True # second time: duplicate + + +def test_different_content_not_duplicate(): + with tempfile.TemporaryDirectory() as tmpdir: + mod = _load_module({"MEMPALACE_PALACE_PATH": tmpdir}) + assert mod._is_duplicate("dentist appointment April 15th", "agent-1") is False + assert mod._is_duplicate("meeting with Bob on Friday afternoon", "agent-1") is False + + +def test_dedup_is_per_agent(): + """Two different agents should have independent dedup stores.""" + with tempfile.TemporaryDirectory() as tmpdir: + mod = _load_module({"MEMPALACE_PALACE_PATH": tmpdir}) + text = "I have a dentist appointment on April 15th with Dr. Smith." + mod._is_duplicate(text, "agent-1") # agent-1 saves it + assert mod._is_duplicate(text, "agent-2") is False # agent-2 has not seen it + + +def test_dedup_rolling_max(): + with tempfile.TemporaryDirectory() as tmpdir: + mod = _load_module({"MEMPALACE_PALACE_PATH": tmpdir, "MEMPALACE_DEDUP_MAX": "3"}) + for i in range(5): + mod._is_duplicate(f"unique content number {i} with enough chars to matter", "agent-1") + seen = json.loads((Path(tmpdir) / ".after_turn_seen_agent-1.json").read_text()) + assert len(seen) == 3 # capped at DEDUP_MAX + + +# --------------------------------------------------------------------------- +# Configurable parameters +# --------------------------------------------------------------------------- + +def test_custom_min_chars_env(): + mod = _load_module({"MEMPALACE_MIN_CHARS": "200"}) + assert mod.MIN_CONTENT_LENGTH == 200 + + +def test_custom_window_size_env(): + mod = _load_module({"MEMPALACE_WINDOW_SIZE": "10"}) + assert mod.WINDOW_SIZE == 10 + + +# --------------------------------------------------------------------------- +# Language detection +# --------------------------------------------------------------------------- + +def test_detect_language_returns_string(): + lang = _mod._detect_language("I have a meeting with the client tomorrow afternoon.") + assert isinstance(lang, str) + assert len(lang) > 0 + + +def test_is_english_known_english(): + assert _mod._is_english("en") is True + + +def test_is_english_unknown_treated_as_english(): + # unknown = langdetect unavailable or failed → don't block on keyword check + assert _mod._is_english("unknown") is True + + +def test_is_english_other_language(): + assert _mod._is_english("zh") is False + assert _mod._is_english("ja") is False + assert _mod._is_english("fr") is False + + +def test_lang_detect_disabled_returns_unknown(): + mod = _load_module({"MEMPALACE_LANG_DETECT": "0"}) + lang = mod._detect_language("Ich habe morgen einen Termin beim Zahnarzt.") + assert lang == "unknown" + + +def test_non_english_skips_relevance_check(): + """A non-English exchange long enough to pass length check should not be + blocked by the English-only RELEVANCE_RE.""" + # This text is in Chinese and contains no English keywords from RELEVANCE_RE, + # so without language detection it would be skipped as "not relevant". + messages = [ + {"role": "user", + "content": "我明天下午三点有个牙医预约，在城市医院，请帮我记住这件事。"}, + {"role": "assistant", + "content": "好的，我已经记住了您明天下午三点在城市医院的牙医预约。"}, + ] + out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages}) + # Should NOT be skipped for "not relevant" — may fail at mempalace import (error) or dedup + assert out.get("reason") != "not relevant" diff --git a/plugins/mempalace-indexer/tests/test_ingest.py b/plugins/mempalace-indexer/tests/test_ingest.py new file mode 100644 index 0000000..d6d18de --- /dev/null +++ b/plugins/mempalace-indexer/tests/test_ingest.py @@ -0,0 +1,114 @@ +"""Tests for the ingest hook (stdin/stdout interface).""" +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +HOOK = Path(__file__).parent.parent / "hooks" / "ingest.py" + + +def run_hook(payload: dict) -> dict: + result = subprocess.run( + [sys.executable, str(HOOK)], + input=json.dumps(payload), + capture_output=True, + text=True, + ) + return json.loads(result.stdout.strip()) + + +def test_bad_json_returns_empty(): + result = subprocess.run( + [sys.executable, str(HOOK)], + input="not json", + capture_output=True, + text=True, + ) + out = json.loads(result.stdout.strip()) + assert out == {"memories": []} + + +def test_empty_message_returns_empty(): + out = run_hook({"type": "ingest", "agent_id": "a1", "message": ""}) + assert out == {"memories": []} + + +def test_short_message_returns_empty(): + out = run_hook({"type": "ingest", "agent_id": "a1", "message": "hi"}) + assert out == {"memories": []} + + +def test_no_mempalace_returns_error_not_crash(): + """Without mempalace installed, ingest returns error field but doesn't crash.""" + out = run_hook({"type": "ingest", "agent_id": "a1", "message": "What are my upcoming meetings?"}) + assert "memories" in out + assert isinstance(out["memories"], list) + assert "error" in out + + +# --------------------------------------------------------------------------- +# Similarity filtering (unit-level, no mempalace needed) +# --------------------------------------------------------------------------- + +def _load_ingest_module(min_similarity="0.3"): + saved = os.environ.get("MEMPALACE_MIN_SIMILARITY") + os.environ["MEMPALACE_MIN_SIMILARITY"] = min_similarity + spec = importlib.util.spec_from_file_location("ingest", HOOK) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + if saved is None: + os.environ.pop("MEMPALACE_MIN_SIMILARITY", None) + else: + os.environ["MEMPALACE_MIN_SIMILARITY"] = saved + return mod + + +def test_truncate_at_word_boundary(): + mod = _load_ingest_module() + text = "one two three four five six seven" + result = mod.truncate_at_word(text, 15) + assert result.endswith("...") + assert len(result) <= 18 + + +def test_truncate_short_unchanged(): + mod = _load_ingest_module() + assert mod.truncate_at_word("hello", 100) == "hello" + + +def test_similarity_threshold_filters_results(): + """Simulate the similarity filtering logic directly.""" + mod = _load_ingest_module(min_similarity="0.5") + + results = [ + {"text": "good match", "source_file": "s1", "wing": "w1", "similarity": 0.8}, + {"text": "bad match", "source_file": "s2", "wing": "w1", "similarity": 0.2}, + {"text": "no score", "source_file": "s3", "wing": "w1"}, + ] + + MIN_SIMILARITY = mod.MIN_SIMILARITY + memories = [] + for r in results: + text = r.get("text", "") + similarity = r.get("similarity") + if not text: + continue + if similarity is not None and MIN_SIMILARITY > 0 and similarity < MIN_SIMILARITY: + continue + memories.append(text) + + assert "good match" in memories + assert "bad match" not in memories + assert "no score" in memories # passthrough when similarity is absent + + +def test_similarity_zero_disables_filtering(): + mod = _load_ingest_module(min_similarity="0") + assert mod.MIN_SIMILARITY == 0.0 + + +def test_min_similarity_default(): + mod = _load_ingest_module() + assert mod.MIN_SIMILARITY == 0.3 diff --git a/plugins/mempalace-indexer/tests/test_prune.py b/plugins/mempalace-indexer/tests/test_prune.py new file mode 100644 index 0000000..90c5ebf --- /dev/null +++ b/plugins/mempalace-indexer/tests/test_prune.py @@ -0,0 +1,116 @@ +"""Tests for the prune hook.""" +import importlib.util +import json +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path + +HOOK = Path(__file__).parent.parent / "hooks" / "prune.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("prune", HOOK) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_mod = _load() + + +def test_disabled_when_max_age_zero(monkeypatch): + monkeypatch.setattr(_mod, "MAX_AGE_DAYS", 0) + result = _mod.prune() + assert result["status"] == "skip" + + +def test_error_when_palace_missing(monkeypatch, tmp_path): + try: + import chromadb # noqa: F401 + except ImportError: + import pytest + pytest.skip("chromadb not installed") + monkeypatch.setattr(_mod, "PALACE_PATH", str(tmp_path / "nonexistent")) + monkeypatch.setattr(_mod, "MAX_AGE_DAYS", 30) + result = _mod.prune() + assert result["status"] == "error" + assert "not found" in result["error"] + + +def test_parse_filed_at_valid(): + dt = _mod._parse_filed_at("2026-01-01T10:00:00") + assert dt is not None + assert dt.year == 2026 + + +def test_parse_filed_at_empty(): + assert _mod._parse_filed_at("") is None + + +def test_parse_filed_at_invalid(): + assert _mod._parse_filed_at("not-a-date") is None + + +def test_parse_filed_at_timezone_aware(): + dt = _mod._parse_filed_at("2026-01-01T10:00:00+05:00") + assert dt.tzinfo is not None + + +def test_prune_dry_run_with_chromadb(monkeypatch, tmp_path): + """Integration test: create a real ChromaDB collection and prune old entries.""" + try: + import chromadb + except ImportError: + import pytest + pytest.skip("chromadb not installed") + + monkeypatch.setattr(_mod, "PALACE_PATH", str(tmp_path)) + monkeypatch.setattr(_mod, "MAX_AGE_DAYS", 30) + + client = chromadb.PersistentClient(path=str(tmp_path)) + col = client.get_or_create_collection("mempalace_drawers") + + old_date = (datetime.now(tz=timezone.utc) - timedelta(days=60)).isoformat() + new_date = datetime.now(tz=timezone.utc).isoformat() + + col.add( + ids=["old-1", "new-1"], + documents=["old memory", "new memory"], + metadatas=[ + {"filed_at": old_date, "wing": "default", "room": "sessions"}, + {"filed_at": new_date, "wing": "default", "room": "sessions"}, + ], + ) + + result = _mod.prune(dry_run=True) + assert result["status"] == "dry-run" + assert result["deleted"] == 1 + assert result["kept"] == 1 + # dry-run: nothing actually deleted + assert col.count() == 2 + + +def test_prune_actually_deletes(monkeypatch, tmp_path): + try: + import chromadb + except ImportError: + import pytest + pytest.skip("chromadb not installed") + + monkeypatch.setattr(_mod, "PALACE_PATH", str(tmp_path)) + monkeypatch.setattr(_mod, "MAX_AGE_DAYS", 30) + + client = chromadb.PersistentClient(path=str(tmp_path)) + col = client.get_or_create_collection("mempalace_drawers") + + old_date = (datetime.now(tz=timezone.utc) - timedelta(days=60)).isoformat() + col.add( + ids=["old-2"], + documents=["stale memory"], + metadatas=[{"filed_at": old_date}], + ) + + result = _mod.prune(dry_run=False) + assert result["status"] == "ok" + assert result["deleted"] == 1 + assert col.count() == 0