feat(plugins): add mempalace-indexer — local semantic memory for agents (#43)
This commit is contained in:
10 files changed
+1305
No files matched your search
@@ -0,0 +1 @@
|
||||
*.pyc
|
||||
@@ -0,0 +1,87 @@
|
||||
# mempalace-indexer
|
||||
|
||||
LibreFang plugin for persistent, local semantic memory via [MemPalace](https://github.com/milla-jovovich/mempalace). No API keys, no cloud.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
librefang plugin install mempalace-indexer
|
||||
librefang plugin requirements mempalace-indexer
|
||||
|
||||
mempalace init /path/to/workspace --yes
|
||||
mempalace mine /path/to/workspace
|
||||
```
|
||||
|
||||
Restart the daemon. Done.
|
||||
|
||||
## Hooks
|
||||
|
||||
| Hook | When | What |
|
||||
|------|------|------|
|
||||
| `ingest` | Message arrives | Searches palace for relevant memories, injects into context |
|
||||
| `after_turn` | After LLM responds | Auto-saves memorable turns with dedup + classification |
|
||||
| `prune` | On demand / scheduled | Deletes drawers older than `MEMPALACE_MAX_AGE_DAYS` |
|
||||
|
||||
## How after_turn saves
|
||||
|
||||
Five filters run before writing to the palace:
|
||||
|
||||
1. **MCP dedup** — skip if the agent already called `mcp_mempalace_add_drawer` this turn
|
||||
2. **Length** — skip exchanges under `MEMPALACE_MIN_CHARS` (default 80)
|
||||
3. **Relevance** — English text must match keywords (decisions, appointments, contacts, etc.); non-English passes on length alone
|
||||
4. **Content dedup** — skip if a near-identical turn was already saved (SHA-256 hash store)
|
||||
5. **Noise** — code blocks are stripped; residual tool/error output is discarded
|
||||
|
||||
Matched turns are classified and written to the appropriate room:
|
||||
|
||||
| Content type | Wing | Room |
|
||||
|---|---|---|
|
||||
| Contacts, email addresses, family | `people` | `contacts` |
|
||||
| Appointments, meetings, reminders | `time` | `calendar` |
|
||||
| Payments, invoices, expenses | `finance` | `transactions` |
|
||||
| Packages, shipments | `logistics` | `orders` |
|
||||
| Decisions, preferences | `knowledge` | `decisions` |
|
||||
| Everything else | `default` | `sessions` |
|
||||
|
||||
A single turn can match multiple rooms and will be written to all of them.
|
||||
|
||||
## Configuration
|
||||
|
||||
All settings are optional environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `MEMPALACE_PALACE_PATH` | `~/.mempalace/palace` | Palace directory |
|
||||
| `MEMPALACE_MIN_CHARS` | `80` | Minimum text length to save |
|
||||
| `MEMPALACE_WINDOW_SIZE` | `6` | Recent messages to consider |
|
||||
| `MEMPALACE_DEDUP_MAX` | `500` | Hash store rolling cap |
|
||||
| `MEMPALACE_LANG_DETECT` | `1` | Set to `0` to disable language detection |
|
||||
| `MEMPALACE_MAX_CHARS` | `300` | Max characters per injected memory snippet |
|
||||
| `MEMPALACE_MIN_SIMILARITY` | `0.3` | Min similarity score for ingest results (0 = disabled) |
|
||||
| `MEMPALACE_N_RESULTS` | `5` | Number of memories to inject per turn |
|
||||
| `MEMPALACE_MAX_AGE_DAYS` | `90` | Prune drawers older than this (0 = disabled) |
|
||||
|
||||
## MCP server (optional)
|
||||
|
||||
Add 19 explicit memory tools to all agents:
|
||||
|
||||
```toml
|
||||
[[mcp_servers]]
|
||||
name = "mempalace"
|
||||
timeout_secs = 60
|
||||
[mcp_servers.transport]
|
||||
type = "stdio"
|
||||
command = "python3"
|
||||
args = ["-m", "mempalace.mcp_server"]
|
||||
```
|
||||
|
||||
## Pruning
|
||||
|
||||
Run manually:
|
||||
|
||||
```bash
|
||||
python3 ~/.librefang/plugins/mempalace-indexer/hooks/prune.py --dry-run
|
||||
python3 ~/.librefang/plugins/mempalace-indexer/hooks/prune.py
|
||||
```
|
||||
|
||||
Or trigger via the LibreFang hook system on a schedule.
|
||||
@@ -0,0 +1,333 @@
|
||||
#!/usr/bin/env python3
|
||||
"""MemPalace after_turn hook for LibreFang.
|
||||
|
||||
Filters conversation turns for relevant memories and saves them to MemPalace.
|
||||
Skips: tool calls, short exchanges, noise, and turns where the agent already
|
||||
used mcp_mempalace tools explicitly (deduplication).
|
||||
|
||||
Input (stdin): {"type": "after_turn", "agent_id": "...", "messages": [...]}
|
||||
Output (stdout): {"status": "..."} (fire-and-forget)
|
||||
|
||||
Install: librefang plugin install mempalace-indexer && librefang plugin requirements mempalace-indexer
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from langdetect import detect as _langdetect, LangDetectException
|
||||
_LANGDETECT_AVAILABLE = True
|
||||
except ImportError:
|
||||
_LANGDETECT_AVAILABLE = False
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration: read from LIBREFANG_PLUGIN_CONFIG (written by the runtime),
|
||||
# fall back to individual environment variables for direct invocation.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_config():
|
||||
cfg_path = os.environ.get("LIBREFANG_PLUGIN_CONFIG")
|
||||
if cfg_path:
|
||||
try:
|
||||
with open(cfg_path) as f:
|
||||
return json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
_cfg = _load_config()
|
||||
|
||||
|
||||
def _cfg_str(key, env_key, default):
|
||||
if key in _cfg:
|
||||
return str(_cfg[key])
|
||||
return os.environ.get(env_key, default)
|
||||
|
||||
|
||||
def _cfg_int(key, env_key, default):
|
||||
if key in _cfg:
|
||||
try:
|
||||
return int(_cfg[key])
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
try:
|
||||
return int(os.environ.get(env_key, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _cfg_bool(key, env_key, default):
|
||||
if key in _cfg:
|
||||
v = _cfg[key]
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
return str(v).lower() not in ("0", "false", "no", "off")
|
||||
raw = os.environ.get(env_key)
|
||||
if raw is None:
|
||||
return default
|
||||
return raw != "0"
|
||||
|
||||
|
||||
PALACE_PATH = _cfg_str("palace_path", "MEMPALACE_PALACE_PATH",
|
||||
os.path.expanduser("~/.mempalace/palace"))
|
||||
# Minimum character length of extracted text to be worth saving.
|
||||
MIN_CONTENT_LENGTH = _cfg_int("min_chars", "MEMPALACE_MIN_CHARS", 80)
|
||||
# How many recent messages to consider (sliding window).
|
||||
WINDOW_SIZE = _cfg_int("window_size", "MEMPALACE_WINDOW_SIZE", 6)
|
||||
# Max content hashes to keep in the dedup store (rolling, oldest dropped first).
|
||||
DEDUP_MAX = _cfg_int("dedup_max", "MEMPALACE_DEDUP_MAX", 500)
|
||||
# When langdetect is available, RELEVANCE_RE is only applied to English text.
|
||||
# Non-English text passes on length + dedup alone. Set to false/0 to disable.
|
||||
LANG_DETECT_ENABLED = _cfg_bool("lang_detect", "MEMPALACE_LANG_DETECT", True)
|
||||
|
||||
# Room classification: all matching rules win (multi-room).
|
||||
# Falls back to ("default", "sessions") when nothing matches.
|
||||
ROOM_RULES: list[tuple[re.Pattern, tuple[str, str]]] = [
|
||||
(
|
||||
re.compile(
|
||||
r"\b(contact|phone|email|address|family|wife|husband"
|
||||
r"|son|daughter|parent|colleague|coworker)\b"
|
||||
r"|\S+@\S+\.\w+", # bare email address pattern
|
||||
re.IGNORECASE,
|
||||
),
|
||||
("people", "contacts"),
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"\b(appointment|deadline|birthday|event|meeting|schedule"
|
||||
r"|remind me|reminder|calendar|due date|due on)\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
("time", "calendar"),
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"\b(budget|expense|transaction|payment|bill|salary|invoice"
|
||||
r"|cost|price|paid|spending|refund)\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
("finance", "transactions"),
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"\b(package|order|shipment|delivery|tracking|shipped|arrived)\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
("logistics", "orders"),
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"\b(decision|decided|prefer|from now on|going forward"
|
||||
r"|we.ll use|i.ll use|switching to|chosen|agreed)\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
("knowledge", "decisions"),
|
||||
),
|
||||
]
|
||||
|
||||
RELEVANCE_RE = re.compile(
|
||||
r"\b(decision|decided|prefer|from now on|going forward|remember that|note that"
|
||||
r"|remind me|don.t forget|important|urgent|critical|keep in mind"
|
||||
r"|appointment|deadline|birthday|event|meeting|schedule|due date"
|
||||
r"|budget|expense|transaction|payment|bill|salary|invoice|cost|price"
|
||||
r"|package|order|shipment|delivery|tracking"
|
||||
r"|contact|phone|email|address"
|
||||
r"|like|dislike|preference|habit|allergy"
|
||||
r"|family|wife|husband|son|daughter|parent|colleague"
|
||||
r"|work|client|project|we.ll use|i.ll use|switching to)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Matches fenced code blocks — stripped from content before relevance checks.
|
||||
CODE_BLOCK_RE = re.compile(r"```.*?```", re.DOTALL)
|
||||
|
||||
# Residual noise patterns after code block stripping.
|
||||
# Patterns are anchored or specific to avoid matching normal prose
|
||||
# ("with the exception of", "no traceback available" in casual writing).
|
||||
NOISE_RE = re.compile(
|
||||
r"\[tool_call\]|\[tool_result\]|\"type\":\s*\"tool"
|
||||
r"|Traceback \(most recent call last\)" # Python traceback header
|
||||
r"|^\s*(Exception|Error|Warning):", # exception/error line starts
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def emit(obj: dict) -> None:
|
||||
json.dump(obj, sys.stdout)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
def _detect_language(text: str) -> str:
|
||||
"""Return ISO 639-1 language code, or 'unknown' on failure."""
|
||||
if not _LANGDETECT_AVAILABLE or not LANG_DETECT_ENABLED:
|
||||
return "unknown"
|
||||
try:
|
||||
return _langdetect(text[:400])
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _is_english(lang: str) -> bool:
|
||||
return lang in ("en", "unknown")
|
||||
|
||||
|
||||
def _classify_rooms(text: str) -> list[tuple[str, str]]:
|
||||
"""Return all matching (wing, room) destinations. Falls back to default."""
|
||||
matches = [dest for pattern, dest in ROOM_RULES if pattern.search(text)]
|
||||
return matches if matches else [("default", "sessions")]
|
||||
|
||||
|
||||
def _strip_code_blocks(text: str) -> str:
|
||||
"""Replace fenced code blocks with a placeholder, preserving surrounding context."""
|
||||
return CODE_BLOCK_RE.sub("[code]", text).strip()
|
||||
|
||||
|
||||
def _content_hash(text: str) -> str:
|
||||
"""SHA-256 of the first 500 chars — stable fingerprint for near-duplicate detection."""
|
||||
return hashlib.sha256(text[:500].encode()).hexdigest()
|
||||
|
||||
|
||||
def _dedup_path(agent_id: str) -> Path:
|
||||
# Per-agent store: prevents one agent's memories from blocking another's
|
||||
# when multiple agents share the same palace.
|
||||
safe_id = re.sub(r"[^\w-]", "_", agent_id)[:64]
|
||||
return Path(PALACE_PATH) / f".after_turn_seen_{safe_id}.json"
|
||||
|
||||
|
||||
def _load_seen(agent_id: str) -> list:
|
||||
try:
|
||||
return json.loads(_dedup_path(agent_id).read_text())
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return []
|
||||
|
||||
|
||||
def _save_seen(agent_id: str, hashes: list) -> None:
|
||||
path = _dedup_path(agent_id)
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(hashes[-DEDUP_MAX:]))
|
||||
except OSError:
|
||||
pass # dedup is best-effort; don't block indexing
|
||||
|
||||
|
||||
def _is_duplicate(text: str, agent_id: str) -> bool:
|
||||
h = _content_hash(text)
|
||||
seen = _load_seen(agent_id)
|
||||
if h in seen:
|
||||
return True
|
||||
seen.append(h)
|
||||
_save_seen(agent_id, seen)
|
||||
return False
|
||||
|
||||
|
||||
def extract_text(messages): # list[dict] -> tuple[str, bool]
|
||||
"""Extract user+assistant text; detect if agent already saved to mempalace."""
|
||||
recent = messages[-WINDOW_SIZE:]
|
||||
parts = []
|
||||
agent_used_mempalace = False
|
||||
|
||||
for msg in recent:
|
||||
role = msg.get("role", "")
|
||||
content = msg.get("content") or ""
|
||||
|
||||
if role in ("tool", "assistant") and "mcp_mempalace" in str(content):
|
||||
agent_used_mempalace = True
|
||||
|
||||
if role not in ("user", "assistant"):
|
||||
continue
|
||||
|
||||
if isinstance(content, list):
|
||||
content = "\n".join(
|
||||
b.get("text", "") for b in content
|
||||
if isinstance(b, dict) and b.get("type") == "text"
|
||||
)
|
||||
|
||||
if not content:
|
||||
continue
|
||||
|
||||
content = _strip_code_blocks(content)
|
||||
|
||||
if not content or NOISE_RE.search(content):
|
||||
continue
|
||||
|
||||
parts.append(f"[{role}] {content}")
|
||||
|
||||
return "\n".join(parts), agent_used_mempalace
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
except (json.JSONDecodeError, EOFError):
|
||||
emit({"status": "skip", "reason": "bad input"})
|
||||
return
|
||||
|
||||
messages = data.get("messages", [])
|
||||
agent_id = data.get("agent_id", "unknown")
|
||||
|
||||
if not messages:
|
||||
emit({"status": "skip", "reason": "no messages"})
|
||||
return
|
||||
|
||||
text, already_saved = extract_text(messages)
|
||||
|
||||
if already_saved:
|
||||
emit({"status": "skip", "reason": "agent used mcp_mempalace"})
|
||||
return
|
||||
|
||||
if len(text) < MIN_CONTENT_LENGTH:
|
||||
emit({"status": "skip", "reason": "too short"})
|
||||
return
|
||||
|
||||
lang = _detect_language(text)
|
||||
# RELEVANCE_RE is English-only; skip it for non-English to avoid false negatives.
|
||||
if _is_english(lang) and not RELEVANCE_RE.search(text):
|
||||
emit({"status": "skip", "reason": "not relevant"})
|
||||
return
|
||||
|
||||
if _is_duplicate(text, agent_id):
|
||||
emit({"status": "skip", "reason": "duplicate"})
|
||||
return
|
||||
|
||||
try:
|
||||
from mempalace.miner import get_collection, add_drawer
|
||||
|
||||
collection = get_collection(PALACE_PATH)
|
||||
source = f"auto-{agent_id}-{datetime.now().strftime('%Y%m%d-%H%M%S%f')}"
|
||||
rooms = _classify_rooms(text)
|
||||
|
||||
for wing, room in rooms:
|
||||
add_drawer(
|
||||
collection=collection,
|
||||
wing=wing,
|
||||
room=room,
|
||||
content=text,
|
||||
source_file=source,
|
||||
chunk_index=0,
|
||||
agent="mempalace-indexer",
|
||||
)
|
||||
|
||||
emit({
|
||||
"status": "indexed",
|
||||
"chars": len(text),
|
||||
"lang": lang,
|
||||
"rooms": [{"wing": w, "room": r} for w, r in rooms],
|
||||
})
|
||||
except Exception as e:
|
||||
emit({"status": "error", "error": str(e)})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""MemPalace ingest hook for LibreFang.
|
||||
|
||||
Searches MemPalace for memories relevant to the incoming user message
|
||||
and injects them into the agent's context as MemoryFragments.
|
||||
|
||||
Input (stdin): {"type": "ingest", "agent_id": "...", "message": "user text"}
|
||||
Output (stdout): {"memories": [{"content": "..."}]}
|
||||
|
||||
Install: librefang plugin install mempalace-indexer && librefang plugin requirements mempalace-indexer
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration: read from LIBREFANG_PLUGIN_CONFIG (written by the runtime),
|
||||
# fall back to individual environment variables for direct invocation.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_config():
|
||||
cfg_path = os.environ.get("LIBREFANG_PLUGIN_CONFIG")
|
||||
if cfg_path:
|
||||
try:
|
||||
with open(cfg_path) as f:
|
||||
return json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
_cfg = _load_config()
|
||||
|
||||
|
||||
def _cfg_str(key, env_key, default):
|
||||
if key in _cfg:
|
||||
return str(_cfg[key])
|
||||
return os.environ.get(env_key, default)
|
||||
|
||||
|
||||
def _cfg_int(key, env_key, default):
|
||||
if key in _cfg:
|
||||
try:
|
||||
return int(_cfg[key])
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return int(os.environ.get(env_key, default))
|
||||
|
||||
|
||||
def _cfg_float(key, env_key, default):
|
||||
if key in _cfg:
|
||||
try:
|
||||
return float(_cfg[key])
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return float(os.environ.get(env_key, default))
|
||||
|
||||
|
||||
PALACE_PATH = _cfg_str("palace_path", "MEMPALACE_PALACE_PATH",
|
||||
os.path.expanduser("~/.mempalace/palace"))
|
||||
MAX_MEMORY_CHARS = _cfg_int("max_chars", "MEMPALACE_MAX_CHARS", "300")
|
||||
|
||||
# MemPalace returns similarity in [0, 1] — higher means more relevant.
|
||||
# Results below MIN_SIMILARITY are too dissimilar to be useful.
|
||||
# Set to 0 (or MEMPALACE_MIN_SIMILARITY=0) to disable filtering.
|
||||
MIN_SIMILARITY = _cfg_float("min_similarity", "MEMPALACE_MIN_SIMILARITY", "0.3")
|
||||
N_RESULTS = _cfg_int("n_results", "MEMPALACE_N_RESULTS", "5")
|
||||
|
||||
|
||||
def emit(obj):
|
||||
"""Write JSON response to stdout with trailing newline."""
|
||||
json.dump(obj, sys.stdout)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
def truncate_at_word(text, max_len):
|
||||
"""Truncate text at nearest word boundary."""
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
truncated = text[:max_len]
|
||||
last_space = truncated.rfind(" ")
|
||||
if last_space > max_len // 2:
|
||||
return truncated[:last_space] + "..."
|
||||
return truncated + "..."
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
except (json.JSONDecodeError, EOFError):
|
||||
emit({"memories": []})
|
||||
return
|
||||
|
||||
message = data.get("message", "")
|
||||
if not message or len(message) < 5:
|
||||
emit({"memories": []})
|
||||
return
|
||||
|
||||
try:
|
||||
from mempalace.searcher import search_memories
|
||||
|
||||
results = search_memories(message, PALACE_PATH, n_results=N_RESULTS)
|
||||
|
||||
memories = []
|
||||
for r in results.get("results", []):
|
||||
text = r.get("text", "")
|
||||
wing = r.get("wing", "") or "memory"
|
||||
room = r.get("room", "")
|
||||
similarity = r.get("similarity")
|
||||
|
||||
if not text:
|
||||
continue
|
||||
|
||||
# Filter out low-relevance results when the backend provides a score.
|
||||
# similarity=None means the backend didn't return one — allow through.
|
||||
if similarity is not None and MIN_SIMILARITY > 0 and similarity < MIN_SIMILARITY:
|
||||
continue
|
||||
|
||||
snippet = truncate_at_word(text, MAX_MEMORY_CHARS)
|
||||
# Use wing/room (semantically meaningful) instead of raw source filename.
|
||||
label = f"{wing}/{room}" if room else wing
|
||||
memories.append({"content": f"[{label}] {snippet}"})
|
||||
|
||||
emit({"memories": memories})
|
||||
except Exception as e:
|
||||
emit({"memories": [], "error": str(e)})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""MemPalace pruner for LibreFang.
|
||||
|
||||
Deletes drawers older than MEMPALACE_MAX_AGE_DAYS from the palace.
|
||||
Intended to be run periodically (e.g. via a LibreFang scheduled hook or cron).
|
||||
|
||||
NOTE: This script reads ChromaDB directly using the collection name
|
||||
("mempalace_drawers") and metadata field ("filed_at") that MemPalace uses
|
||||
internally. If a future MemPalace release changes these, update accordingly.
|
||||
|
||||
Usage:
|
||||
python3 prune.py [--dry-run]
|
||||
|
||||
Input (stdin): {"type": "prune", "agent_id": "..."} (optional, for hook mode)
|
||||
Output (stdout): {"status": "...", "deleted": N, "kept": N}
|
||||
|
||||
Environment:
|
||||
MEMPALACE_PALACE_PATH Path to the palace directory (default: ~/.mempalace/palace)
|
||||
MEMPALACE_MAX_AGE_DAYS Delete drawers older than this many days (default: 90, 0 = disabled)
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
PALACE_PATH = os.environ.get(
|
||||
"MEMPALACE_PALACE_PATH",
|
||||
os.path.expanduser("~/.mempalace/palace"),
|
||||
)
|
||||
MAX_AGE_DAYS = int(os.environ.get("MEMPALACE_MAX_AGE_DAYS", "90"))
|
||||
|
||||
# MemPalace internal constants — update if upstream changes them.
|
||||
_COLLECTION_NAME = "mempalace_drawers"
|
||||
_FILED_AT_FIELD = "filed_at"
|
||||
|
||||
|
||||
def emit(obj: dict) -> None:
|
||||
json.dump(obj, sys.stdout)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
def _parse_filed_at(value: str): # -> datetime | None
|
||||
"""Parse MemPalace's ISO timestamp into a timezone-aware datetime."""
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromisoformat(value)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def prune(dry_run: bool = False) -> dict:
|
||||
if MAX_AGE_DAYS <= 0:
|
||||
return {"status": "skip", "reason": "MEMPALACE_MAX_AGE_DAYS=0 (disabled)"}
|
||||
|
||||
cutoff = datetime.now(tz=timezone.utc) - timedelta(days=MAX_AGE_DAYS)
|
||||
|
||||
try:
|
||||
import chromadb
|
||||
except ImportError:
|
||||
return {"status": "error", "error": "chromadb not installed (install mempalace first)"}
|
||||
|
||||
palace = Path(PALACE_PATH)
|
||||
if not palace.exists():
|
||||
return {"status": "error", "error": f"Palace path not found: {PALACE_PATH}"}
|
||||
|
||||
try:
|
||||
client = chromadb.PersistentClient(path=str(palace))
|
||||
collection = client.get_collection(_COLLECTION_NAME)
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": f"Could not open collection '{_COLLECTION_NAME}': {e}"}
|
||||
|
||||
# Fetch all drawers. For large palaces this is O(n) in memory — acceptable
|
||||
# for a personal-scale deployment (tens of thousands of items at most).
|
||||
try:
|
||||
result = collection.get(include=["metadatas"])
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": f"collection.get() failed: {e}"}
|
||||
|
||||
ids = result.get("ids", [])
|
||||
metadatas = result.get("metadatas", [])
|
||||
|
||||
to_delete = []
|
||||
kept = 0
|
||||
|
||||
for doc_id, meta in zip(ids, metadatas):
|
||||
filed_at = _parse_filed_at((meta or {}).get(_FILED_AT_FIELD, ""))
|
||||
if filed_at is not None and filed_at < cutoff:
|
||||
to_delete.append(doc_id)
|
||||
else:
|
||||
kept += 1
|
||||
|
||||
if to_delete and not dry_run:
|
||||
try:
|
||||
collection.delete(ids=to_delete)
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": f"delete failed: {e}", "attempted": len(to_delete)}
|
||||
|
||||
return {
|
||||
"status": "dry-run" if dry_run else "ok",
|
||||
"deleted": len(to_delete),
|
||||
"kept": kept,
|
||||
"cutoff": cutoff.isoformat(),
|
||||
"max_age_days": MAX_AGE_DAYS,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
dry_run = "--dry-run" in sys.argv
|
||||
|
||||
# Also accept hook-style JSON input (stdin), ignoring the payload content.
|
||||
if not sys.stdin.isatty():
|
||||
try:
|
||||
json.load(sys.stdin)
|
||||
except (json.JSONDecodeError, EOFError):
|
||||
pass
|
||||
|
||||
emit(prune(dry_run=dry_run))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,121 @@
|
||||
name = "mempalace-indexer"
|
||||
version = "0.3.0"
|
||||
description = "Auto-index conversations into MemPalace and recall relevant memories. No API keys, no cloud."
|
||||
author = "LibreFang Community"
|
||||
requirements = "requirements.txt"
|
||||
librefang_min_version = "2026.4.0"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hook execution settings
|
||||
#
|
||||
# MUST sit above [hooks] so they land on the top-level table instead of being
|
||||
# folded into [hooks] by TOML table scoping rules (validate.py iterates
|
||||
# data["hooks"].items() expecting every value to be a hook file path).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ingest is fast (vector search) — 30 s is generous.
|
||||
# after_turn does classification + write — 60 s covers a slow machine or large turn.
|
||||
hook_timeout_secs = 30
|
||||
|
||||
# A temporary mempalace outage should not block the agent turn.
|
||||
on_hook_failure = "warn"
|
||||
|
||||
# One silent retry handles transient ChromaDB lock contention.
|
||||
max_retries = 1
|
||||
retry_delay_ms = 500
|
||||
|
||||
# Python interpreter startup is ~200 ms. Keep subprocesses alive between calls
|
||||
# to eliminate that overhead on every user message.
|
||||
persistent_subprocess = true
|
||||
|
||||
# ingest results are deterministic for the same message text — cache for 60 s
|
||||
# to avoid redundant embedding lookups on rapid follow-up turns.
|
||||
hook_cache_ttl_secs = 60
|
||||
|
||||
# Priority 0 (default). Raise to e.g. 10 to run before other recall plugins.
|
||||
priority = 0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System binary requirements
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
[[requires]]
|
||||
binary = "python3"
|
||||
install_hint = "Install Python 3.9+ from https://python.org or via your system package manager (apt install python3 / brew install python)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hook scripts
|
||||
#
|
||||
# Every value in this section MUST be a path relative to the plugin directory.
|
||||
# Execution settings live above at the top level.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
[hooks]
|
||||
ingest = "hooks/ingest.py"
|
||||
after_turn = "hooks/after_turn.py"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# [config] — user-configurable settings
|
||||
#
|
||||
# The runtime merges user overrides with these defaults and writes the result
|
||||
# as JSON to the path in LIBREFANG_PLUGIN_CONFIG before each hook subprocess.
|
||||
# Hooks should read that file (falling back to environment variables for
|
||||
# backward compatibility with direct invocation).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
[config.palace_path]
|
||||
type = "string"
|
||||
default = "~/.mempalace/palace"
|
||||
description = "Path to the MemPalace directory (passed to mempalace APIs and ChromaDB)."
|
||||
|
||||
[config.min_chars]
|
||||
type = "number"
|
||||
default = 80
|
||||
description = "Minimum character length of extracted conversation text to be worth saving. Shorter exchanges are skipped by after_turn."
|
||||
|
||||
[config.window_size]
|
||||
type = "number"
|
||||
default = 6
|
||||
description = "Number of recent messages the after_turn hook considers when extracting text (sliding window from the end of the conversation)."
|
||||
|
||||
[config.dedup_max]
|
||||
type = "number"
|
||||
default = 500
|
||||
description = "Maximum number of SHA-256 content hashes kept in the per-agent deduplication store. Oldest entries are dropped first when the cap is reached."
|
||||
|
||||
[config.lang_detect]
|
||||
type = "boolean"
|
||||
default = true
|
||||
description = "Enable language detection via langdetect. When true, non-English exchanges bypass the English keyword relevance filter and are indexed on length alone."
|
||||
|
||||
[config.max_chars]
|
||||
type = "number"
|
||||
default = 300
|
||||
description = "Maximum characters per injected memory snippet in the ingest hook. Longer texts are truncated at the nearest word boundary."
|
||||
|
||||
[config.min_similarity]
|
||||
type = "number"
|
||||
default = 0.3
|
||||
description = "Minimum similarity score (0-1, higher = more relevant) for memories returned by the ingest hook. Results below this threshold are dropped. Set to 0 to disable filtering."
|
||||
|
||||
[config.n_results]
|
||||
type = "number"
|
||||
default = 5
|
||||
description = "Number of memories the ingest hook retrieves from MemPalace per turn."
|
||||
|
||||
[config.max_age_days]
|
||||
type = "number"
|
||||
default = 90
|
||||
description = "Prune drawers older than this many days when running hooks/prune.py. Set to 0 to disable TTL pruning."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# [integrity] — SHA-256 hashes verified at load time
|
||||
#
|
||||
# Regenerate after editing hook files:
|
||||
# sha256sum hooks/ingest.py hooks/after_turn.py hooks/prune.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
[integrity]
|
||||
"hooks/ingest.py" = "621b17f2d0afbd6142a49963fb57187a500cfeac057c13cb80961b6222bd03e2"
|
||||
"hooks/after_turn.py" = "1c5bcdade4ca0d785878d60910105208fcff38f74872426ee821363ce5bf9811"
|
||||
"hooks/prune.py" = "2dceeb9bb0d0b7ee224291bfce600f629f38b4a853cd437774d9b66ef226838d"
|
||||
@@ -0,0 +1,2 @@
|
||||
mempalace>=3.0.0,<4
|
||||
langdetect>=1.0.9,<2
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Tests for the after_turn hook (stdin/stdout interface)."""
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
HOOK = Path(__file__).parent.parent / "hooks" / "after_turn.py"
|
||||
|
||||
|
||||
def run_hook(payload: dict, env=None) -> dict:
|
||||
import os
|
||||
run_env = os.environ.copy()
|
||||
if env:
|
||||
run_env.update(env)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(HOOK)],
|
||||
input=json.dumps(payload),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=run_env,
|
||||
)
|
||||
return json.loads(result.stdout.strip())
|
||||
|
||||
|
||||
def _load_module(env_overrides=None):
|
||||
import os
|
||||
saved = {}
|
||||
if env_overrides:
|
||||
for k, v in env_overrides.items():
|
||||
saved[k] = os.environ.get(k)
|
||||
os.environ[k] = v
|
||||
spec = importlib.util.spec_from_file_location("after_turn", HOOK)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
for k, v in saved.items():
|
||||
if v is None:
|
||||
os.environ.pop(k, None)
|
||||
else:
|
||||
os.environ[k] = v
|
||||
return mod
|
||||
|
||||
|
||||
_mod = _load_module()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Skip cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_empty_messages_skipped():
|
||||
out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": []})
|
||||
assert out["status"] == "skip"
|
||||
assert out["reason"] == "no messages"
|
||||
|
||||
|
||||
def test_bad_json_skipped():
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(HOOK)],
|
||||
input="not json",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
out = json.loads(result.stdout.strip())
|
||||
assert out["status"] == "skip"
|
||||
assert out["reason"] == "bad input"
|
||||
|
||||
|
||||
def test_short_exchange_skipped():
|
||||
messages = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
]
|
||||
out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages})
|
||||
assert out["status"] == "skip"
|
||||
assert out["reason"] == "too short"
|
||||
|
||||
|
||||
def test_irrelevant_exchange_skipped():
|
||||
messages = [
|
||||
{"role": "user", "content": "What is the capital of France and why is it historically significant?"},
|
||||
{"role": "assistant", "content": "Paris has been the capital since the 10th century and is the cultural hub."},
|
||||
]
|
||||
out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages})
|
||||
assert out["status"] == "skip"
|
||||
assert out["reason"] == "not relevant"
|
||||
|
||||
|
||||
def test_agent_mempalace_tool_call_skipped():
|
||||
messages = [
|
||||
{"role": "user", "content": "Save this to my memory please."},
|
||||
{"role": "tool", "content": '{"tool": "mcp_mempalace_add_drawer", "result": "ok"}'},
|
||||
{"role": "assistant", "content": "Done, I saved it using mcp_mempalace."},
|
||||
]
|
||||
out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages})
|
||||
assert out["status"] == "skip"
|
||||
assert out["reason"] == "agent used mcp_mempalace"
|
||||
|
||||
|
||||
def test_user_mentioning_mempalace_not_skipped():
|
||||
"""User saying 'mcp_mempalace' should NOT trigger the dedup skip."""
|
||||
messages = [
|
||||
{"role": "user", "content": "Can you use mcp_mempalace to save my dentist appointment on April 15th?"},
|
||||
{"role": "assistant", "content": "I will remember your dentist appointment on April 15th."},
|
||||
]
|
||||
out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages})
|
||||
assert out.get("reason") != "agent used mcp_mempalace"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Code block stripping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_noise_re_does_not_match_common_prose():
|
||||
"""'exception' and 'traceback' as normal words must not trigger noise filter."""
|
||||
assert not _mod.NOISE_RE.search("There's no exception to this rule.")
|
||||
assert not _mod.NOISE_RE.search("Let me traceback the history of this decision.")
|
||||
|
||||
|
||||
def test_noise_re_matches_python_traceback():
|
||||
assert _mod.NOISE_RE.search("Traceback (most recent call last):\n File foo.py")
|
||||
|
||||
|
||||
def test_noise_re_matches_error_line():
|
||||
assert _mod.NOISE_RE.search("Exception: something went wrong")
|
||||
assert _mod.NOISE_RE.search("Error: connection refused")
|
||||
|
||||
|
||||
def test_code_block_stripped_not_whole_message_skipped():
|
||||
"""A message with a code block but meaningful surrounding text should not be skipped."""
|
||||
text = "My email is alice@example.com. Here is the script:\n```bash\necho hello\n```\nRun it daily."
|
||||
result = _mod._strip_code_blocks(text)
|
||||
assert "alice@example.com" in result
|
||||
assert "```" not in result
|
||||
assert "[code]" in result
|
||||
|
||||
|
||||
def test_pure_code_block_skipped_after_strip():
|
||||
"""A message that is only a code block becomes empty after stripping → too short → skip."""
|
||||
messages = [
|
||||
{"role": "user", "content": "here's the script"},
|
||||
{"role": "assistant", "content": "```python\nfor i in range(10):\n print(i)\n```"},
|
||||
]
|
||||
out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages})
|
||||
# Should be skipped due to length or irrelevance after stripping
|
||||
assert out["status"] == "skip"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-room classification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_classify_single_room():
|
||||
assert _mod._classify_rooms("Her email is alice@example.com") == [("people", "contacts")]
|
||||
|
||||
|
||||
def test_classify_multiple_rooms():
|
||||
# "meeting" → calendar, "email" → contacts — both should match
|
||||
rooms = _mod._classify_rooms("Schedule a meeting with alice@example.com next Tuesday")
|
||||
assert ("people", "contacts") in rooms
|
||||
assert ("time", "calendar") in rooms
|
||||
|
||||
|
||||
def test_classify_finance():
|
||||
assert _mod._classify_rooms("Invoice for $500 is due next week") == [("finance", "transactions")]
|
||||
|
||||
|
||||
def test_classify_logistics():
|
||||
assert _mod._classify_rooms("The shipment arrived today") == [("logistics", "orders")]
|
||||
|
||||
|
||||
def test_classify_decisions():
|
||||
assert _mod._classify_rooms("We decided to use Postgres going forward") == [("knowledge", "decisions")]
|
||||
|
||||
|
||||
def test_classify_default_fallback():
|
||||
assert _mod._classify_rooms("The sky is blue and the grass is green.") == [("default", "sessions")]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content deduplication
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_duplicate_detection():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
mod = _load_module({"MEMPALACE_PALACE_PATH": tmpdir})
|
||||
text = "I have a dentist appointment on April 15th with Dr. Smith."
|
||||
assert mod._is_duplicate(text, "agent-1") is False # first time: not a duplicate
|
||||
assert mod._is_duplicate(text, "agent-1") is True # second time: duplicate
|
||||
|
||||
|
||||
def test_different_content_not_duplicate():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
mod = _load_module({"MEMPALACE_PALACE_PATH": tmpdir})
|
||||
assert mod._is_duplicate("dentist appointment April 15th", "agent-1") is False
|
||||
assert mod._is_duplicate("meeting with Bob on Friday afternoon", "agent-1") is False
|
||||
|
||||
|
||||
def test_dedup_is_per_agent():
|
||||
"""Two different agents should have independent dedup stores."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
mod = _load_module({"MEMPALACE_PALACE_PATH": tmpdir})
|
||||
text = "I have a dentist appointment on April 15th with Dr. Smith."
|
||||
mod._is_duplicate(text, "agent-1") # agent-1 saves it
|
||||
assert mod._is_duplicate(text, "agent-2") is False # agent-2 has not seen it
|
||||
|
||||
|
||||
def test_dedup_rolling_max():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
mod = _load_module({"MEMPALACE_PALACE_PATH": tmpdir, "MEMPALACE_DEDUP_MAX": "3"})
|
||||
for i in range(5):
|
||||
mod._is_duplicate(f"unique content number {i} with enough chars to matter", "agent-1")
|
||||
seen = json.loads((Path(tmpdir) / ".after_turn_seen_agent-1.json").read_text())
|
||||
assert len(seen) == 3 # capped at DEDUP_MAX
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configurable parameters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_custom_min_chars_env():
|
||||
mod = _load_module({"MEMPALACE_MIN_CHARS": "200"})
|
||||
assert mod.MIN_CONTENT_LENGTH == 200
|
||||
|
||||
|
||||
def test_custom_window_size_env():
|
||||
mod = _load_module({"MEMPALACE_WINDOW_SIZE": "10"})
|
||||
assert mod.WINDOW_SIZE == 10
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Language detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_detect_language_returns_string():
|
||||
lang = _mod._detect_language("I have a meeting with the client tomorrow afternoon.")
|
||||
assert isinstance(lang, str)
|
||||
assert len(lang) > 0
|
||||
|
||||
|
||||
def test_is_english_known_english():
|
||||
assert _mod._is_english("en") is True
|
||||
|
||||
|
||||
def test_is_english_unknown_treated_as_english():
|
||||
# unknown = langdetect unavailable or failed → don't block on keyword check
|
||||
assert _mod._is_english("unknown") is True
|
||||
|
||||
|
||||
def test_is_english_other_language():
|
||||
assert _mod._is_english("zh") is False
|
||||
assert _mod._is_english("ja") is False
|
||||
assert _mod._is_english("fr") is False
|
||||
|
||||
|
||||
def test_lang_detect_disabled_returns_unknown():
|
||||
mod = _load_module({"MEMPALACE_LANG_DETECT": "0"})
|
||||
lang = mod._detect_language("Ich habe morgen einen Termin beim Zahnarzt.")
|
||||
assert lang == "unknown"
|
||||
|
||||
|
||||
def test_non_english_skips_relevance_check():
|
||||
"""A non-English exchange long enough to pass length check should not be
|
||||
blocked by the English-only RELEVANCE_RE."""
|
||||
# This text is in Chinese and contains no English keywords from RELEVANCE_RE,
|
||||
# so without language detection it would be skipped as "not relevant".
|
||||
messages = [
|
||||
{"role": "user",
|
||||
"content": "我明天下午三点有个牙医预约，在城市医院，请帮我记住这件事。"},
|
||||
{"role": "assistant",
|
||||
"content": "好的，我已经记住了您明天下午三点在城市医院的牙医预约。"},
|
||||
]
|
||||
out = run_hook({"type": "after_turn", "agent_id": "a1", "messages": messages})
|
||||
# Should NOT be skipped for "not relevant" — may fail at mempalace import (error) or dedup
|
||||
assert out.get("reason") != "not relevant"
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Tests for the ingest hook (stdin/stdout interface)."""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HOOK = Path(__file__).parent.parent / "hooks" / "ingest.py"
|
||||
|
||||
|
||||
def run_hook(payload: dict) -> dict:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(HOOK)],
|
||||
input=json.dumps(payload),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(result.stdout.strip())
|
||||
|
||||
|
||||
def test_bad_json_returns_empty():
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(HOOK)],
|
||||
input="not json",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
out = json.loads(result.stdout.strip())
|
||||
assert out == {"memories": []}
|
||||
|
||||
|
||||
def test_empty_message_returns_empty():
|
||||
out = run_hook({"type": "ingest", "agent_id": "a1", "message": ""})
|
||||
assert out == {"memories": []}
|
||||
|
||||
|
||||
def test_short_message_returns_empty():
|
||||
out = run_hook({"type": "ingest", "agent_id": "a1", "message": "hi"})
|
||||
assert out == {"memories": []}
|
||||
|
||||
|
||||
def test_no_mempalace_returns_error_not_crash():
|
||||
"""Without mempalace installed, ingest returns error field but doesn't crash."""
|
||||
out = run_hook({"type": "ingest", "agent_id": "a1", "message": "What are my upcoming meetings?"})
|
||||
assert "memories" in out
|
||||
assert isinstance(out["memories"], list)
|
||||
assert "error" in out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Similarity filtering (unit-level, no mempalace needed)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_ingest_module(min_similarity="0.3"):
|
||||
saved = os.environ.get("MEMPALACE_MIN_SIMILARITY")
|
||||
os.environ["MEMPALACE_MIN_SIMILARITY"] = min_similarity
|
||||
spec = importlib.util.spec_from_file_location("ingest", HOOK)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
if saved is None:
|
||||
os.environ.pop("MEMPALACE_MIN_SIMILARITY", None)
|
||||
else:
|
||||
os.environ["MEMPALACE_MIN_SIMILARITY"] = saved
|
||||
return mod
|
||||
|
||||
|
||||
def test_truncate_at_word_boundary():
|
||||
mod = _load_ingest_module()
|
||||
text = "one two three four five six seven"
|
||||
result = mod.truncate_at_word(text, 15)
|
||||
assert result.endswith("...")
|
||||
assert len(result) <= 18
|
||||
|
||||
|
||||
def test_truncate_short_unchanged():
|
||||
mod = _load_ingest_module()
|
||||
assert mod.truncate_at_word("hello", 100) == "hello"
|
||||
|
||||
|
||||
def test_similarity_threshold_filters_results():
|
||||
"""Simulate the similarity filtering logic directly."""
|
||||
mod = _load_ingest_module(min_similarity="0.5")
|
||||
|
||||
results = [
|
||||
{"text": "good match", "source_file": "s1", "wing": "w1", "similarity": 0.8},
|
||||
{"text": "bad match", "source_file": "s2", "wing": "w1", "similarity": 0.2},
|
||||
{"text": "no score", "source_file": "s3", "wing": "w1"},
|
||||
]
|
||||
|
||||
MIN_SIMILARITY = mod.MIN_SIMILARITY
|
||||
memories = []
|
||||
for r in results:
|
||||
text = r.get("text", "")
|
||||
similarity = r.get("similarity")
|
||||
if not text:
|
||||
continue
|
||||
if similarity is not None and MIN_SIMILARITY > 0 and similarity < MIN_SIMILARITY:
|
||||
continue
|
||||
memories.append(text)
|
||||
|
||||
assert "good match" in memories
|
||||
assert "bad match" not in memories
|
||||
assert "no score" in memories # passthrough when similarity is absent
|
||||
|
||||
|
||||
def test_similarity_zero_disables_filtering():
|
||||
mod = _load_ingest_module(min_similarity="0")
|
||||
assert mod.MIN_SIMILARITY == 0.0
|
||||
|
||||
|
||||
def test_min_similarity_default():
|
||||
mod = _load_ingest_module()
|
||||
assert mod.MIN_SIMILARITY == 0.3
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Tests for the prune hook."""
|
||||
import importlib.util
|
||||
import json
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
HOOK = Path(__file__).parent.parent / "hooks" / "prune.py"
|
||||
|
||||
|
||||
def _load():
|
||||
spec = importlib.util.spec_from_file_location("prune", HOOK)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
_mod = _load()
|
||||
|
||||
|
||||
def test_disabled_when_max_age_zero(monkeypatch):
|
||||
monkeypatch.setattr(_mod, "MAX_AGE_DAYS", 0)
|
||||
result = _mod.prune()
|
||||
assert result["status"] == "skip"
|
||||
|
||||
|
||||
def test_error_when_palace_missing(monkeypatch, tmp_path):
|
||||
try:
|
||||
import chromadb # noqa: F401
|
||||
except ImportError:
|
||||
import pytest
|
||||
pytest.skip("chromadb not installed")
|
||||
monkeypatch.setattr(_mod, "PALACE_PATH", str(tmp_path / "nonexistent"))
|
||||
monkeypatch.setattr(_mod, "MAX_AGE_DAYS", 30)
|
||||
result = _mod.prune()
|
||||
assert result["status"] == "error"
|
||||
assert "not found" in result["error"]
|
||||
|
||||
|
||||
def test_parse_filed_at_valid():
|
||||
dt = _mod._parse_filed_at("2026-01-01T10:00:00")
|
||||
assert dt is not None
|
||||
assert dt.year == 2026
|
||||
|
||||
|
||||
def test_parse_filed_at_empty():
|
||||
assert _mod._parse_filed_at("") is None
|
||||
|
||||
|
||||
def test_parse_filed_at_invalid():
|
||||
assert _mod._parse_filed_at("not-a-date") is None
|
||||
|
||||
|
||||
def test_parse_filed_at_timezone_aware():
|
||||
dt = _mod._parse_filed_at("2026-01-01T10:00:00+05:00")
|
||||
assert dt.tzinfo is not None
|
||||
|
||||
|
||||
def test_prune_dry_run_with_chromadb(monkeypatch, tmp_path):
|
||||
"""Integration test: create a real ChromaDB collection and prune old entries."""
|
||||
try:
|
||||
import chromadb
|
||||
except ImportError:
|
||||
import pytest
|
||||
pytest.skip("chromadb not installed")
|
||||
|
||||
monkeypatch.setattr(_mod, "PALACE_PATH", str(tmp_path))
|
||||
monkeypatch.setattr(_mod, "MAX_AGE_DAYS", 30)
|
||||
|
||||
client = chromadb.PersistentClient(path=str(tmp_path))
|
||||
col = client.get_or_create_collection("mempalace_drawers")
|
||||
|
||||
old_date = (datetime.now(tz=timezone.utc) - timedelta(days=60)).isoformat()
|
||||
new_date = datetime.now(tz=timezone.utc).isoformat()
|
||||
|
||||
col.add(
|
||||
ids=["old-1", "new-1"],
|
||||
documents=["old memory", "new memory"],
|
||||
metadatas=[
|
||||
{"filed_at": old_date, "wing": "default", "room": "sessions"},
|
||||
{"filed_at": new_date, "wing": "default", "room": "sessions"},
|
||||
],
|
||||
)
|
||||
|
||||
result = _mod.prune(dry_run=True)
|
||||
assert result["status"] == "dry-run"
|
||||
assert result["deleted"] == 1
|
||||
assert result["kept"] == 1
|
||||
# dry-run: nothing actually deleted
|
||||
assert col.count() == 2
|
||||
|
||||
|
||||
def test_prune_actually_deletes(monkeypatch, tmp_path):
|
||||
try:
|
||||
import chromadb
|
||||
except ImportError:
|
||||
import pytest
|
||||
pytest.skip("chromadb not installed")
|
||||
|
||||
monkeypatch.setattr(_mod, "PALACE_PATH", str(tmp_path))
|
||||
monkeypatch.setattr(_mod, "MAX_AGE_DAYS", 30)
|
||||
|
||||
client = chromadb.PersistentClient(path=str(tmp_path))
|
||||
col = client.get_or_create_collection("mempalace_drawers")
|
||||
|
||||
old_date = (datetime.now(tz=timezone.utc) - timedelta(days=60)).isoformat()
|
||||
col.add(
|
||||
ids=["old-2"],
|
||||
documents=["stale memory"],
|
||||
metadatas=[{"filed_at": old_date}],
|
||||
)
|
||||
|
||||
result = _mod.prune(dry_run=False)
|
||||
assert result["status"] == "ok"
|
||||
assert result["deleted"] == 1
|
||||
assert col.count() == 0
|
||||
Reference in new issue
Block a user