ci: build plugins-index.json in-repo so worker refresh stays under budget

Walking ~40+ plugin TOMLs via the GitHub Contents API from the worker
exceeded the Workers Free 50-subrequest-per-invocation limit, leaving
the daemon's signed plugins index either empty or partial after every
forced refresh.

Move the walk into the repo: scripts/build-plugins-index.mjs reads each
plugins/<name>/plugin.toml directly from the checked-out tree and emits
a sorted flat array (name, version?, description?, needs?) at
plugins-index.json. The CI workflow regenerates and commits this file
on every push under plugins/, then pokes the worker's
/api/registry/refresh — which now fetches the single committed
plugins-index.json (1 subrequest), validates the JSON shape, and
re-signs it with Ed25519. Refresh cost is now constant in registry
size, not linear.

The dashboard's dict-shaped /api/registry payload is unchanged — that
still rebuilds via the daily 02:00 UTC cron.

.github/workflows/refresh-cache.yml

@@ -1,31 +1,52 @@
 name: Refresh registry-worker cache
 
-# Triggers the registry-worker to rebuild its D1 cache and re-sign the
-# plugins index right after content changes — without this, dashboard /
-# daemon would have to wait for the next 02:00 UTC cron tick.
+# On every push that changes plugins/, regenerate plugins-index.json
+# (the daemon-shaped flat array the registry-worker signs and the
+# librefang daemon installs from), commit it back, then poke the
+# worker's forced-refresh endpoint so it re-signs the new bytes
+# immediately — without this, daemon installs lag up to ~24h behind
+# the next 02:00 UTC cron tick.
 #
-# Auth: REGISTRY_REFRESH_TOKEN (repo secret) is matched against the
-# REGISTRY_REFRESH_TOKEN worker secret in librefang-registry. Until both
-# are set, the worker endpoint returns 503.
+# Walking the on-disk repo (instead of the worker hitting GitHub
+# Contents API per-file) keeps the worker's refresh path under the
+# Workers Free 50-subrequest budget regardless of registry size.
 
 on:
   push:
     branches: [main]
     paths:
       - 'plugins/**'
-      - 'agents/**'
-      - 'skills/**'
-      - 'hands/**'
-      - 'channels/**'
-      - 'providers/**'
-      - 'workflows/**'
-      - 'mcp/**'
-  workflow_dispatch: # manual trigger for ops
+      - 'scripts/build-plugins-index.mjs'
+  workflow_dispatch: # manual trigger for ops / first-deploy
+
+permissions:
+  contents: write   # needed to commit the regenerated index back
 
 jobs:
   refresh:
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Rebuild plugins-index.json
+        run: node scripts/build-plugins-index.mjs
+
+      - name: Commit regenerated index if changed
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add plugins-index.json
+          if git diff --cached --quiet; then
+            echo "plugins-index.json already up-to-date"
+          else
+            git commit -m "chore: regenerate plugins-index.json"
+            git push
+          fi
+
       - name: Trigger worker refresh
         env:
           REGISTRY_REFRESH_TOKEN: ${{ secrets.REGISTRY_REFRESH_TOKEN }}

plugins-index.json

@@ -0,0 +1 @@
+[{"name":"auto-summarizer","version":"0.1.0","description":"Maintains a running conversation summary to help agents handle long conversations without losing context"},{"name":"context-decay","version":"0.1.0","description":"Time-based memory decay with relevance scoring for natural context forgetting"},{"name":"conversation-logger","version":"0.1.0","description":"Logs all conversations to JSONL files for auditing, analytics, and debugging"},{"name":"episodic-memory","version":"0.1.0","description":"Episode-based memory segmentation and recall for cross-conversation context continuity"},{"name":"guardrails","version":"0.1.0","description":"Safety filter that detects potentially harmful content patterns and injects warnings into agent context"},{"name":"keyword-memory","version":"0.1.0","description":"Extracts keywords and named entities from user messages and returns them as contextual memories"},{"name":"mempalace-indexer","version":"0.3.0","description":"Auto-index conversations into MemPalace and recall relevant memories. No API keys, no cloud."},{"name":"sentiment-tracker","version":"0.1.0","description":"Analyzes user message sentiment and injects emotional context so agents can respond with appropriate tone"},{"name":"todo-tracker","version":"0.1.0","description":"Detects action items and tasks mentioned in conversations, persists them, and recalls them as context"},{"name":"topic-memory","version":"0.1.0","description":"Topic-aware memory recall with keyword clustering for cross-conversation context"},{"name":"user-profile","version":"0.1.0","description":"Persistent user profiling from conversation patterns for personalized agent responses"}]

scripts/build-plugins-index.mjs

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+// Build plugins-index.json from the on-disk plugins/ tree.
+//
+// Walks plugins/<name>/plugin.toml, extracts the fields the LibreFang
+// daemon actually consumes (name, version?, description?, needs?), sorts
+// the result by name for byte-determinism, and writes it to the repo
+// root. The committed file is what the registry-worker forced-refresh
+// path signs — keeping the worker's input to a single subrequest
+// regardless of how large the registry grows.
+//
+// Run by .github/workflows/refresh-cache.yml on every push under
+// plugins/**, and committed back via github-actions[bot].
+
+import fs from 'node:fs'
+import path from 'node:path'
+
+const repoRoot = path.resolve(new URL('.', import.meta.url).pathname, '..')
+const pluginsDir = path.join(repoRoot, 'plugins')
+const outPath = path.join(repoRoot, 'plugins-index.json')
+
+function pickString(text, key) {
+  const m = text.match(new RegExp(`^${key}\\s*=\\s*"([^"]*)"`, 'm'))
+  return m ? m[1] : ''
+}
+
+function pickStringArray(text, key) {
+  const m = text.match(new RegExp(`^${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'm'))
+  if (!m) return undefined
+  const items = m[1].match(/"([^"]*)"/g)?.map(s => s.replace(/"/g, ''))
+  return items?.length ? items : undefined
+}
+
+const entries = []
+for (const dir of fs.readdirSync(pluginsDir, { withFileTypes: true })) {
+  if (!dir.isDirectory()) continue
+  const manifest = path.join(pluginsDir, dir.name, 'plugin.toml')
+  if (!fs.existsSync(manifest)) continue
+  const text = fs.readFileSync(manifest, 'utf8')
+
+  const name = pickString(text, 'name')
+  if (!name) continue
+
+  const out = { name }
+  const version = pickString(text, 'version')
+  if (version) out.version = version
+  const description = pickString(text, 'description')
+  if (description) out.description = description
+  const needs = pickStringArray(text, 'needs')
+  if (needs?.length) out.needs = needs
+  entries.push(out)
+}
+
+entries.sort((a, b) => a.name.localeCompare(b.name))
+
+const json = JSON.stringify(entries)
+const prev = fs.existsSync(outPath) ? fs.readFileSync(outPath, 'utf8') : null
+if (prev === json) {
+  console.log(`plugins-index.json unchanged (${entries.length} entries)`)
+  process.exit(0)
+}
+fs.writeFileSync(outPath, json)
+console.log(`plugins-index.json updated: ${entries.length} entries`)