From f9821d5867fd094110ce2711aa9b52107811f594 Mon Sep 17 00:00:00 2001 From: Evan Hu Date: Tue, 5 May 2026 00:37:50 +0900 Subject: [PATCH] ci: build plugins-index.json in-repo so worker refresh stays under budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Walking ~40+ plugin TOMLs via the GitHub Contents API from the worker exceeded the Workers Free 50-subrequest-per-invocation limit, leaving the daemon's signed plugins index either empty or partial after every forced refresh. Move the walk into the repo: scripts/build-plugins-index.mjs reads each plugins//plugin.toml directly from the checked-out tree and emits a sorted flat array (name, version?, description?, needs?) at plugins-index.json. The CI workflow regenerates and commits this file on every push under plugins/, then pokes the worker's /api/registry/refresh — which now fetches the single committed plugins-index.json (1 subrequest), validates the JSON shape, and re-signs it with Ed25519. Refresh cost is now constant in registry size, not linear. The dashboard's dict-shaped /api/registry payload is unchanged — that still rebuilds via the daily 02:00 UTC cron. --- .github/workflows/refresh-cache.yml | 49 ++++++++++++++++------- plugins-index.json | 1 + scripts/build-plugins-index.mjs | 62 +++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 14 deletions(-) create mode 100644 plugins-index.json create mode 100644 scripts/build-plugins-index.mjs diff --git a/.github/workflows/refresh-cache.yml b/.github/workflows/refresh-cache.yml index 51d3f72..71f01d6 100644 --- a/.github/workflows/refresh-cache.yml +++ b/.github/workflows/refresh-cache.yml @@ -1,31 +1,52 @@ name: Refresh registry-worker cache -# Triggers the registry-worker to rebuild its D1 cache and re-sign the -# plugins index right after content changes — without this, dashboard / -# daemon would have to wait for the next 02:00 UTC cron tick. +# On every push that changes plugins/, regenerate plugins-index.json +# (the daemon-shaped flat array the registry-worker signs and the +# librefang daemon installs from), commit it back, then poke the +# worker's forced-refresh endpoint so it re-signs the new bytes +# immediately — without this, daemon installs lag up to ~24h behind +# the next 02:00 UTC cron tick. # -# Auth: REGISTRY_REFRESH_TOKEN (repo secret) is matched against the -# REGISTRY_REFRESH_TOKEN worker secret in librefang-registry. Until both -# are set, the worker endpoint returns 503. +# Walking the on-disk repo (instead of the worker hitting GitHub +# Contents API per-file) keeps the worker's refresh path under the +# Workers Free 50-subrequest budget regardless of registry size. on: push: branches: [main] paths: - 'plugins/**' - - 'agents/**' - - 'skills/**' - - 'hands/**' - - 'channels/**' - - 'providers/**' - - 'workflows/**' - - 'mcp/**' - workflow_dispatch: # manual trigger for ops + - 'scripts/build-plugins-index.mjs' + workflow_dispatch: # manual trigger for ops / first-deploy + +permissions: + contents: write # needed to commit the regenerated index back jobs: refresh: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Rebuild plugins-index.json + run: node scripts/build-plugins-index.mjs + + - name: Commit regenerated index if changed + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add plugins-index.json + if git diff --cached --quiet; then + echo "plugins-index.json already up-to-date" + else + git commit -m "chore: regenerate plugins-index.json" + git push + fi + - name: Trigger worker refresh env: REGISTRY_REFRESH_TOKEN: ${{ secrets.REGISTRY_REFRESH_TOKEN }} diff --git a/plugins-index.json b/plugins-index.json new file mode 100644 index 0000000..749af2e --- /dev/null +++ b/plugins-index.json @@ -0,0 +1 @@ +[{"name":"auto-summarizer","version":"0.1.0","description":"Maintains a running conversation summary to help agents handle long conversations without losing context"},{"name":"context-decay","version":"0.1.0","description":"Time-based memory decay with relevance scoring for natural context forgetting"},{"name":"conversation-logger","version":"0.1.0","description":"Logs all conversations to JSONL files for auditing, analytics, and debugging"},{"name":"episodic-memory","version":"0.1.0","description":"Episode-based memory segmentation and recall for cross-conversation context continuity"},{"name":"guardrails","version":"0.1.0","description":"Safety filter that detects potentially harmful content patterns and injects warnings into agent context"},{"name":"keyword-memory","version":"0.1.0","description":"Extracts keywords and named entities from user messages and returns them as contextual memories"},{"name":"mempalace-indexer","version":"0.3.0","description":"Auto-index conversations into MemPalace and recall relevant memories. No API keys, no cloud."},{"name":"sentiment-tracker","version":"0.1.0","description":"Analyzes user message sentiment and injects emotional context so agents can respond with appropriate tone"},{"name":"todo-tracker","version":"0.1.0","description":"Detects action items and tasks mentioned in conversations, persists them, and recalls them as context"},{"name":"topic-memory","version":"0.1.0","description":"Topic-aware memory recall with keyword clustering for cross-conversation context"},{"name":"user-profile","version":"0.1.0","description":"Persistent user profiling from conversation patterns for personalized agent responses"}] \ No newline at end of file diff --git a/scripts/build-plugins-index.mjs b/scripts/build-plugins-index.mjs new file mode 100644 index 0000000..5d65334 --- /dev/null +++ b/scripts/build-plugins-index.mjs @@ -0,0 +1,62 @@ +#!/usr/bin/env node +// Build plugins-index.json from the on-disk plugins/ tree. +// +// Walks plugins//plugin.toml, extracts the fields the LibreFang +// daemon actually consumes (name, version?, description?, needs?), sorts +// the result by name for byte-determinism, and writes it to the repo +// root. The committed file is what the registry-worker forced-refresh +// path signs — keeping the worker's input to a single subrequest +// regardless of how large the registry grows. +// +// Run by .github/workflows/refresh-cache.yml on every push under +// plugins/**, and committed back via github-actions[bot]. + +import fs from 'node:fs' +import path from 'node:path' + +const repoRoot = path.resolve(new URL('.', import.meta.url).pathname, '..') +const pluginsDir = path.join(repoRoot, 'plugins') +const outPath = path.join(repoRoot, 'plugins-index.json') + +function pickString(text, key) { + const m = text.match(new RegExp(`^${key}\\s*=\\s*"([^"]*)"`, 'm')) + return m ? m[1] : '' +} + +function pickStringArray(text, key) { + const m = text.match(new RegExp(`^${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'm')) + if (!m) return undefined + const items = m[1].match(/"([^"]*)"/g)?.map(s => s.replace(/"/g, '')) + return items?.length ? items : undefined +} + +const entries = [] +for (const dir of fs.readdirSync(pluginsDir, { withFileTypes: true })) { + if (!dir.isDirectory()) continue + const manifest = path.join(pluginsDir, dir.name, 'plugin.toml') + if (!fs.existsSync(manifest)) continue + const text = fs.readFileSync(manifest, 'utf8') + + const name = pickString(text, 'name') + if (!name) continue + + const out = { name } + const version = pickString(text, 'version') + if (version) out.version = version + const description = pickString(text, 'description') + if (description) out.description = description + const needs = pickStringArray(text, 'needs') + if (needs?.length) out.needs = needs + entries.push(out) +} + +entries.sort((a, b) => a.name.localeCompare(b.name)) + +const json = JSON.stringify(entries) +const prev = fs.existsSync(outPath) ? fs.readFileSync(outPath, 'utf8') : null +if (prev === json) { + console.log(`plugins-index.json unchanged (${entries.length} entries)`) + process.exit(0) +} +fs.writeFileSync(outPath, json) +console.log(`plugins-index.json updated: ${entries.length} entries`)