Evan
21c82e335c
Community-maintained TOML catalog for LibreFang. New models can be added via PR without requiring a LibreFang binary release. Includes validation script, bilingual docs, and GitHub templates.
234 lines
7.6 KiB
Python
Executable File
234 lines
7.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Validate all TOML files in the model catalog against the schema.
|
|
|
|
Usage:
|
|
python scripts/validate.py
|
|
|
|
Checks:
|
|
- All provider TOML files parse correctly
|
|
- Required provider fields exist
|
|
- Required model fields exist for each [[models]] entry
|
|
- Tier values are one of: frontier, smart, balanced, fast, local
|
|
- Cost values are non-negative
|
|
- context_window and max_output_tokens are positive integers
|
|
- No duplicate model IDs within the same provider file
|
|
- No duplicate model IDs across ALL provider files (same provider)
|
|
- aliases.toml parses correctly
|
|
|
|
Exit code 0 on success, 1 on any validation error.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import tomllib # Python 3.11+
|
|
except ImportError:
|
|
try:
|
|
import tomli as tomllib # pip install tomli
|
|
except ImportError:
|
|
print("ERROR: Python 3.11+ required, or install 'tomli': pip install tomli")
|
|
sys.exit(1)
|
|
|
|
VALID_TIERS = {"frontier", "smart", "balanced", "fast", "local"}
|
|
|
|
REQUIRED_PROVIDER_FIELDS = {"id", "display_name", "api_key_env", "base_url", "key_required"}
|
|
|
|
REQUIRED_MODEL_FIELDS = {
|
|
"id",
|
|
"display_name",
|
|
"tier",
|
|
"context_window",
|
|
"max_output_tokens",
|
|
"input_cost_per_m",
|
|
"output_cost_per_m",
|
|
}
|
|
|
|
|
|
def validate_provider_file(filepath: Path) -> list[str]:
|
|
"""Validate a single provider TOML file. Returns list of error messages."""
|
|
errors = []
|
|
|
|
try:
|
|
with open(filepath, "rb") as f:
|
|
data = tomllib.load(f)
|
|
except Exception as e:
|
|
return [f"{filepath.name}: Failed to parse TOML: {e}"]
|
|
|
|
# Validate [provider] section
|
|
provider = data.get("provider")
|
|
if provider is None:
|
|
errors.append(f"{filepath.name}: Missing [provider] section")
|
|
else:
|
|
for field in REQUIRED_PROVIDER_FIELDS:
|
|
if field not in provider:
|
|
errors.append(f"{filepath.name}: Missing provider field '{field}'")
|
|
|
|
# Validate [[models]] entries
|
|
models = data.get("models", [])
|
|
seen_ids = set()
|
|
|
|
for i, model in enumerate(models):
|
|
model_label = model.get("id", f"models[{i}]")
|
|
|
|
# Check required fields
|
|
for field in REQUIRED_MODEL_FIELDS:
|
|
if field not in model:
|
|
errors.append(f"{filepath.name}: Model '{model_label}' missing field '{field}'")
|
|
|
|
# Validate tier
|
|
tier = model.get("tier")
|
|
if tier is not None and tier not in VALID_TIERS:
|
|
errors.append(
|
|
f"{filepath.name}: Model '{model_label}' has invalid tier '{tier}' "
|
|
f"(must be one of: {', '.join(sorted(VALID_TIERS))})"
|
|
)
|
|
|
|
# Validate costs
|
|
for cost_field in ("input_cost_per_m", "output_cost_per_m"):
|
|
val = model.get(cost_field)
|
|
if val is not None and (not isinstance(val, (int, float)) or val < 0):
|
|
errors.append(
|
|
f"{filepath.name}: Model '{model_label}' has invalid {cost_field}: {val} "
|
|
f"(must be >= 0)"
|
|
)
|
|
|
|
# Validate context_window and max_output_tokens
|
|
for int_field in ("context_window", "max_output_tokens"):
|
|
val = model.get(int_field)
|
|
if val is not None and (not isinstance(val, int) or val <= 0):
|
|
errors.append(
|
|
f"{filepath.name}: Model '{model_label}' has invalid {int_field}: {val} "
|
|
f"(must be a positive integer)"
|
|
)
|
|
|
|
# Check for duplicate IDs within the file
|
|
model_id = model.get("id")
|
|
if model_id:
|
|
if model_id in seen_ids:
|
|
errors.append(
|
|
f"{filepath.name}: Duplicate model ID '{model_id}' within file"
|
|
)
|
|
seen_ids.add(model_id)
|
|
|
|
return errors
|
|
|
|
|
|
def validate_aliases_file(filepath: Path) -> list[str]:
|
|
"""Validate aliases.toml. Returns list of error messages."""
|
|
if not filepath.exists():
|
|
return [f"aliases.toml: File not found at {filepath}"]
|
|
|
|
try:
|
|
with open(filepath, "rb") as f:
|
|
data = tomllib.load(f)
|
|
except Exception as e:
|
|
return [f"aliases.toml: Failed to parse TOML: {e}"]
|
|
|
|
errors = []
|
|
aliases = data.get("aliases", {})
|
|
if not isinstance(aliases, dict):
|
|
errors.append("aliases.toml: [aliases] must be a table of string -> string mappings")
|
|
else:
|
|
for alias, target in aliases.items():
|
|
if not isinstance(target, str):
|
|
errors.append(f"aliases.toml: Alias '{alias}' must map to a string, got {type(target).__name__}")
|
|
|
|
return errors
|
|
|
|
|
|
def main():
|
|
# Find the catalog root
|
|
script_dir = Path(__file__).resolve().parent
|
|
catalog_root = script_dir.parent
|
|
providers_dir = catalog_root / "providers"
|
|
|
|
if not providers_dir.is_dir():
|
|
print(f"ERROR: providers/ directory not found at {providers_dir}")
|
|
sys.exit(1)
|
|
|
|
all_errors = []
|
|
total_models = 0
|
|
provider_counts = {}
|
|
global_model_ids = {} # model_id -> (provider_id, filename)
|
|
|
|
# Validate each provider file
|
|
toml_files = sorted(providers_dir.glob("*.toml"))
|
|
if not toml_files:
|
|
print("ERROR: No .toml files found in providers/")
|
|
sys.exit(1)
|
|
|
|
for filepath in toml_files:
|
|
errors = validate_provider_file(filepath)
|
|
all_errors.extend(errors)
|
|
|
|
# Count models if file parsed successfully
|
|
if not any("Failed to parse" in e for e in errors):
|
|
try:
|
|
with open(filepath, "rb") as f:
|
|
data = tomllib.load(f)
|
|
provider_id = data.get("provider", {}).get("id", filepath.stem)
|
|
models = data.get("models", [])
|
|
count = len(models)
|
|
total_models += count
|
|
provider_counts[provider_id] = count
|
|
|
|
# Track global model IDs for cross-file duplicate detection
|
|
for model in models:
|
|
mid = model.get("id")
|
|
mprov = model.get("provider", provider_id)
|
|
if mid:
|
|
key = (mid, mprov)
|
|
if key in global_model_ids:
|
|
prev_file = global_model_ids[key]
|
|
# Only flag if same provider in different files
|
|
if prev_file != filepath.name:
|
|
all_errors.append(
|
|
f"Cross-file duplicate: model '{mid}' (provider '{mprov}') "
|
|
f"found in both {prev_file} and {filepath.name}"
|
|
)
|
|
else:
|
|
global_model_ids[key] = filepath.name
|
|
except Exception:
|
|
pass
|
|
|
|
# Validate aliases.toml
|
|
aliases_file = catalog_root / "aliases.toml"
|
|
all_errors.extend(validate_aliases_file(aliases_file))
|
|
|
|
# Print results
|
|
print("=" * 60)
|
|
print("LibreFang Model Catalog Validation")
|
|
print("=" * 60)
|
|
print()
|
|
|
|
if all_errors:
|
|
print(f"ERRORS ({len(all_errors)}):")
|
|
for error in all_errors:
|
|
print(f" - {error}")
|
|
print()
|
|
|
|
print(f"Provider files: {len(toml_files)}")
|
|
print(f"Total models: {total_models}")
|
|
print()
|
|
|
|
print("Per-provider model counts:")
|
|
for provider_id in sorted(provider_counts.keys()):
|
|
count = provider_counts[provider_id]
|
|
if count > 0:
|
|
print(f" {provider_id:30s} {count:>3}")
|
|
|
|
print()
|
|
|
|
if all_errors:
|
|
print(f"VALIDATION FAILED with {len(all_errors)} error(s)")
|
|
sys.exit(1)
|
|
else:
|
|
print("VALIDATION PASSED")
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|