eval_cli.py

Go to the documentation of this file.

# Copyright (c) 2025-present Aquin Labs Private Limited. All Rights Reserved.
"""aquin eval custom | boundary | suppress | consistency"""
 
import sys
 
# eval subcommand -> canonical CLI verb dispatched through cmd_tool
_EVAL_TOOL_VERBS: dict[str, str] = {
    "custom": "eval",
    "boundary": "boundary-eval",
    "suppress": "suppression-eval",
    "suppression": "suppression-eval",
    "consistency": "consistency-eval",
}
 
 
def _print_help() -> None:
    print("Behavioral and custom evals.")
    print("")
    print("Usage: aquin eval <subcommand> ...")
    print("")
    print("  custom       Your prompts + reference answers (pass/fail gate)")
    print("               aquin eval custom --name <id> --prompts <json>")
    print("                             --reference_answers <json>")
    print("")
    print("  consistency  Same ask, paraphrased - does behavior flip?")
    print("               aquin eval consistency --query <text> --templates <json>")
    print("")
    print("  suppress     Topic avoidance / hedging vs neutral baseline")
    print("               aquin eval suppress --topics <json>")
    print("")
    print("  boundary     Robustness to typos / surface corruption")
    print("               aquin eval boundary --prompts <json>")
    print("")
    print("Docs: https://aquin.app/docs/evals/llm")
 
 
def cmd_eval(args: list[str]) -> None:
    if not args or args[0] in ("-h", "--help", "help"):
        _print_help()
        return
 
    # Old bare form: aquin eval --name ...
    if args[0].startswith("-"):
        print("Use: aquin eval custom ...", file=sys.stderr)
        sys.exit(1)
 
    sub = args[0]
    rest = args[1:]
 
    if sub in _EVAL_TOOL_VERBS:
        from aquin.cli import cmd_tool
 
        cmd_tool(_EVAL_TOOL_VERBS[sub], rest)
        return
 
    print(f"Unknown eval subcommand: {sub}")
    print("Run: aquin eval --help")
    sys.exit(1)