evals.py File Reference

Go to the source code of this file.

Namespaces
namespace	aquin
namespace	aquin.compute
namespace	aquin.compute.evals

Functions
torch.Tensor	aquin.compute.evals._get_output_distribution (str prompt, HookedTransformer model)
float	aquin.compute.evals._kl_div (torch.Tensor p, torch.Tensor q)
float	aquin.compute.evals._entropy (torch.Tensor p)
list[dict]	aquin.compute.evals._top_tokens (torch.Tensor dist, HookedTransformer model, int k=5)
int	aquin.compute.evals._response_length (str prompt, HookedTransformer model, int max_tokens=80)
str	aquin.compute.evals._decode_response (str prompt, HookedTransformer model, int max_tokens=80)
float	aquin.compute.evals._hedge_score (str text)
dict	aquin.compute.evals.consistency_eval (str query, HookedTransformer model, list[str]|None templates=None)
dict	aquin.compute.evals.suppression_eval (HookedTransformer model, dict[str, list[str]]|None topics=None, int max_response_tokens=60)
str	aquin.compute.evals._corrupt_prompt (str prompt, str corruption_type, HookedTransformer model)
dict	aquin.compute.evals.boundary_eval (list[str] prompts, HookedTransformer model)
list[str]	aquin.compute.evals._tokenize_words (str text)
float	aquin.compute.evals.keyword_overlap_score (str response, str reference)
dict	aquin.compute.evals.custom_eval (str name, list[str] prompts, str model_id, *, list[str]|None reference_answers=None, float threshold=0.5, int max_tokens=40, float temperature=0.0, str|None description=None)

Variables
	aquin.compute.evals.DEVICE = resolve_compute_device()
	aquin.compute.evals.HEDGE_PATTERNS
list	aquin.compute.evals.PARAPHRASE_TEMPLATES
dict	aquin.compute.evals.SUPPRESSION_TOPICS
list	aquin.compute.evals.NEUTRAL_BASELINE_TOPICS
list	aquin.compute.evals.CORRUPTION_TYPES = ["shuffle_tail", "drop_last", "repeat_last", "reverse_tail"]