experiments.evaluators.base

class CodeEvaluator(*args: Any, **kwargs: Any)

Bases: Evaluator, ABC

A convenience super class for defining code evaluators.

This class is intended to be subclassed, and should not be instantiated directly.

class Evaluator(*args: Any, **kwargs: Any)

Bases: ABC

A helper super class to guide the implementation of an Evaluator object. Subclasses must implement either the evaluate or async_evaluate method. Implementing both methods is recommended, but not required.

This Class is intended to be subclassed, and should not be instantiated directly.

async async_evaluate(*, output: Dict[str, Any] | List[Any] | str | int | float | bool | None = None, expected: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None] | None = None, metadata: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None] = mappingproxy({}), input: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None] = mappingproxy({}), **kwargs: Any) → EvaluationResult

evaluate(*, output: Dict[str, Any] | List[Any] | str | int | float | bool | None = None, expected: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None] | None = None, metadata: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None] = mappingproxy({}), input: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None] = mappingproxy({}), **kwargs: Any) → EvaluationResult

property kind: str

property name: str

class LLMEvaluator(*args: Any, **kwargs: Any)

Bases: Evaluator, ABC

A convenience super class for defining LLM evaluators.

This class is intended to be subclassed, and should not be instantiated directly.