experiments.types

class AnnotatorKind(value, names=<not given>, *values, module=None, qualname=None, type=None, start=1, boundary=None)

Bases: Enum

CODE = 'CODE'

LLM = 'LLM'

class Dataset(id: 'DatasetId', version_id: 'DatasetVersionId', examples: 'Mapping[ExampleId, Example]' = <factory>)

Bases: object

as_dataframe(drop_empty_columns: bool = True) → DataFrame

examples: Mapping[str, Example]

classmethod from_dict(obj: Mapping[str, Any]) → Dataset

id: str

version_id: str

class EvaluationParameters(eval_names: 'FrozenSet[str]', exp_params: 'ExperimentParameters')

Bases: object

eval_names: FrozenSet[str]

exp_params: ExperimentParameters

class EvaluationResult(score: 'Optional[float]' = None, label: 'Optional[str]' = None, explanation: 'Optional[str]' = None, metadata: 'Mapping[str, JSONSerializable]' = <factory>)

Bases: object

explanation: str | None = None

classmethod from_dict(obj: Mapping[str, Any] | None) → EvaluationResult | None

label: str | None = None

metadata: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None]

score: float | None = None

class EvaluationSummary(*args: Any, **kwargs: Any)

Bases: _HasStats

Summary statistics of experiment evaluations.

Users should not instantiate this directly.

classmethod from_eval_runs(params: EvaluationParameters, *eval_runs: ExperimentEvaluationRun | None) → EvaluationSummary

class Example(id: 'ExampleId', updated_at: 'datetime', input: 'Mapping[str, JSONSerializable]' = <factory>, output: 'Mapping[str, JSONSerializable]' = <factory>, metadata: 'Mapping[str, JSONSerializable]' = <factory>)

Bases: object

classmethod from_dict(obj: Mapping[str, Any]) → Example

id: str

input: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None]

metadata: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None]

output: Mapping[str, Dict[str, Any] | List[Any] | str | int | float | bool | None]

updated_at: datetime

class Experiment(id: 'ExperimentId', dataset_id: 'DatasetId', dataset_version_id: 'DatasetVersionId', repetitions: 'int', project_name: 'str')

Bases: object

dataset_id: str

dataset_version_id: str

classmethod from_dict(obj: Mapping[str, Any]) → Experiment

id: str

project_name: str

repetitions: int

class ExperimentEvaluationRun(experiment_run_id: 'ExperimentRunId', start_time: 'datetime', end_time: 'datetime', name: 'str', annotator_kind: 'str', error: 'Optional[str]' = None, result: 'Optional[EvaluationResult]' = None, id: 'str' = <factory>, trace_id: 'Optional[TraceId]' = None)

Bases: object

annotator_kind: str

end_time: datetime

error: str | None = None

experiment_run_id: str

classmethod from_dict(obj: Mapping[str, Any]) → ExperimentEvaluationRun

id: str

name: str

result: EvaluationResult | None = None

start_time: datetime

trace_id: str | None = None

class ExperimentParameters(n_examples: 'int', n_repetitions: 'int' = 1)

Bases: object

property count: int

n_examples: int

n_repetitions: int = 1

class ExperimentRun(start_time: 'datetime', end_time: 'datetime', experiment_id: 'ExperimentId', dataset_example_id: 'ExampleId', repetition_number: 'RepetitionNumber', experiment_run_output: 'ExperimentRunOutput', error: 'Optional[str]' = None, id: 'ExperimentRunId' = <factory>, trace_id: 'Optional[TraceId]' = None)

Bases: object

dataset_example_id: str

end_time: datetime

error: str | None = None

experiment_id: str

experiment_run_output: ExperimentRunOutput

classmethod from_dict(obj: Mapping[str, Any]) → ExperimentRun

id: str

property output: Dict[str, Any] | List[Any] | str | int | float | bool | None

repetition_number: int

start_time: datetime

trace_id: str | None = None

class ExperimentRunOutput(task_output: 'TaskOutput')

Bases: object

classmethod from_dict(obj: Mapping[str, Any] | None) → ExperimentRunOutput

task_output: Dict[str, Any] | List[Any] | str | int | float | bool | None

class RanExperiment(*args: Any, **kwargs: Any)

Bases: Experiment

An experiment that has been run.

Users should not instantiate this object directly.

add(eval_summary: EvaluationSummary, *eval_runs: ExperimentEvaluationRun | None) → RanExperiment

as_dataframe(drop_empty_columns: bool = True) → DataFrame

dataset: Dataset

eval_runs: Tuple[ExperimentEvaluationRun, ...] = ()

eval_summaries: Tuple[EvaluationSummary, ...] = ()

get_evaluations(drop_empty_columns: bool = True) → DataFrame

property info: str

params: ExperimentParameters

runs: Mapping[str, ExperimentRun]

task_summary: TaskSummary

property url: str

class TaskSummary(*args: Any, **kwargs: Any)

Bases: _HasStats

Summary statistics of experiment task executions.

Users should not instantiate this object directly.

classmethod from_task_runs(params: ExperimentParameters, task_runs: Iterable[ExperimentRun | None]) → TaskSummary

class TestCase(example: 'Example', repetition_number: 'RepetitionNumber')

Bases: object

example: Example

repetition_number: int

getrandbits(k) → x.  Generates an int with k random bits.