Metric

A Metric to let optimizers know how to handle numeric values properly.

A Metric is defined by a .name: str and whether it is better to .minimize: bool the metric. Further, you can specify .bounds: tuple[lower, upper] which can help optimizers and other code know how to treat metrics.

To easily convert between loss, score of a a value in a Metric.Value object.

If the metric is bounded, you can also get the distance_to_optimal which is the distance to the optimal value.

from amltk.optimization import Metric

acc = Metric("accuracy", minimize=False, bounds=(0.0, 1.0))

acc_value = acc.as_value(0.9)
print(f"Cost: {acc_value.distance_to_optimal}")  # Distance to optimal.
print(f"Loss: {acc_value.loss}")  # Something that can be minimized
print(f"Score: {acc_value.score}")  # Something that can be maximized

Cost: 0.09999999999999998
Loss: -0.9
Score: 0.9

class Metric
dataclass

A metric with a given name, optimal direction, and possible bounds.

name: str
attr

The name of the metric.

minimize: bool
classvar attr

Whether to minimize or maximize the metric.

bounds: tuple[float, float] | None
classvar attr

The bounds of the metric, if any.

worst: Metric.Value
prop

The worst possible value of the metric.

optimal: Metric.Value
prop

The optimal value of the metric.

class Value
dataclass

A recorded value of an metric.

metric: Metric
classvar attr

The metric.

value: float
classvar attr

The value of the metric.

minimize: bool
prop

Whether to minimize or maximize the metric.

bounds: tuple[float, float] | None
prop

Whether to minimize or maximize the metric.

name: str
prop

The name of the metric.

loss: float
prop

Convert a value to a loss.

score: float
prop

Convert a value to a score.

distance_to_optimal: float | None
prop

The distance to the optimal value, using the bounds if possible.

def __float__()

Convert a value to a float.

Source code in src/amltk/optimization/metric.py

def __float__(self) -> float:
    """Convert a value to a float."""
    return float(self.value)

def __eq__(__value)

Check if two values are equal.

Source code in src/amltk/optimization/metric.py

@override
def __eq__(self, __value: object) -> bool:
    """Check if two values are equal."""
    if isinstance(__value, Metric.Value):
        return self.value == __value.value
    if isinstance(__value, float | int):
        return self.value == float(__value)
    return NotImplemented

def from_str(s)
classmethod

Create an metric from a str.

from amltk.optimization import Metric

s = "loss (minimize)"
metric = Metric.from_str(s)
print(metric)

s = "accuracy [0.0, 1.0] (maximize)"
metric = Metric.from_str(s)
print(metric)

loss (minimize)
accuracy [0.0, 1.0] (maximize)

PARAMETER	DESCRIPTION
s	The string to parse. TYPE: str

RETURNS	DESCRIPTION
Self	The parsed metric.

Source code in src/amltk/optimization/metric.py

@classmethod
def from_str(cls, s: str) -> Self:
    """Create an metric from a str.

    ```python exec="true" source="material-block" result="python"
    from amltk.optimization import Metric

    s = "loss (minimize)"
    metric = Metric.from_str(s)
    print(metric)

    s = "accuracy [0.0, 1.0] (maximize)"
    metric = Metric.from_str(s)
    print(metric)
    ```

    Args:
        s: The string to parse.

    Returns:
        The parsed metric.
    """
    splits = s.split(" ")
    # No bounds
    if len(splits) == 2:  # noqa: PLR2004
        name, minimize_str = splits
        bounds = None
    else:
        name, lower_str, upper_str, minimize_str = splits
        bounds = (float(lower_str[1:-1]), float(upper_str[:-1]))

    minimize = minimize_str == "(minimize)"
    return cls(name=name, minimize=minimize, bounds=bounds)

def as_value(value)

Convert a value to an metric value.

Source code in src/amltk/optimization/metric.py

def as_value(self, value: float | int) -> Metric.Value:
    """Convert a value to an metric value."""
    return Metric.Value(metric=self, value=float(value))

def __call__(value)

Convert a value to an metric value.

Source code in src/amltk/optimization/metric.py

def __call__(self, value: float | int) -> Metric.Value:
    """Convert a value to an metric value."""
    return Metric.Value(metric=self, value=float(value))