benchmark

class JAHSConfig
dataclass

Bases: Config

The config for JAHSBench, useful to have regardless of the configspace used.

github.com/automl/jahs_bench_201/blob/main/jahs_bench/lib/core/configspace.py

def validate()

Validate this config incase required.

Source code in src/mfpbench/jahs/benchmark.py

def validate(self) -> None:
    """Validate this config incase required."""
    # Just being explicit to catch bugs easily, we can remove later
    assert self.N in [1, 3, 5]
    assert self.W in [4, 8, 16]
    assert self.Op1 in [0, 1, 2, 3, 4, 5]
    assert self.Op2 in [0, 1, 2, 3, 4, 5]
    assert self.Op3 in [0, 1, 2, 3, 4, 5]
    assert self.Op4 in [0, 1, 2, 3, 4, 5]
    assert self.Op5 in [0, 1, 2, 3, 4, 5]
    assert self.Op6 in [0, 1, 2, 3, 4, 5]
    assert self.Resolution in [0.25, 0.5, 1.0]
    assert isinstance(self.TrivialAugment, bool)
    assert self.Activation in ["ReLU", "Hardswish", "Mish"]
    assert self.Optimizer in ["SGD"]
    assert 1e-3 <= self.LearningRate <= 1e0
    assert 1e-5 <= self.WeightDecay <= 1e-2

class JAHSResult
dataclass

Bases: Result[JAHSConfig, int]

score: float
prop

The score of interest.

error: float
prop

The error of interest.

test_score: float
prop

The score on the test set.

test_error: float
prop

The error on the test set.

val_score: float
prop

The score on the validation set.

val_error: float
prop

The error on the validation set.

cost: float
prop

The time taken (assumed to be seconds).

class JAHSBenchmark(task_id, *, datadir=None, seed=None, prior=None, perturb_prior=None)

Bases: Benchmark[JAHSConfig, JAHSResult, int], ABC

PARAMETER	DESCRIPTION
task_id	The specific task to use. TYPE: Literal['CIFAR10', 'ColorectalHistology', 'FashionMNIST']
datadir	The path to where mfpbench stores it data. If left to None, will use _default_download_dir = "./data/jahs-bench-data". TYPE: str | Path | None DEFAULT: None
seed	The seed to give this benchmark instance TYPE: int | None DEFAULT: None
prior	The prior to use for the benchmark. if str - A preset if Path - path to a file if dict, Config, Configuration - A config if None - Use the default if available TYPE: str | Path | JAHSConfig | Mapping[str, Any] | None DEFAULT: None
perturb_prior	If given, will perturb the prior by this amount. Only used if prior= is given as a config. TYPE: float | None DEFAULT: None

Source code in src/mfpbench/jahs/benchmark.py

def __init__(
    self,
    task_id: Literal["CIFAR10", "ColorectalHistology", "FashionMNIST"],
    *,
    datadir: str | Path | None = None,
    seed: int | None = None,
    prior: str | Path | JAHSConfig | Mapping[str, Any] | None = None,
    perturb_prior: float | None = None,
):
    """Initialize the benchmark.

    Args:
        task_id: The specific task to use.
        datadir: The path to where mfpbench stores it data. If left to `None`,
            will use `#!python _default_download_dir = "./data/jahs-bench-data"`.
        seed: The seed to give this benchmark instance
        prior: The prior to use for the benchmark.

            * if `str` - A preset
            * if `Path` - path to a file
            * if `dict`, Config, Configuration - A config
            * if `None` - Use the default if available

        perturb_prior: If given, will perturb the prior by this amount.
            Only used if `prior=` is given as a config.
    """
    cls = self.__class__
    if datadir is None:
        datadir = JAHSBenchSource.default_location()

    datadir = Path(datadir)

    if not datadir.exists():
        raise FileNotFoundError(
            f"Can't find folder at {datadir}."
            f"\n`python -m mfpbench download --status --data-dir {datadir}`",
        )

    # Loaded on demand with `@property`
    self._bench: jahs_bench.Benchmark | None = None
    self.datadir = datadir
    self.task_id = task_id

    name = f"jahs_{task_id}"
    super().__init__(
        seed=seed,
        name=name,
        space=cls._jahs_configspace(name=name, seed=seed),
        prior=prior,
        perturb_prior=perturb_prior,
    )

task_ids: tuple[str, ...]
classvar attr

('CIFAR10', 'ColorectalHistology', 'FashionMNIST')

bench: jahs_bench.Benchmark
prop

The underlying benchmark used.

def load()

Pre-load JAHS XGBoost model before querying the first time.

Source code in src/mfpbench/jahs/benchmark.py

def load(self) -> None:
    """Pre-load JAHS XGBoost model before querying the first time."""
    # Access the property
    _ = self.bench