get

def get(name, *, prior=None, preload=False, **kwargs)

Get a benchmark.

PARAMETER	DESCRIPTION
name	The name of the benchmark TYPE: str
prior	The prior to use for the benchmark. * str - If it ends in {.json} or {.yaml, .yml}, it will convert it to a path and use it as if it is a path to a config. Otherwise, it is treated as preset * Path - path to a file * Config - A Config object * None - Use the default if available TYPE: str | Path | Config | None DEFAULT: None
preload	Whether to preload the benchmark data in TYPE: bool DEFAULT: False
**kwargs	Extra arguments, optional or required for other benchmarks. Please look up the associated benchmarks. TYPE: Any DEFAULT: {}

For the **kwargs, please see the benchmarks listed below by name=

name='lcbench' (YAHPO-GYM)

Possible task_id=:

('3945', '7593', '34539', '126025', '126026', '126029', '146212', '167104', '167149', '167152', '167161', '167168', '167181', '167184', '167185', '167190', '167200', '167201', '168329', '168330', '168331', '168335', '168868', '168908', '168910', '189354', '189862', '189865', '189866', '189873', '189905', '189906', '189908', '189909')

PARAMETER	DESCRIPTION
task_id	The task id to choose. TYPE: str
seed	The seed to use TYPE: int | None DEFAULT: None
datadir	The path to where mfpbench stores it data. If left to None, will use the _default_download_dir = ./data/yahpo-gym-data. TYPE: str | Path | None DEFAULT: None
seed	The seed for the benchmark instance TYPE: int | None DEFAULT: None
prior	The prior to use for the benchmark. If None, no prior is used. If a str, will check the local location first for a prior specific for this benchmark, otherwise assumes it to be a Path. If a Path, will load the prior from the path. If a Mapping, will be used directly. TYPE: str | Path | C | Mapping[str, Any] | None DEFAULT: None
perturb_prior	If given, will perturb the prior by this amount. Only used if prior= is given as a config. TYPE: float | None DEFAULT: None
session	The onnxruntime session to use. If None, will create a new one. Not for faint hearted This is only a backdoor for onnx compatibility issues with YahpoGym. You are advised not to use this unless you know what you are doing. TYPE: InferenceSession | None DEFAULT: None

name='lm1b_transformer_2048' (PD1)

PARAMETER	DESCRIPTION
datadir	Path to the data directory TYPE: str | Path | None DEFAULT: None
seed	The seed to use for the space TYPE: int | None DEFAULT: None
prior	Any prior to use for the benchmark TYPE: str | Path | C | Mapping[str, Any] | None DEFAULT: None
perturb_prior	Whether to perturb the prior. If specified, this is interpreted as the std of a normal from which to perturb numerical hyperparameters of the prior, and the raw probability of swapping a categorical value. TYPE: float | None DEFAULT: None

name='uniref50_transformer_128' (PD1)

PARAMETER	DESCRIPTION
datadir	Path to the data directory TYPE: str | Path | None DEFAULT: None
seed	The seed to use for the space TYPE: int | None DEFAULT: None
prior	Any prior to use for the benchmark TYPE: str | Path | C | Mapping[str, Any] | None DEFAULT: None
perturb_prior	Whether to perturb the prior. If specified, this is interpreted as the std of a normal from which to perturb numerical hyperparameters of the prior, and the raw probability of swapping a categorical value. TYPE: float | None DEFAULT: None

name='cifar100_wideresnet_2048' (PD1)

PARAMETER	DESCRIPTION
datadir	Path to the data directory TYPE: str | Path | None DEFAULT: None
seed	The seed to use for the space TYPE: int | None DEFAULT: None
prior	Any prior to use for the benchmark TYPE: str | Path | C | Mapping[str, Any] | None DEFAULT: None
perturb_prior	Whether to perturb the prior. If specified, this is interpreted as the std of a normal from which to perturb numerical hyperparameters of the prior, and the raw probability of swapping a categorical value. TYPE: float | None DEFAULT: None

name='imagenet_resnet_512' (PD1)

PARAMETER	DESCRIPTION
datadir	Path to the data directory TYPE: str | Path | None DEFAULT: None
seed	The seed to use for the space TYPE: int | None DEFAULT: None
prior	Any prior to use for the benchmark TYPE: str | Path | C | Mapping[str, Any] | None DEFAULT: None
perturb_prior	Whether to perturb the prior. If specified, this is interpreted as the std of a normal from which to perturb numerical hyperparameters of the prior, and the raw probability of swapping a categorical value. TYPE: float | None DEFAULT: None

name='jahs'

Possible task_id=:

('CIFAR10', 'ColorectalHistology', 'FashionMNIST')

PARAMETER	DESCRIPTION
task_id	The specific task to use. TYPE: Literal['CIFAR10', 'ColorectalHistology', 'FashionMNIST']
datadir	The path to where mfpbench stores it data. If left to None, will use _default_download_dir = "./data/jahs-bench-data". TYPE: str | Path | None DEFAULT: None
seed	The seed to give this benchmark instance TYPE: int | None DEFAULT: None
prior	The prior to use for the benchmark. if str - A preset if Path - path to a file if dict, Config, Configuration - A config if None - Use the default if available TYPE: str | Path | JAHSConfig | Mapping[str, Any] | None DEFAULT: None
perturb_prior	If given, will perturb the prior by this amount. Only used if prior= is given as a config. TYPE: float | None DEFAULT: None

name='mfh3'

PARAMETER	DESCRIPTION
seed	The seed to use. TYPE: int | None DEFAULT: None
bias	How much bias to introduce TYPE: float | None DEFAULT: None
noise	How much noise to introduce TYPE: float | None DEFAULT: None
prior	The prior to use for the benchmark. if Path - path to a file if Mapping - Use directly if None - There is no prior TYPE: str | Path | C | Mapping[str, Any] | None DEFAULT: None
perturb_prior	If not None, will perturb the prior by this amount. For numericals, while for categoricals, this is interpreted as the probability of swapping the value for a random one. TYPE: float | None DEFAULT: None

name='mfh6'

PARAMETER	DESCRIPTION
seed	The seed to use. TYPE: int | None DEFAULT: None
bias	How much bias to introduce TYPE: float | None DEFAULT: None
noise	How much noise to introduce TYPE: float | None DEFAULT: None
prior	The prior to use for the benchmark. if Path - path to a file if Mapping - Use directly if None - There is no prior TYPE: str | Path | C | Mapping[str, Any] | None DEFAULT: None
perturb_prior	If not None, will perturb the prior by this amount. For numericals, while for categoricals, this is interpreted as the probability of swapping the value for a random one. TYPE: float | None DEFAULT: None

name='lcbench_tabular'

Possible task_id=:

('adult', 'airlines', 'albert', 'Amazon_employee_access', 'APSFailure', 'Australian', 'bank-marketing', 'blood-transfusion-service-center', 'car', 'christine', 'cnae-9', 'connect-4', 'covertype', 'credit-g', 'dionis', 'fabert', 'Fashion-MNIST', 'helena', 'higgs', 'jannis', 'jasmine', 'jungle_chess_2pcs_raw_endgame_complete', 'kc1', 'KDDCup09_appetency', 'kr-vs-kp', 'mfeat-factors', 'MiniBooNE', 'nomao', 'numerai28.6', 'phoneme', 'segment', 'shuttle', 'sylvine', 'vehicle', 'volkert')

PARAMETER	DESCRIPTION
task_id	The task to benchmark on. TYPE: str
datadir	The directory to look for the data in. If None, uses the default download directory. TYPE: str | Path | None DEFAULT: None
remove_constants	Whether to remove constant config columns from the data or not. TYPE: bool DEFAULT: False
seed	The seed to use. TYPE: int | None DEFAULT: None
prior	The prior to use for the benchmark. If None, no prior is used. If a str, will check the local location first for a prior specific for this benchmark, otherwise assumes it to be a Path. If a Path, will load the prior from the path. If a Mapping, will be used directly. TYPE: str | Path | LCBenchTabularConfig | Mapping[str, Any] | None DEFAULT: None
perturb_prior	If not None, will perturb the prior by this amount. For numericals, this is interpreted as the standard deviation of a normal distribution while for categoricals, this is interpreted as the probability of swapping the value for a random one. TYPE: float | None DEFAULT: None

Source code in src/mfpbench/get.py

def get(
    name: str,
    *,
    prior: str | Path | Config | None = None,
    preload: bool = False,
    **kwargs: Any,
) -> Benchmark:
    """Get a benchmark.

    Args:
        name: The name of the benchmark
        prior: The prior to use for the benchmark.
            * str -
                If it ends in {.json} or {.yaml, .yml}, it will convert it to a path and
                use it as if it is a path to a config. Otherwise, it is treated as preset
            * Path - path to a file
            * Config - A Config object
            * None - Use the default if available
        preload: Whether to preload the benchmark data in
        **kwargs: Extra arguments, optional or required for other benchmarks. Please
            look up the associated benchmarks.

    For the `#!python **kwargs`, please see the benchmarks listed below by `name=`

    ??? note "`#!python name='lcbench'` (YAHPO-GYM)"

        Possible `#!python task_id=`:

        ::: mfpbench.LCBenchBenchmark.yahpo_instances

        ::: mfpbench.LCBenchBenchmark.__init__
            options:
                show_source: false

    ??? note "`#!python name='lm1b_transformer_2048'` (PD1)"

        ::: mfpbench.PD1lm1b_transformer_2048.__init__
            options:
                show_source: false

    ??? note "`#!python name='uniref50_transformer_128'` (PD1)"

        ::: mfpbench.PD1uniref50_transformer_128.__init__
            options:
                show_source: false

    ??? note "`#!python name='cifar100_wideresnet_2048'` (PD1)"

        ::: mfpbench.PD1cifar100_wideresnet_2048.__init__
            options:
                show_source: false

    ??? note "`#!python name='imagenet_resnet_512'` (PD1)"

        ::: mfpbench.PD1imagenet_resnet_512.__init__
            options:
                show_source: false

    ??? note "`#!python name='jahs'`"

        Possible `#!python task_id=`:

        ::: mfpbench.JAHSBenchmark.task_ids

        ::: mfpbench.JAHSBenchmark.__init__
            options:
                show_source: false

    ??? note "`#!python name='mfh3'`"

        ::: mfpbench.MFHartmann3Benchmark.__init__
            options:
                show_source: false

    ??? note "`#!python name='mfh6'`"

        ::: mfpbench.MFHartmann6Benchmark.__init__
            options:
                show_source: false

    ??? note "`#!python name='lcbench_tabular'`"

        Possible `#!python task_id=`:

        ::: mfpbench.LCBenchTabularBenchmark.task_ids

        ::: mfpbench.LCBenchTabularBenchmark.__init__
            options:
                show_source: false


    """  # noqa: E501
    b = _mapping.get(name, None)
    bench: Benchmark

    if b is None:
        raise ValueError(f"{name} is not a benchmark in {list(_mapping.keys())}")

    if isinstance(prior, str) and any(
        prior.endswith(suffix) for suffix in [".json", ".yaml", ".yml"]
    ):
        prior = Path(prior)

    bench = b(prior=prior, **kwargs)

    if preload:
        bench.load()

    return bench