Skip to content

Utils

neps.optimizers.multi_fidelity_prior.utils #

compute_config_dist #

compute_config_dist(
    config1: SearchSpace, config2: SearchSpace
) -> float

Computes distance between two configurations.

Divides the search space into continuous and categorical subspaces. Normalizes all the continuous values while gives numerical encoding to categories. Distance returned is the sum of the Euclidean distance of the continous subspace and the Hamming distance of the categorical subspace.

Source code in neps/optimizers/multi_fidelity_prior/utils.py 
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
def compute_config_dist(config1: SearchSpace, config2: SearchSpace) -> float:
    """Computes distance between two configurations.

    Divides the search space into continuous and categorical subspaces.
    Normalizes all the continuous values while gives numerical encoding to categories.
    Distance returned is the sum of the Euclidean distance of the continous subspace and
    the Hamming distance of the categorical subspace.
    """
    config1 = config1.get_normalized_hp_categories(ignore_fidelity=True)
    config2 = config2.get_normalized_hp_categories(ignore_fidelity=True)

    # adding a dim with 0 to all subspaces in case the search space is not mixed type

    # computing euclidean distance over the continuous subspace
    diff = np.array(config1["continuous"] + [0]) - np.array(config2["continuous"] + [0])
    d_cont = np.linalg.norm(diff, ord=2)

    # TODO: can we consider the number of choices per dimension
    # computing hamming distance over the categorical subspace
    d_cat = scipy.spatial.distance.hamming(
        config1["categorical"] + [0], config2["categorical"] + [0]
    )

    distance = d_cont + d_cat
    return distance

compute_scores #

compute_scores(
    config: SearchSpace,
    prior: SearchSpace,
    inc: SearchSpace,
) -> Union[float, float]

Scores the config by a Gaussian around the prior and the incumbent.

Source code in neps/optimizers/multi_fidelity_prior/utils.py 
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def compute_scores(
    config: SearchSpace, prior: SearchSpace, inc: SearchSpace
) -> Union[float, float]:
    """Scores the config by a Gaussian around the prior and the incumbent."""
    _prior = deepcopy(prior)
    _prior.set_hyperparameters_from_dict(config.hp_values(), defaults=False)
    # compute the score of config if it was sampled from the prior (as the default)
    prior_score = _prior.compute_prior()

    _inc = deepcopy(inc)
    # setting the default to be the incumbent
    _inc.set_defaults_to_current_values()
    _inc.set_hyperparameters_from_dict(config.hp_values(), defaults=False)
    # compute the score of config if it was sampled from the inc (as the default)
    inc_score = _inc.compute_prior()

    return prior_score, inc_score

custom_crossover #

custom_crossover(
    config1: SearchSpace,
    config2: SearchSpace,
    crossover_prob: float = 0.5,
    patience: int = 50,
) -> SearchSpace

Performs a crossover of config2 into config1.

Returns a configuration where each HP in config1 has crossover_prob% chance of getting config2's value of the corresponding HP. By default, crossover rate is 50%.

Source code in neps/optimizers/multi_fidelity_prior/utils.py 
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
def custom_crossover(
    config1: SearchSpace,
    config2: SearchSpace,
    crossover_prob: float = 0.5,
    patience: int = 50,
) -> SearchSpace:
    """Performs a crossover of config2 into config1.

    Returns a configuration where each HP in config1 has `crossover_prob`% chance of
    getting config2's value of the corresponding HP. By default, crossover rate is 50%.
    """
    for _ in range(patience):
        child_config = deepcopy(config1)
        for key, hyperparameter in config1.items():
            if not hyperparameter.is_fidelity and np.random.random() < crossover_prob:
                # crossing over config2 values into config1
                child_config[key].value = config2[key].value
        if not child_config.is_equal_value(config1):
            # breaks and returns only if the generated config is not the same
            # thus requiring at least 1 HP to be crossed-over
            break
    else:
        # fail safe check to handle edge cases where config1=config2 or
        # config1 extremely local to config2 such that crossover fails to
        # generate new config in a discrete (sub-)space
        child_config = config1.sample(
            patience=patience, user_priors=False, ignore_fidelity=True
        )
    return child_config

get_prior_weight_for_decay #

get_prior_weight_for_decay(
    resources_used: float, eta: int, min_budget, max_budget
) -> float

Creates a step function schedule for the prior weight decay.

The prior weight ratio is decayed every time the total resources used is equivalent to the cost of one successive halving bracket within the HB schedule. This is approximately eta imes max_budget resources for one evaluation.

Source code in neps/optimizers/multi_fidelity_prior/utils.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
def get_prior_weight_for_decay(
    resources_used: float, eta: int, min_budget, max_budget
) -> float:
    """Creates a step function schedule for the prior weight decay.

    The prior weight ratio is decayed every time the total resources used is
    equivalent to the cost of one successive halving bracket within the HB schedule.
    This is approximately eta \times max_budget resources for one evaluation.
    """
    # decay factor for the prior
    decay = 2
    unit_resources = eta * max_budget
    idx = resources_used // unit_resources
    weight = 1 / decay**idx
    return weight

local_mutation #

local_mutation(
    config: SearchSpace,
    std: float = 0.25,
    mutation_rate: float = 0.5,
    patience: int = 50,
) -> SearchSpace

Performs a local search by mutating randomly chosen hyperparameters.

Source code in neps/optimizers/multi_fidelity_prior/utils.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def local_mutation(
    config: SearchSpace, std: float = 0.25, mutation_rate: float = 0.5, patience: int = 50
) -> SearchSpace:
    """Performs a local search by mutating randomly chosen hyperparameters."""
    for _ in range(patience):
        new_config = deepcopy(config)
        config_hp_names = list(new_config.keys())
        for hp_name in config_hp_names:
            hp = new_config.get(hp_name)
            if hp.is_fidelity or np.random.uniform() > mutation_rate:
                continue
            kwargs = {"mutation_strategy": "local_search"}
            if isinstance(hp, CategoricalParameter):
                confidences = {hp.value: len(hp.choices)}
                kwargs["confidences"] = confidences
            elif isinstance(hp, IntegerParameter) or isinstance(hp, FloatParameter):
                kwargs["std"] = std
            _val = hp.mutate(**kwargs).value
            hp.value = _val
        if not config.is_equal_value(new_config, include_fidelity=False):
            # if the new config doesn't differ from the original config then regenerate
            break

    return new_config