Skip to content

Mf ei

neps.optimizers.bayesian_optimization.acquisition_functions.mf_ei #

MFEI #

MFEI(
    pipeline_space: SearchSpace,
    surrogate_model_name: str = None,
    augmented_ei: bool = False,
    xi: float = 0.0,
    in_fill: str = "best",
    log_ei: bool = False,
)

Bases: ComprehensiveExpectedImprovement

Source code in neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
def __init__(
    self,
    pipeline_space: SearchSpace,
    surrogate_model_name: str = None,
    augmented_ei: bool = False,
    xi: float = 0.0,
    in_fill: str = "best",
    log_ei: bool = False,
):
    super().__init__(augmented_ei, xi, in_fill, log_ei)
    self.pipeline_space = pipeline_space
    self.surrogate_model_name = surrogate_model_name
    self.surrogate_model = None
    self.observations = None
    self.b_step = None

eval_gp_ei #

eval_gp_ei(
    x: Iterable, inc_list: Iterable
) -> Union[ndarray, Tensor, float]

Vanilla-EI modified to preprocess samples and accept list of incumbents.

Source code in neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py 
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
def eval_gp_ei(
    self, x: Iterable, inc_list: Iterable
) -> Union[np.ndarray, torch.Tensor, float]:
    """Vanilla-EI modified to preprocess samples and accept list of incumbents."""
    # x, inc_list = self.preprocess(x)  # IMPORTANT change from vanilla-EI
    _x = x.copy()
    try:
        mu, cov = self.surrogate_model.predict(_x)
    except ValueError as e:
        raise e
        # return -1.0  # in case of error. return ei of -1
    std = torch.sqrt(torch.diag(cov))

    mu_star = inc_list.to(mu.device)  # IMPORTANT change from vanilla-EI

    gauss = Normal(torch.zeros(1, device=mu.device), torch.ones(1, device=mu.device))
    # u = (mu - mu_star - self.xi) / std
    # ei = std * updf + (mu - mu_star - self.xi) * ucdf
    if self.log_ei:
        # we expect that f_min is in log-space
        f_min = mu_star - self.xi
        v = (f_min - mu) / std
        ei = torch.exp(f_min) * gauss.cdf(v) - torch.exp(
            0.5 * torch.diag(cov) + mu
        ) * gauss.cdf(v - std)
    else:
        u = (mu_star - mu - self.xi) / std
        ucdf = gauss.cdf(u)
        updf = torch.exp(gauss.log_prob(u))
        ei = std * updf + (mu_star - mu - self.xi) * ucdf
    if self.augmented_ei:
        sigma_n = self.surrogate_model.likelihood
        ei *= 1.0 - torch.sqrt(torch.tensor(sigma_n, device=mu.device)) / torch.sqrt(
            sigma_n + torch.diag(cov)
        )
    return ei

eval_pfn_ei #

eval_pfn_ei(
    x: Iterable, inc_list: Iterable
) -> Union[ndarray, Tensor, float]

PFN-EI modified to preprocess samples and accept list of incumbents.

Source code in neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py 
144
145
146
147
148
149
150
151
152
153
def eval_pfn_ei(
    self, x: Iterable, inc_list: Iterable
) -> Union[np.ndarray, torch.Tensor, float]:
    """PFN-EI modified to preprocess samples and accept list of incumbents."""
    # x, inc_list = self.preprocess(x)  # IMPORTANT change from vanilla-EI
    # _x = x.copy()
    ei = self.surrogate_model.get_ei(x.to(self.surrogate_model.device), inc_list)
    if len(ei.shape) == 2:
        ei = ei.flatten()
    return ei

preprocess #

preprocess(x: Series) -> Tuple[Iterable, Iterable]

Prepares the configurations for appropriate EI calculation.

Takes a set of points and computes the budget and incumbent for each point, as required by the multi-fidelity Expected Improvement acquisition function.

Source code in neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py 
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def preprocess(self, x: pd.Series) -> Tuple[Iterable, Iterable]:
    """Prepares the configurations for appropriate EI calculation.

    Takes a set of points and computes the budget and incumbent for each point, as
    required by the multi-fidelity Expected Improvement acquisition function.
    """
    budget_list = []

    if self.pipeline_space.has_tabular:
        # preprocess tabular space differently
        # expected input: IDs pertaining to the tabular data
        # expected output: IDs pertaining to current observations and set of HPs
        x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space)
    indices_to_drop = []
    for i, config in x.items():
        target_fidelity = config.fidelity.lower
        if i <= max(self.observations.seen_config_ids):
            # IMPORTANT to set the fidelity at which EI will be calculated only for
            # the partial configs that have been observed already
            target_fidelity = config.fidelity.value + self.b_step

            if np.less_equal(target_fidelity, config.fidelity.upper):
                # only consider the configs with fidelity lower than the max fidelity
                config.fidelity.value = target_fidelity
                budget_list.append(self.get_budget_level(config))
            else:
                # if the target_fidelity higher than the max drop the configuration
                indices_to_drop.append(i)
        else:
            config.fidelity.value = target_fidelity
            budget_list.append(self.get_budget_level(config))

    # Drop unused configs
    x.drop(labels=indices_to_drop, inplace=True)

    performances = self.observations.get_best_performance_for_each_budget()
    inc_list = []
    for budget_level in budget_list:
        if budget_level in performances.index:
            inc = performances[budget_level]
        else:
            inc = self.observations.get_best_seen_performance()
        inc_list.append(inc)

    return x, torch.Tensor(inc_list)

preprocess_pfn #

preprocess_pfn(
    x: Iterable,
) -> Tuple[Iterable, Iterable, Iterable]

Prepares the configurations for appropriate EI calculation.

Takes a set of points and computes the budget and incumbent for each point, as required by the multi-fidelity Expected Improvement acquisition function.

Source code in neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py 
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def preprocess_pfn(self, x: Iterable) -> Tuple[Iterable, Iterable, Iterable]:
    """Prepares the configurations for appropriate EI calculation.

    Takes a set of points and computes the budget and incumbent for each point, as
    required by the multi-fidelity Expected Improvement acquisition function.
    """
    _x, inc_list = self.preprocess(x.copy())
    _x_tok = self.observations.tokenize(_x, as_tensor=True)
    len_partial = len(self.observations.seen_config_ids)
    z_min = x[0].fidelity.lower
    # converting fidelity to the discrete budget level
    # STRICT ASSUMPTION: fidelity is the first dimension
    _x_tok[:len_partial, 0] = (
        _x_tok[:len_partial, 0] + self.b_step - z_min
    ) / self.b_step
    return _x_tok, _x, inc_list