Skip to content

Utils

Utils module for specifying custom error classes and config space search interfaces.

This module defines specific error classes for simpler debugging and interfaces for searching config spaces.

Aggregation

Bases: Enum

Enum of aggregation functions for summarizing numpy arrays.

Source code in src/hypershap/utils.py 
27
28
29
30
31
32
33
class Aggregation(Enum):
    """Enum of aggregation functions for summarizing numpy arrays."""

    AVG = "avg"
    MAX = "max"
    MIN = "min"
    VAR = "var"

ConfigSpaceSearcher

Bases: ABC

Abstract base class for searching the configuration space.

Provides an interface for retrieving performance values based on a coalition of hyperparameters.

Source code in src/hypershap/utils.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
class ConfigSpaceSearcher(ABC):
    """Abstract base class for searching the configuration space.

    Provides an interface for retrieving performance values based on a coalition
    of hyperparameters.
    """

    def __init__(
        self,
        explanation_task: BaselineExplanationTask,
        mode: Aggregation,
    ) -> None:
        """Initialize the searcher with the explanation task.

        Args:
            explanation_task: The explanation task containing the configuration
                space and surrogate model.
            mode: The aggregation mode for performance values.

        """
        self.explanation_task = explanation_task
        self.mode = mode

    @abstractmethod
    def search(self, coalition: np.ndarray) -> float:
        """Search the configuration space based on the coalition.

        Args:
            coalition: A boolean array indicating which hyperparameters are
                constrained by the coalition.

        Returns:
            The aggregated performance value based on the search results.

        """

__init__(explanation_task, mode)

Initialize the searcher with the explanation task.

Parameters:

Name Type Description Default
explanation_task BaselineExplanationTask

The explanation task containing the configuration space and surrogate model.

 required
mode Aggregation

The aggregation mode for performance values.

 required
Source code in src/hypershap/utils.py 
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def __init__(
    self,
    explanation_task: BaselineExplanationTask,
    mode: Aggregation,
) -> None:
    """Initialize the searcher with the explanation task.

    Args:
        explanation_task: The explanation task containing the configuration
            space and surrogate model.
        mode: The aggregation mode for performance values.

    """
    self.explanation_task = explanation_task
    self.mode = mode

search(coalition) abstractmethod

Search the configuration space based on the coalition.

Parameters:

Name Type Description Default
coalition ndarray

A boolean array indicating which hyperparameters are constrained by the coalition.

 required

Returns:

Type Description
float

The aggregated performance value based on the search results.
Source code in src/hypershap/utils.py 
70
71
72
73
74
75
76
77
78
79
80
81
@abstractmethod
def search(self, coalition: np.ndarray) -> float:
    """Search the configuration space based on the coalition.

    Args:
        coalition: A boolean array indicating which hyperparameters are
            constrained by the coalition.

    Returns:
        The aggregated performance value based on the search results.

    """

RandomConfigSpaceSearcher

Bases: ConfigSpaceSearcher

A searcher that randomly samples the configuration space and evaluates them using the surrogate model.

Useful for establishing baseline performance or approximating game values.

Source code in src/hypershap/utils.py 
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
class RandomConfigSpaceSearcher(ConfigSpaceSearcher):
    """A searcher that randomly samples the configuration space and evaluates them using the surrogate model.

    Useful for establishing baseline performance or approximating game values.
    """

    def __init__(
        self,
        explanation_task: BaselineExplanationTask,
        mode: Aggregation = Aggregation.MAX,
        n_samples: int = 10_000,
        seed: int | None = 0,
    ) -> None:
        """Initialize the random configuration space searcher.

        Args:
            explanation_task: The explanation task containing the configuration
                space and surrogate model.
            mode: The aggregation mode for performance values.
            n_samples: The number of configurations to sample.
            seed: The random seed for sampling configurations from the config space.

        """
        super().__init__(explanation_task, mode=mode)
        cs = deepcopy(explanation_task.config_space)
        if seed is not None:
            cs.seed(seed)
        sampled_configurations = cs.sample_configuration(size=n_samples)
        self.random_sample = np.array([config.get_array() for config in sampled_configurations])

        # cache coalition values to ensure monotonicity for min/max
        self.coalition_cache = {}

    def _is_valid(self, config: np.ndarray) -> bool:
        """Check whether a configuration is valid with respect to conditions of the configuration space."""
        try:
            self.explanation_task.config_space.check_configuration_vector_representation(config)
        except (
            ActiveHyperparameterNotSetError,
            IllegalVectorizedValueError,
            InactiveHyperparameterSetError,
            ForbiddenValueError,
        ):
            return False
        else:
            return True

    def search(self, coalition: np.ndarray) -> float:
        """Search the configuration space based on the coalition.

        Args:
            coalition: A boolean array indicating which hyperparameters are
                constrained by the coalition.

        Returns:
            The aggregated performance value based on the search results.

        """
        # copy the sampled configurations
        temp_random_sample = self.random_sample.copy()

        # blind configurations according to coalition
        blind_coalition = ~coalition
        column_index = np.where(blind_coalition)
        temp_random_sample[:, column_index] = self.explanation_task.baseline_config.get_array()[column_index]

        # in case of conditions in the config space, it might happen that through blinding hyperparameter values
        # configurations might become invalid and those should not be considered for calculating vals
        if len(self.explanation_task.config_space.conditions) > 0:
            # filter invalid configurations
            validity = np.apply_along_axis(self._is_valid, axis=1, arr=temp_random_sample)
            filtered_samples = temp_random_sample[validity]

            if len(filtered_samples) < 0.05 * len(temp_random_sample):  # pragma: no cover
                logger.warning(
                    "WARNING: Due to blinding less than 5% of the samples in the random search remain valid. "
                    "Consider increasing the sampling budget of the random search.",
                )

            # predict performance values with the help of the surrogate model for the filtered configurations
            vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples))
        else:
            vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(temp_random_sample))

        return evaluate_aggregation(self.mode, vals)

__init__(explanation_task, mode=Aggregation.MAX, n_samples=10000, seed=0)

Initialize the random configuration space searcher.

Parameters:

Name Type Description Default
explanation_task BaselineExplanationTask

The explanation task containing the configuration space and surrogate model.

 required
mode Aggregation

The aggregation mode for performance values.

 MAX
n_samples int

The number of configurations to sample.

 10000
seed int | None

The random seed for sampling configurations from the config space.

 0
Source code in src/hypershap/utils.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def __init__(
    self,
    explanation_task: BaselineExplanationTask,
    mode: Aggregation = Aggregation.MAX,
    n_samples: int = 10_000,
    seed: int | None = 0,
) -> None:
    """Initialize the random configuration space searcher.

    Args:
        explanation_task: The explanation task containing the configuration
            space and surrogate model.
        mode: The aggregation mode for performance values.
        n_samples: The number of configurations to sample.
        seed: The random seed for sampling configurations from the config space.

    """
    super().__init__(explanation_task, mode=mode)
    cs = deepcopy(explanation_task.config_space)
    if seed is not None:
        cs.seed(seed)
    sampled_configurations = cs.sample_configuration(size=n_samples)
    self.random_sample = np.array([config.get_array() for config in sampled_configurations])

    # cache coalition values to ensure monotonicity for min/max
    self.coalition_cache = {}

search(coalition)

Search the configuration space based on the coalition.

Parameters:

Name Type Description Default
coalition ndarray

A boolean array indicating which hyperparameters are constrained by the coalition.

 required

Returns:

Type Description
float

The aggregated performance value based on the search results.
Source code in src/hypershap/utils.py 
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def search(self, coalition: np.ndarray) -> float:
    """Search the configuration space based on the coalition.

    Args:
        coalition: A boolean array indicating which hyperparameters are
            constrained by the coalition.

    Returns:
        The aggregated performance value based on the search results.

    """
    # copy the sampled configurations
    temp_random_sample = self.random_sample.copy()

    # blind configurations according to coalition
    blind_coalition = ~coalition
    column_index = np.where(blind_coalition)
    temp_random_sample[:, column_index] = self.explanation_task.baseline_config.get_array()[column_index]

    # in case of conditions in the config space, it might happen that through blinding hyperparameter values
    # configurations might become invalid and those should not be considered for calculating vals
    if len(self.explanation_task.config_space.conditions) > 0:
        # filter invalid configurations
        validity = np.apply_along_axis(self._is_valid, axis=1, arr=temp_random_sample)
        filtered_samples = temp_random_sample[validity]

        if len(filtered_samples) < 0.05 * len(temp_random_sample):  # pragma: no cover
            logger.warning(
                "WARNING: Due to blinding less than 5% of the samples in the random search remain valid. "
                "Consider increasing the sampling budget of the random search.",
            )

        # predict performance values with the help of the surrogate model for the filtered configurations
        vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples))
    else:
        vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(temp_random_sample))

    return evaluate_aggregation(self.mode, vals)

evaluate_aggregation(aggregation, values)

Evaluate an aggregation function for a numpy array summarizing it to a single float.

Source code in src/hypershap/utils.py 
36
37
38
39
40
41
42
43
44
def evaluate_aggregation(aggregation: Aggregation, values: np.ndarray) -> float:
    """Evaluate an aggregation function for a numpy array summarizing it to a single float."""
    if aggregation == Aggregation.AVG:
        return values.mean()
    if aggregation == Aggregation.MAX:
        return values.max()
    if aggregation == Aggregation.MIN:
        return values.min()
    return values.var()