fmfn / BayesianOptimization

@@ -1,7 +1,9 @@
Loading
1 1
import warnings
2 2
from queue import Queue, Empty
3 3
4 -
from .target_space import TargetSpace
4 +
from bayes_opt.constraint import ConstraintModel
5 +
6 +
from .target_space import TargetSpace, ConstrainedTargetSpace
5 7
from .event import Events, DEFAULT_EVENTS
6 8
from .logger import _get_default_logger
7 9
from .util import UtilityFunction, acq_max, ensure_rng
@@ -16,6 +18,7 @@
Loading
16 18
    Inspired/Taken from
17 19
        https://www.protechtraining.com/blog/post/879#simple-observer
18 20
    """
21 +
19 22
    def __init__(self, events):
20 23
        # maps event names to subscribers
21 24
        # str -> dict
@@ -52,9 +55,12 @@
Loading
52 55
        Dictionary with parameters names as keys and a tuple with minimum
53 56
        and maximum values.
54 57
58 +
    constraint: A ConstraintModel. Note that the names of arguments of the
59 +
        constraint function and of f need to be the same.
60 +
55 61
    random_state: int or numpy.random.RandomState, optional(default=None)
56 62
        If the value is an integer, it is used as the seed for creating a
57 -
        numpy.random.RandomState. Otherwise the random state provided it is used.
63 +
        numpy.random.RandomState. Otherwise the random state provided is used.
58 64
        When set to None, an unseeded random state is generated.
59 65
60 66
    verbose: int, optional(default=2)
@@ -76,14 +82,16 @@
Loading
76 82
    set_bounds()
77 83
        Allows changing the lower and upper searching bounds
78 84
    """
79 -
    def __init__(self, f, pbounds, random_state=None, verbose=2,
85 +
86 +
    def __init__(self,
87 +
                 f,
88 +
                 pbounds,
89 +
                 constraint=None,
90 +
                 random_state=None,
91 +
                 verbose=2,
80 92
                 bounds_transformer=None):
81 93
        self._random_state = ensure_rng(random_state)
82 94
83 -
        # Data structure containing the function to be optimized, the bounds of
84 -
        # its domain, and a record of the evaluations we have done so far
85 -
        self._space = TargetSpace(f, pbounds, random_state)
86 -
87 95
        self._queue = Queue()
88 96
89 97
        # Internal GP regressor
@@ -95,6 +103,27 @@
Loading
95 103
            random_state=self._random_state,
96 104
        )
97 105
106 +
        if constraint is None:
107 +
            # Data structure containing the function to be optimized, the
108 +
            # bounds of its domain, and a record of the evaluations we have
109 +
            # done so far
110 +
            self._space = TargetSpace(f, pbounds, random_state)
111 +
            self.is_constrained = False
112 +
        else:
113 +
            constraint_ = ConstraintModel(
114 +
                constraint.fun,
115 +
                constraint.lb,
116 +
                constraint.ub,
117 +
                random_state=random_state
118 +
            )
119 +
            self._space = ConstrainedTargetSpace(
120 +
                f,
121 +
                constraint_,
122 +
                pbounds,
123 +
                random_state
124 +
            )
125 +
            self.is_constrained = True
126 +
98 127
        self._verbose = verbose
99 128
        self._bounds_transformer = bounds_transformer
100 129
        if self._bounds_transformer:
@@ -110,6 +139,12 @@
Loading
110 139
    def space(self):
111 140
        return self._space
112 141
142 +
    @property
143 +
    def constraint(self):
144 +
        if self.is_constrained:
145 +
            return self._space.constraint
146 +
        return None
147 +
113 148
    @property
114 149
    def max(self):
115 150
        return self._space.max()
@@ -136,6 +171,7 @@
Loading
136 171
            If True, the optimizer will evaluate the points when calling
137 172
            maximize(). Otherwise it will evaluate it at the moment.
138 173
        """
174 +
139 175
        if lazy:
140 176
            self._queue.put(params)
141 177
        else:
@@ -152,15 +188,17 @@
Loading
152 188
        with warnings.catch_warnings():
153 189
            warnings.simplefilter("ignore")
154 190
            self._gp.fit(self._space.params, self._space.target)
191 +
            if self.is_constrained:
192 +
                self.constraint.fit(self._space.params,
193 +
                                    self._space._constraint_values)
155 194
156 195
        # Finding argmax of the acquisition function.
157 -
        suggestion = acq_max(
158 -
            ac=utility_function.utility,
159 -
            gp=self._gp,
160 -
            y_max=self._space.target.max(),
161 -
            bounds=self._space.bounds,
162 -
            random_state=self._random_state
163 -
        )
196 +
        suggestion = acq_max(ac=utility_function.utility,
197 +
                             gp=self._gp,
198 +
                             constraint=self.constraint,
199 +
                             y_max=self._space.target.max(),
200 +
                             bounds=self._space.bounds,
201 +
                             random_state=self._random_state)
164 202
165 203
        return self._space.array_to_params(suggestion)
166 204
@@ -211,15 +249,15 @@
Loading
211 249
        kappa: float, optional(default=2.576)
212 250
            Parameter to indicate how closed are the next parameters sampled.
213 251
                Higher value = favors spaces that are least explored.
214 -
                Lower value = favors spaces where the regression function is the
215 -
                highest.
252 +
                Lower value = favors spaces where the regression function is
253 +
                the highest.
216 254
217 255
        kappa_decay: float, optional(default=1)
218 256
            `kappa` is multiplied by this factor every iteration.
219 257
220 258
        kappa_decay_delay: int, optional(default=0)
221 -
            Number of iterations that must have passed before applying the decay
222 -
            to `kappa`.
259 +
            Number of iterations that must have passed before applying the
260 +
            decay to `kappa`.
223 261
224 262
        xi: float, optional(default=0.0)
225 263
            [unused]
@@ -242,12 +280,11 @@
Loading
242 280
                util.update_params()
243 281
                x_probe = self.suggest(util)
244 282
                iteration += 1
245 -
246 283
            self.probe(x_probe, lazy=False)
247 284
248 285
            if self._bounds_transformer and iteration > 0:
249 -
                # The bounds transformer should only modify the bounds after the init_points points (only for the true
250 -
                # iterations)
286 +
                # The bounds transformer should only modify the bounds after
287 +
                # the init_points points (only for the true iterations)
251 288
                self.set_bounds(
252 289
                    self._bounds_transformer.transform(self._space))
253 290

@@ -2,9 +2,11 @@
Loading
2 2
from .domain_reduction import SequentialDomainReductionTransformer
3 3
from .util import UtilityFunction
4 4
from .logger import ScreenLogger, JSONLogger
5 +
from .constraint import ConstraintModel
5 6
6 7
__all__ = [
7 8
    "BayesianOptimization",
9 +
    "ConstraintModel"
8 10
    "UtilityFunction",
9 11
    "Events",
10 12
    "ScreenLogger",

@@ -0,0 +1,154 @@
Loading
1 +
import numpy as np
2 +
from sklearn.gaussian_process.kernels import Matern
3 +
from sklearn.gaussian_process import GaussianProcessRegressor
4 +
from scipy.stats import norm
5 +
6 +
7 +
class ConstraintModel():
8 +
    """
9 +
    This class takes the function to optimize as well as the parameters bounds
10 +
    in order to find which values for the parameters yield the maximum value
11 +
    using bayesian optimization.
12 +
13 +
    Parameters
14 +
    ----------
15 +
    fun: function
16 +
        Constraint function. If multiple constraints are handled, this should
17 +
        return a numpy.ndarray of appropriate size.
18 +
19 +
    lb: numeric or numpy.ndarray
20 +
        Upper limit(s) for the constraints. The return value of `fun` should
21 +
        have exactly this shape.
22 +
23 +
    ub: numeric or numpy.ndarray
24 +
        Upper limit(s) for the constraints. The return value of `fun` should
25 +
        have exactly this shape.
26 +
27 +
    random_state: int or numpy.random.RandomState, optional(default=None)
28 +
        If the value is an integer, it is used as the seed for creating a
29 +
        numpy.random.RandomState. Otherwise the random state provided is used.
30 +
        When set to None, an unseeded random state is generated.
31 +
32 +
    Note
33 +
    ----
34 +
    In case of multiple constraints, this model assumes conditional
35 +
    independence. This means that for each constraint, the probability of
36 +
    fulfillment is the cdf of a univariate Gaussian. The overall probability
37 +
    is a simply the product of the individual probabilities.
38 +
    """
39 +
40 +
    def __init__(self, fun, lb, ub, random_state=None):
41 +
        self.fun = fun
42 +
43 +
        if isinstance(lb, float):
44 +
            self._lb = np.array([lb])
45 +
        else:
46 +
            self._lb = lb
47 +
        
48 +
        if isinstance(ub, float):
49 +
            self._ub = np.array([ub])
50 +
        else:
51 +
            self._ub = ub
52 +
        
53 +
        
54 +
        basis = lambda: GaussianProcessRegressor(
55 +
            kernel=Matern(nu=2.5),
56 +
            alpha=1e-6,
57 +
            normalize_y=True,
58 +
            n_restarts_optimizer=5,
59 +
            random_state=random_state,
60 +
        )
61 +
        self._model = [basis() for _ in range(len(self._lb))]
62 +
63 +
    @property
64 +
    def lb(self):
65 +
        return self._lb
66 +
67 +
    @property
68 +
    def ub(self):
69 +
        return self._ub
70 +
    
71 +
    @property
72 +
    def model(self):
73 +
        return self._model
74 +
75 +
    def eval(self, **kwargs):
76 +
        """
77 +
        Evaluates the constraint function.
78 +
        """
79 +
        try:
80 +
            return self.fun(**kwargs)
81 +
        except TypeError as e:
82 +
            msg = (
83 +
                "Encountered TypeError when evaluating constraint " +
84 +
                "function. This could be because your constraint function " +
85 +
                "doesn't use the same keyword arguments as the target " +
86 +
                f"function. Original error message:\n\n{e}"
87 +
                )
88 +
            e.args = (msg,)
89 +
            raise
90 +
91 +
    def fit(self, X, Y):
92 +
        """
93 +
        Fits internal GaussianProcessRegressor's to the data.
94 +
        """
95 +
        if len(self._model) == 1:
96 +
            self._model[0].fit(X, Y)
97 +
        else:
98 +
            for i, gp in enumerate(self._model):
99 +
                gp.fit(X, Y[:, i])
100 +
101 +
    def predict(self, X):
102 +
        """
103 +
        Returns the probability that the constraint is fulfilled at `X` based
104 +
        on the internal Gaussian Process Regressors.
105 +
106 +
        Note that this does not try to approximate the values of the constraint
107 +
        function, but probability that the constraint function is fulfilled.
108 +
        For the former, see `ConstraintModel.approx()`.
109 +
        """
110 +
        X_shape = X.shape
111 +
        X = X.reshape((-1, self._model[0].n_features_in_))
112 +
        if len(self._model) == 1:
113 +
            y_mean, y_std = self._model[0].predict(X, return_std=True)
114 +
115 +
            p_lower = (norm(loc=y_mean, scale=y_std).cdf(self._lb[0])
116 +
                            if self._lb[0] != -np.inf else np.array([0]))
117 +
            p_upper = (norm(loc=y_mean, scale=y_std).cdf(self._ub[0])
118 +
                            if self._lb[0] != np.inf else np.array([1]))
119 +
            result = p_upper - p_lower
120 +
            return result.reshape(X_shape[:-1])
121 +
        else:
122 +
            result = np.ones(X.shape[0])
123 +
            for j, gp in enumerate(self._model):
124 +
                y_mean, y_std = gp.predict(X, return_std=True)
125 +
                p_lower = (norm(loc=y_mean, scale=y_std).cdf(self._lb[j])
126 +
                           if self._lb[j] != -np.inf else np.array([0]))
127 +
                p_upper = (norm(loc=y_mean, scale=y_std).cdf(self._ub[j])
128 +
                           if self._lb[j] != np.inf else np.array([1]))
129 +
                result = result * (p_upper - p_lower)
130 +
            return result.reshape(X_shape[:-1])
131 +
132 +
    def approx(self, X):
133 +
        """
134 +
        Returns the approximation of the constraint function using the internal
135 +
        Gaussian Process Regressors.
136 +
        """
137 +
        X_shape = X.shape
138 +
        X = X.reshape((-1, self._model[0].n_features_in_))
139 +
        if len(self._model) == 1:
140 +
            return self._model[0].predict(X).reshape(X_shape[:-1])
141 +
        else:
142 +
            result = np.column_stack([gp.predict(X) for gp in self._model])
143 +
            return result.reshape(X_shape[:-1] + (len(self._lb), ))
144 +
145 +
    def allowed(self, constraint_values):
146 +
        """
147 +
        Checks whether `constraint_values` are below the specified limits.
148 +
        """
149 +
        if self._lb.size == 1:
150 +
            return (np.less_equal(self._lb, constraint_values)
151 +
                    & np.less_equal(constraint_values, self._ub))
152 +
153 +
        return (np.all(constraint_values <= self._ub, axis=-1)
154 +
                    & np.all(constraint_values >= self._lb, axis=-1))

@@ -4,7 +4,7 @@
Loading
4 4
from scipy.optimize import minimize
5 5
6 6
7 -
def acq_max(ac, gp, y_max, bounds, random_state, n_warmup=10000, n_iter=10):
7 +
def acq_max(ac, gp, y_max, bounds, random_state, constraint=None, n_warmup=10000, n_iter=10):
8 8
    """
9 9
    A function to find the maximum of the acquisition function
10 10
@@ -29,6 +29,9 @@
Loading
29 29
    :param random_state:
30 30
        instance of np.RandomState random number generator
31 31
32 +
    :param constraint:
33 +
        A ConstraintModel.
34 +
32 35
    :param n_warmup:
33 36
        number of times to randomly sample the acquisition function
34 37
@@ -50,9 +53,28 @@
Loading
50 53
    # Explore the parameter space more throughly
51 54
    x_seeds = random_state.uniform(bounds[:, 0], bounds[:, 1],
52 55
                                   size=(n_iter, bounds.shape[0]))
56 +
57 +
    if constraint is not None:
58 +
        def to_minimize(x):
59 +
            target = -ac(x.reshape(1, -1), gp=gp, y_max=y_max)
60 +
            p_constraint = constraint.predict(x.reshape(1, -1))
61 +
62 +
            # TODO: This is not exactly how Gardner et al do it.
63 +
            # Their way would require the result of the acquisition function
64 +
            # to be strictly positive (or negative), which is not the case
65 +
            # here. For a negative target value, we use Gardner's version. If
66 +
            # the target is positive, we instead slightly rescale the target
67 +
            # depending on the probability estimate to fulfill the constraint.
68 +
            if target < 0:
69 +
                return target * p_constraint
70 +
            else:
71 +
                return target / (0.5 + p_constraint)
72 +
    else:
73 +
        to_minimize = lambda x: -ac(x.reshape(1, -1), gp=gp, y_max=y_max)
74 +
53 75
    for x_try in x_seeds:
54 76
        # Find the minimum of minus the acquisition function
55 -
        res = minimize(lambda x: -ac(x.reshape(1, -1), gp=gp, y_max=y_max),
77 +
        res = minimize(lambda x: to_minimize(x),
56 78
                       x_try,
57 79
                       bounds=bounds,
58 80
                       method="L-BFGS-B")
@@ -83,7 +105,7 @@
Loading
83 105
        self._kappa_decay_delay = kappa_decay_delay
84 106
85 107
        self.xi = xi
86 -
        
108 +
87 109
        self._iters_counter = 0
88 110
89 111
        if kind not in ['ucb', 'ei', 'poi']:
@@ -121,7 +143,7 @@
Loading
121 143
        with warnings.catch_warnings():
122 144
            warnings.simplefilter("ignore")
123 145
            mean, std = gp.predict(x, return_std=True)
124 -
  
146 +
125 147
        a = (mean - y_max - xi)
126 148
        z = a / std
127 149
        return a * norm.cdf(z) + std * norm.pdf(z)

@@ -25,8 +25,8 @@
Loading
25 25
            self._iterations += 1
26 26
27 27
            current_max = instance.max
28 -
            if (self._previous_max is None or
29 -
                current_max["target"] > self._previous_max):
28 +
            if (self._previous_max is None
29 +
                    or current_max["target"] > self._previous_max):
30 30
                self._previous_max = current_max["target"]
31 31
                self._previous_max_params = current_max["params"]
32 32

@@ -1,5 +1,6 @@
Loading
1 1
import numpy as np
2 2
from .util import ensure_rng
3 +
from .constraint import ConstraintModel
3 4
4 5
5 6
def _hashable(x):
@@ -119,8 +120,7 @@
Loading
119 120
        except AssertionError:
120 121
            raise ValueError(
121 122
                "Size of array ({}) is different than the ".format(len(x)) +
122 -
                "expected number of parameters ({}).".format(len(self.keys))
123 -
            )
123 +
                "expected number of parameters ({}).".format(len(self.keys)))
124 124
        return x
125 125
126 126
    def register(self, params, target):
@@ -212,14 +212,13 @@
Loading
212 212
        >>> space.random_points(1)
213 213
        array([[ 55.33253689,   0.54488318]])
214 214
        """
215 -
        # TODO: support integer, category, and basic scipy.optimize constraints
216 215
        data = np.empty((1, self.dim))
217 216
        for col, (lower, upper) in enumerate(self._bounds):
218 217
            data.T[col] = self.random_state.uniform(lower, upper, size=1)
219 218
        return data.ravel()
220 219
221 220
    def max(self):
222 -
        """Get maximum target value found and corresponding parametes."""
221 +
        """Get maximum target value found and corresponding parameters."""
223 222
        try:
224 223
            res = {
225 224
                'target': self.target.max(),
@@ -252,3 +251,101 @@
Loading
252 251
        for row, key in enumerate(self.keys):
253 252
            if key in new_bounds:
254 253
                self._bounds[row] = new_bounds[key]
254 +
255 +
256 +
class ConstrainedTargetSpace(TargetSpace):
257 +
    """
258 +
    Expands TargetSpace to incorporate constraints.
259 +
    """
260 +
    def __init__(self,
261 +
                 target_func,
262 +
                 constraint: ConstraintModel,
263 +
                 pbounds,
264 +
                 random_state=None):
265 +
        super().__init__(target_func, pbounds, random_state)
266 +
267 +
        self._constraint = constraint
268 +
269 +
        # preallocated memory for constraint fulfillement
270 +
        if constraint.lb.size == 1:
271 +
            self._constraint_values = np.empty(shape=(0), dtype=float)
272 +
        else:
273 +
            self._constraint_values = np.empty(shape=(0, constraint.lb.size), dtype=float)
274 +
275 +
    @property
276 +
    def constraint(self):
277 +
        return self._constraint
278 +
279 +
    @property
280 +
    def constraint_values(self):
281 +
        return self._constraint_values
282 +
283 +
    def register(self, params, target, constraint_value):
284 +
        x = self._as_array(params)
285 +
        if x in self:
286 +
            raise KeyError('Data point {} is not unique'.format(x))
287 +
288 +
        # Insert data into unique dictionary
289 +
        self._cache[_hashable(x.ravel())] = (target, constraint_value)
290 +
291 +
        self._params = np.concatenate([self._params, x.reshape(1, -1)])
292 +
        self._target = np.concatenate([self._target, [target]])
293 +
        self._constraint_values = np.concatenate([self._constraint_values,
294 +
                                                 [constraint_value]])
295 +
296 +
    def probe(self, params):
297 +
        x = self._as_array(params)
298 +
299 +
        try:
300 +
            return self._cache[_hashable(x)]
301 +
        except KeyError:
302 +
            params = dict(zip(self._keys, x))
303 +
            target = self.target_func(**params)
304 +
            constraint_value = self._constraint.eval(**params)
305 +
            self.register(x, target, constraint_value)
306 +
        return target, constraint_value
307 +
308 +
    def max(self):
309 +
        """Get maximum target value found and corresponding parametes provided
310 +
        that they fulfill the constraints."""
311 +
        allowed = self._constraint.allowed(self._constraint_values)
312 +
        if allowed.any():
313 +
            # Getting of all points that fulfill the constraints, find the
314 +
            # one with the maximum value for the target function.
315 +
            sorted = np.argsort(self.target)
316 +
            idx = sorted[allowed[sorted]][-1]
317 +
            # there must be a better way to do this, right?
318 +
            res = {
319 +
                'target': self.target[idx],
320 +
                'params': dict(
321 +
                    zip(self.keys, self.params[idx])
322 +
                ),
323 +
                'constraint': self._constraint_values[idx]
324 +
            }
325 +
        else:
326 +
            res = {
327 +
                'target': None,
328 +
                'params': None,
329 +
                'constraint': None
330 +
            }
331 +
        return res
332 +
333 +
    def res(self):
334 +
        """Get all target values and constraint fulfillment for all parameters.
335 +
        """
336 +
        params = [dict(zip(self.keys, p)) for p in self.params]
337 +
338 +
        return [
339 +
            {
340 +
                "target": target,
341 +
                "constraint": constraint_value,
342 +
                "params": param,
343 +
                "allowed": allowed
344 +
            }
345 +
            for target, constraint_value, param, allowed in zip(
346 +
                self.target,
347 +
                self._constraint_values,
348 +
                params,
349 +
                self._constraint.allowed(self._constraint_values)
350 +
                )
351 +
        ]
Files Coverage
bayes_opt 99.25%
Project Totals (8 files) 99.25%
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading