Compare b8ef9a9 ... +249 ... 280c487

"""

This module contains the loss classes.

Specific losses are used for regression, binary classification or multiclass

classification.

"""

# Author: Nicolas Hug

    """Base class for a loss."""

        """Return initial gradients and hessians.

        Unless hessians are constant, arrays are initialized with undefined

        values.

        Parameters

        ----------

        n_samples : int

            The number of samples passed to `fit()`.

        prediction_dim : int

            The dimension of a raw prediction, i.e. the number of trees

            built at each iteration. Equals 1 for regression and binary

            classification, or K where K is the number of classes for

            multiclass classification.

        Returns

        -------

        gradients : ndarray, shape (prediction_dim, n_samples)

            The initial gradients. The array is not initialized.

        hessians : ndarray, shape (prediction_dim, n_samples)

            If hessians are constant (e.g. for `LeastSquares` loss, the

            array is initialized to ``1``. Otherwise, the array is allocated

            without being initialized.

        """

            # if the hessians are constant, we consider they are equal to 1.

            # this is correct as long as we adjust the gradients. See e.g. LS

            # loss

        else:

    def get_baseline_prediction(self, y_train, prediction_dim):

        """Return initial predictions (before the first iteration).

        Parameters

        ----------

        y_train : ndarray, shape (n_samples,)

            The target training values.

        prediction_dim : int

            The dimension of one prediction: 1 for binary classification and

            regression, n_classes for multiclass classification.

        Returns

        -------

        baseline_prediction : float or ndarray, shape (1, prediction_dim)

            The baseline prediction.

        """

    def update_gradients_and_hessians(self, gradients, hessians, y_true,

                                      raw_predictions):

        """Update gradients and hessians arrays, inplace.

        The gradients (resp. hessians) are the first (resp. second) order

        derivatives of the loss for each sample with respect to the

        predictions of model, evaluated at iteration ``i - 1``.

        Parameters

        ----------

        gradients : ndarray, shape (prediction_dim, n_samples)

            The gradients (treated as OUT array).

        hessians : ndarray, shape (prediction_dim, n_samples) or \

            (1,)

            The hessians (treated as OUT array).

        y_true : ndarray, shape (n_samples,)

            The true target values or each training sample.

        raw_predictions : ndarray, shape (prediction_dim, n_samples)

            The raw_predictions (i.e. values from the trees) of the tree

            ensemble at iteration ``i - 1``.

        """

    """Least squares loss, for regression.

    For a given sample x_i, least squares loss is defined as::

        loss(x_i) = 0.5 * (y_true_i - raw_pred_i)**2

    This actually computes the half least squares loss to optimize simplify

    the computation of the gradients and get a unit hessian (and be consistent

    with what is done in LightGBM).

    """

        # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to

        # return a view.

    def inverse_link_function(raw_predictions):

                                      raw_predictions):

        # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to

        # return a view.

    """Binary cross-entropy loss, for binary classification.

    For a given sample x_i, the binary cross-entropy loss is defined as the

    negative log-likelihood of the model which can be expressed as::

        loss(x_i) = log(1 + exp(raw_pred_i)) - y_true_i * raw_pred_i

    See The Elements of Statistical Learning, by Hastie, Tibshirani, Friedman,

    section 4.4.1 (about logistic regression).

    """

        # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to

        # return a view.

        # logaddexp(0, x) = log(1 + exp(x))

                "loss='binary_crossentropy' is not defined for multiclass"

                " classification with n_classes=%d, use"

                " loss='categorical_crossentropy' instead" % prediction_dim)

        # log(x / 1 - x) is the anti function of sigmoid, or the link function

        # of the Binomial model.

                                      raw_predictions):

        # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to

        # return a view.

            gradients, hessians, y_true, raw_predictions)

        # shape (1, n_samples) --> (n_samples,). reshape(-1) is more likely to

        # return a view.

    """Categorical cross-entropy loss, for multiclass classification.

    For a given sample x_i, the categorical cross-entropy loss is defined as

    the negative log-likelihood of the model and generalizes the binary

    cross-entropy to more than 2 classes.

    """

                (one_hot_true * raw_predictions).sum(axis=0))

                                      raw_predictions):

            gradients, hessians, y_true, raw_predictions)

        # TODO: This could be done in parallel

        # compute softmax (using exp(log(softmax)))

                       logsumexp(raw_predictions, axis=0)[np.newaxis, :])

    'least_squares': LeastSquares,

    'binary_crossentropy': BinaryCrossEntropy,

    'categorical_crossentropy': CategoricalCrossEntropy

}

                'HistGradientBoostingRegressor'):

        # The default min_samples_leaf (20) isn't appropriate for small

        # datasets (only very shallow trees are built) that the checks use.

    sklearn.ensemble.HistGradientBoostingClassifier,

    sklearn.ensemble.HistGradientBoostingRegressor,

    sklearn.tree.DecisionTreeRegressor, RandomForestRegressor

"""

This module contains the TreeGrower class.

TreeGrowee builds a regression tree fitting a Newton-Raphson step, based on

the gradients and hessians of the training data.

"""

# Author: Nicolas Hug

    """Tree Node class used in TreeGrower.

    This isn't used for prediction purposes, only for training (see

    TreePredictor).

    Parameters

    ----------

    depth : int

        The depth of the node, i.e. its distance from the root.

    sample_indices : ndarray of unsigned int, shape (n_samples_at_node,)

        The indices of the samples at the node.

    sum_gradients : float

        The sum of the gradients of the samples at the node.

    sum_hessians : float

        The sum of the hessians of the samples at the node.

    parent : TreeNode or None, optional (default=None)

        The parent of the node. None for root.

    Attributes

    ----------

    depth : int

        The depth of the node, i.e. its distance from the root.

    sample_indices : ndarray of unsigned int, shape (n_samples_at_node,)

        The indices of the samples at the node.

    sum_gradients : float

        The sum of the gradients of the samples at the node.

    sum_hessians : float

        The sum of the hessians of the samples at the node.

    parent : TreeNode or None

        The parent of the node. None for root.

    split_info : SplitInfo or None

        The result of the split evaluation.

    left_child : TreeNode or None

        The left child of the node. None for leaves.

    right_child : TreeNode or None

        The right child of the node. None for leaves.

    value : float or None

        The value of the leaf, as computed in finalize_leaf(). None for

        non-leaf nodes.

    partition_start : int

        start position of the node's sample_indices in splitter.partition.

    partition_stop : int

        stop position of the node's sample_indices in splitter.partition.

    """

    # start and stop indices of the node in the splitter.partition

    # array. Concretely,

    # self.sample_indices = view(self.splitter.partition[start:stop])

    # Please see the comments about splitter.partition and

    # splitter.split_indices for more info about this design.

    # These 2 attributes are only used in _update_raw_prediction, because we

    # need to iterate over the leaves and I don't know how to efficiently

    # store the sample_indices views because they're all of different sizes.

                 sum_hessians, parent=None):

        """Comparison for priority queue.

        Nodes with high gain are higher priority than nodes with low gain.

        heapq.heappush only need the '<' operator.

        heapq.heappop take the smallest item first (smaller is higher

        priority).

        Parameters

        -----------

        other_node : TreeNode

            The node to compare with.

        """

    """Tree grower class used to build a tree.

    The tree is fitted to predict the values of a Newton-Raphson step. The

    splits are considered in a best-first fashion, and the quality of a

    split is defined in splitting._split_gain.

    Parameters

    ----------

    X_binned : ndarray of int, shape (n_samples, n_features)

        The binned input samples. Must be Fortran-aligned.

    gradients : ndarray, shape (n_samples,)

        The gradients of each training sample. Those are the gradients of the

        loss w.r.t the predictions, evaluated at iteration ``i - 1``.

    hessians : ndarray, shape (n_samples,)

        The hessians of each training sample. Those are the hessians of the

        loss w.r.t the predictions, evaluated at iteration ``i - 1``.

    max_leaf_nodes : int or None, optional (default=None)

        The maximum number of leaves for each tree. If None, there is no

        maximum limit.

    max_depth : int or None, optional (default=None)

        The maximum depth of each tree. The depth of a tree is the number of

        nodes to go from the root to the deepest leaf.

    min_samples_leaf : int, optional (default=20)

        The minimum number of samples per leaf.

    min_gain_to_split : float, optional (default=0.)

        The minimum gain needed to split a node. Splits with lower gain will

        be ignored.

    max_bins : int, optional (default=256)

        The maximum number of bins. Used to define the shape of the

        histograms.

    actual_n_bins : ndarray of int or int, optional (default=None)

        The actual number of bins needed for each feature, which is lower or

        equal to ``max_bins``. If it's an int, all features are considered to

        have the same number of bins. If None, all features are considered to

        have ``max_bins`` bins.

    l2_regularization : float, optional (default=0)

        The L2 regularization parameter.

    min_hessian_to_split : float, optional (default=1e-3)

        The minimum sum of hessians needed in each node. Splits that result in

        at least one child having a sum of hessians less than

        ``min_hessian_to_split`` are discarded.

    shrinkage : float, optional (default=1)

        The shrinkage parameter to apply to the leaves values, also known as

        learning rate.

    """

                 max_depth=None, min_samples_leaf=20, min_gain_to_split=0.,

                 max_bins=256, actual_n_bins=None, l2_regularization=0.,

                 min_hessian_to_split=1e-3, shrinkage=1.):

                                  min_samples_leaf, min_gain_to_split,

                                  l2_regularization, min_hessian_to_split)

                [actual_n_bins] * X_binned.shape[1],

                dtype=np.uint32)

        else:

            X_binned, max_bins, gradients, hessians, hessians_are_constant)

            X_binned, max_bins, actual_n_bins, l2_regularization,

            min_hessian_to_split, min_samples_leaf, min_gain_to_split,

            hessians_are_constant)

                             min_samples_leaf, min_gain_to_split,

                             l2_regularization, min_hessian_to_split):

        """Validate parameters passed to __init__.

        Also validate parameters passed to splitter.

        """

                "X_binned must be of type uint8.")

                "X_binned should be passed as Fortran contiguous "

                "array for maximum efficiency.")

                             ' smaller than 2'.format(max_leaf_nodes))

                             ' smaller than 2'.format(max_depth))

                             'not be smaller than 1'.format(min_samples_leaf))

                             'must be positive.'.format(min_gain_to_split))

                             'positive.'.format(l2_regularization))

                             'must be positive.'.format(min_hessian_to_split))

        """Grow the tree, from root to leaves."""

        """Initialize root node and finalize it if needed."""

        else:

            depth=depth,

            sample_indices=self.splitter.partition,

            sum_gradients=sum_gradients,

            sum_hessians=sum_hessians

        )

            # Do not even bother computing any splitting statistics.

            self.root.sample_indices)

        """Compute the best possible split (SplitInfo) of a given node.

        Also push it in the heap of splittable nodes if gain isn't zero.

        The gain of a node is 0 if either all the leaves are pure

        (best gain = 0), or if no split would satisfy the constraints,

        (min_hessians_to_split, min_gain_to_split, min_samples_leaf)

        """

            node.sample_indices, node.histograms, node.sum_gradients,

            node.sum_hessians)

        else:

        """Split the node with highest potential gain.

        Returns

        -------

        left : TreeNode

            The resulting left child.

        right : TreeNode

            The resulting right child.

        """

        # Consider the node with the highest loss reduction (a.k.a. gain)

         sample_indices_right,

         right_child_pos) = self.splitter.split_indices(node.split_info,

                                                        node.sample_indices)

                                   sample_indices_left,

                                   node.split_info.sum_gradient_left,

                                   node.split_info.sum_hessian_left,

                                   parent=node)

                                    sample_indices_right,

                                    node.split_info.sum_gradient_right,

                                    node.split_info.sum_hessian_right,

                                    parent=node)

        # set start and stop indices

                and n_leaf_nodes == self.max_leaf_nodes):

        # Compute histograms of childs, and compute their best possible split

        # (if needed)

            # We will compute the histograms of both nodes even if one of them

            # is a leaf, since computing the second histogram is very cheap

            # (using histogram subtraction).

            else:

            # We use the brute O(n_samples) method on the child that has the

            # smallest number of samples, and the subtraction trick O(n_bins)

            # on the other one.

                self.histogram_builder.compute_histograms_brute(

                    smallest_child.sample_indices)

                self.histogram_builder.compute_histograms_subtraction(

                    node.histograms, smallest_child.histograms)

        """Compute the prediction value that minimizes the objective function.

        This sets the node.value attribute (node is a leaf iff node.value is

        not None).

        See Equation 5 of:

        XGBoost: A Scalable Tree Boosting System, T. Chen, C. Guestrin, 2016

        https://arxiv.org/abs/1603.02754

        """

            node.sum_hessians + self.splitter.l2_regularization)

        """Transform all splittable nodes into leaves.

        Used when some constraint is met e.g. maximum number of leaves or

        maximum depth."""

        """Make a TreePredictor object out of the current tree.

        Parameters

        ----------

        bin_thresholds : array-like of floats, optional (default=None)

            The actual thresholds values of each bin.

        Returns

        -------

        A TreePredictor object.

        """

                                   bin_thresholds=bin_thresholds)

                               bin_thresholds, next_free_idx=0):

    """Helper used in make_predictor to set the TreePredictor fields."""

    else:

        # Leaf node

    else:

        # Decision node

            predictor_nodes, grower_node.left_child,

            bin_thresholds=bin_thresholds, next_free_idx=next_free_idx)

            predictor_nodes, grower_node.right_child,

            bin_thresholds=bin_thresholds, next_free_idx=next_free_idx)

    # Histogram-based gradient boosting files

        "_hist_gradient_boosting._gradient_boosting",

        sources=["_hist_gradient_boosting/_gradient_boosting.pyx"],

        include_dirs=[numpy.get_include()])

                         sources=["_hist_gradient_boosting/histogram.pyx"],

                         include_dirs=[numpy.get_include()])

                         sources=["_hist_gradient_boosting/splitting.pyx"],

                         include_dirs=[numpy.get_include()])

                         sources=["_hist_gradient_boosting/_binning.pyx"],

                         include_dirs=[numpy.get_include()])

                         sources=["_hist_gradient_boosting/_predictor.pyx"],

                         include_dirs=[numpy.get_include()])

                         sources=["_hist_gradient_boosting/_loss.pyx"],

                         include_dirs=[numpy.get_include()])

                         sources=["_hist_gradient_boosting/types.pyx"],

                         include_dirs=[numpy.get_include()])

                         sources=["_hist_gradient_boosting/utils.pyx"],

                         include_dirs=[numpy.get_include()])