#1397 Subclass SliceScorer from Scorer

Open Vincent Chen vincentschen
Coverage Reach
classification/training/loggers/checkpointer.py classification/training/loggers/log_manager.py classification/training/loggers/log_writer.py classification/training/loggers/tensorboard_writer.py classification/training/trainer.py classification/training/schedulers/shuffled_scheduler.py classification/training/schedulers/scheduler.py classification/training/schedulers/sequential_scheduler.py classification/training/schedulers/__init__.py classification/multitask_classifier.py classification/data.py classification/utils.py classification/task.py classification/loss.py labeling/model/label_model.py labeling/model/baselines.py labeling/model/logger.py labeling/model/graph_utils.py labeling/lf/nlp.py labeling/lf/core.py labeling/lf/__init__.py labeling/apply/core.py labeling/apply/pandas.py labeling/apply/dask.py labeling/analysis.py labeling/utils.py slicing/utils.py slicing/slicing_classifier.py slicing/monitor.py slicing/modules/slice_combiner.py slicing/sf/core.py slicing/sf/nlp.py slicing/apply/core.py augmentation/apply/core.py augmentation/apply/pandas.py augmentation/policy/core.py augmentation/policy/sampling.py augmentation/tf.py utils/core.py utils/optimizers.py utils/lr_schedulers.py utils/config_utils.py utils/data_operators.py map/core.py analysis/metrics.py analysis/scorer.py analysis/error_analysis.py analysis/__init__.py preprocess/nlp.py preprocess/core.py synthetic/synthetic_data.py types/data.py types/__init__.py types/classifier.py version.py

No flags found

Use flags to group coverage reports by test type, project and/or folders.
Then setup custom commit statuses and notifications for each flag.

e.g., #unittest #integration

#production #enterprise

#frontend #backend

Learn more about Codecov Flags here.

Showing 1 of 2 files from the diff.

@@ -1,4 +1,4 @@
Loading
1 -
from typing import Dict, List, Union
1 +
from typing import Callable, Dict, List, Mapping, Optional, Union
2 2
3 3
import numpy as np
4 4
import pandas as pd
@@ -10,29 +10,40 @@
Loading
10 10
SliceMetricsDict = Dict[str, Dict[str, float]]
11 11
12 12
13 -
class SliceScorer:
14 -
    """Scorer that returns metrics on overall performance, as well as slices.
13 +
class SliceScorer(Scorer):
14 +
    """Scorer with methods to return overall, as well as slice, metrics.
15 15
16 16
    Parameters
17 17
    ----------
18 -
    scorer
19 -
        A pre-defined ``Scorer`` used for evaluation
20 18
    slice_names
21 -
        A list of slice names corresponding to columns of ``S_matrix`` (accepted by score method)
19 +
        A list of slice names corresponding to columns of ``S_matrix``
20 +
        (accepted by ``score_slices`` method)
21 +
    metrics
22 +
        See ``snorkel.analysis.Scorer``
23 +
    custom_metric_funcs
24 +
        See ``snorkel.analysis.Scorer``
25 +
    abstain_label
26 +
        See ``snorkel.analysis.Scorer``
22 27
23 28
    Attributes
24 29
    ----------
25 -
    scorer
26 -
        See above
27 30
    slice_names
28 31
        See above.
29 32
    """
30 33
31 -
    def __init__(self, scorer: Scorer, slice_names: List[str]):
32 -
        self.scorer = scorer
34 +
    def __init__(
35 +
        self,
36 +
        slice_names: List[str],
37 +
        metrics: Optional[List[str]] = [
38 +
            "f1"
39 +
        ],  # default to F1 for class-imbalanced slices
40 +
        custom_metric_funcs: Optional[Mapping[str, Callable[..., float]]] = None,
41 +
        abstain_label: Optional[int] = -1,
42 +
    ):
43 +
        super().__init__(metrics, custom_metric_funcs, abstain_label)
33 44
        self.slice_names = slice_names
34 45
35 -
    def score(
46 +
    def score_slices(
36 47
        self,
37 48
        S: np.ndarray,
38 49
        golds: np.ndarray,
@@ -67,13 +78,13 @@
Loading
67 78
68 79
        # Include overall metrics
69 80
        metrics_dict = dict()
70 -
        metrics_dict.update({"overall": self.scorer.score(golds, preds, probs)})
81 +
        metrics_dict.update({"overall": self.score(golds, preds, probs)})
71 82
72 83
        # Include slice metrics
73 84
        for idx, slice_name in enumerate(self.slice_names):
74 85
            mask = S[:, idx].astype(bool)
75 86
            metrics_dict.update(
76 -
                {slice_name: self.scorer.score(golds[mask], preds[mask], probs[mask])}
87 +
                {slice_name: self.score(golds[mask], preds[mask], probs[mask])}
77 88
            )
78 89
79 90
        if as_dataframe:
@@ -85,15 +96,15 @@
Loading
85 96
def slice_dataframe(
86 97
    df: pd.DataFrame, slicing_function: LabelingFunction
87 98
) -> pd.DataFrame:
88 -
    """Return a dataframe with examples corresponding to specified SlicingFunction.
99 +
    """Return a dataframe with examples corresponding to specified ``SlicingFunction``.
89 100
90 101
    Parameters
91 102
    ----------
92 103
    df
93 104
        A pandas DataFrame that will be sliced
94 105
    slicing_function
95 106
        SlicingFunction which will operate over df to return a subset of examples;
96 -
        function returns a subset of data for which slicing_function output is True
107 +
        function returns a subset of data for which ``slicing_function`` output is True
97 108
98 109
    Returns
99 110
    -------

Everything is accounted for!

No changes detected that need to be reviewed.
What changes does Codecov check for?
Lines, not adjusted in diff, that have changed coverage data.
Files that introduced coverage data that had none before.
Files that have missing coverage data that once were tracked.
Files Coverage
snorkel 97.54%
Project Totals (55 files) 97.54%
Loading