alhoo / jf

@@ -20,7 +20,7 @@
Loading
20 20
    True
21 21
    """
22 22
    from dateparser import parse as parsedate
23 -
        
23 +
24 24
    logger.info("Calculating the age of a column (%s)", datecol)
25 25
26 26
    def fn(datestr):
@@ -56,6 +56,7 @@
Loading
56 56
57 57
class Jfislice(JFTransformation):
58 58
    """jf wrapper for itertools.islice"""
59 +
59 60
    def _fn(self, arr):
60 61
        args = self.args
61 62
        start = None
@@ -129,6 +130,7 @@
Loading
129 130
    >>> pprint(list(Flatten().transform([{'a': 1, 'b':{'c': 2}}])))
130 131
    [{'a': 1, 'b.c': 2}]
131 132
    """
133 +
132 134
    def _fn(self, *args):
133 135
        logger.info("Flattening")
134 136
        arr = args[-1]
@@ -144,6 +146,7 @@
Loading
144 146
    >>> list(sorted(map(lambda x: list(x.items()), Transpose().transform(arr)), key=lambda x: x[0][1]))
145 147
    [[(0, 1), (1, 2)], [(0, 2), (1, 3)]]
146 148
    """
149 +
147 150
    def _fn(self, X):
148 151
        import pandas as pd
149 152
@@ -165,6 +168,7 @@
Loading
165 168
    >>> list(YieldAll(Col().data).transform([{"data": [1,2,3]}]))
166 169
    [1, 2, 3]
167 170
    """
171 +
168 172
    def _fn(self, arr):
169 173
        for items in arr:
170 174
            for val in self.args[0](items):
@@ -179,6 +183,7 @@
Loading
179 183
    >>> list(sorted(map(lambda x: len(x), list(GroupBy(x.item).transform(arr))[0].values())))
180 184
    [1, 2]
181 185
    """
186 +
182 187
    def _fn(self, arr):
183 188
        ret = {}
184 189
        for item in arr:
@@ -187,7 +192,7 @@
Loading
187 192
                ret[val].append(item)
188 193
            else:
189 194
                ret[val] = [item]
190 -
        yield {k:v for k, v in ret.items()}
195 +
        yield {k: v for k, v in ret.items()}
191 196
192 197
193 198
class Unique(JFTransformation):
@@ -198,8 +203,8 @@
Loading
198 203
    >>> len(list(Unique(x.b).transform(data)))
199 204
    2
200 205
    """
201 -
    def _fn(self, X):
202 206
207 +
    def _fn(self, X):
203 208
        def fun(x):
204 209
            return repr(x)
205 210
@@ -222,9 +227,12 @@
Loading
222 227
    >>> Hide("a").transform([{"a": 1, "id": 1}, {"a": 2, "id": 3}])
223 228
    [{'id': 1}, {'id': 3}]
224 229
    """
230 +
225 231
    def _fn(self, arr):
226 232
        elements = self.args
227 -
        ret = map(lambda item: {k: v for k, v in item.items() if k not in elements}, arr)
233 +
        ret = map(
234 +
            lambda item: {k: v for k, v in item.items() if k not in elements}, arr
235 +
        )
228 236
        if self.gen:
229 237
            return ret
230 238
        return list(ret)
@@ -237,6 +245,7 @@
Loading
237 245
    >>> Firstnlast(2).transform([1,2,3,4,5])
238 246
    [[1, 2], [4, 5]]
239 247
    """
248 +
240 249
    def _fn(self, arr):
241 250
        shown = 1
242 251
        if len(self.args) == 1:
@@ -253,6 +262,7 @@
Loading
253 262
    >>> First().transform([{"id": 99, "a": 1}, {"id": 199, "a": 2}])
254 263
    [{'id': 99, 'a': 1}]
255 264
    """
265 +
256 266
    def _fn(self, arr):
257 267
        shown = 1
258 268
        if len(self.args) == 1:
@@ -281,6 +291,7 @@
Loading
281 291
    >>> x.id({"id": 235})
282 292
    235
283 293
    """
294 +
284 295
    _opstrings = []
285 296
286 297
    def __setstate__(self, state):
@@ -431,6 +442,7 @@
Loading
431 442
                return Fn(getattr(mod, x))
432 443
            # Keep all the rest as they come
433 444
            return getattr(mod, x)
445 +
434 446
    return FnMod()
435 447
436 448
@@ -445,11 +457,13 @@
Loading
445 457
    >>> Fn(len)(x.id)({"id": "123"})
446 458
    3
447 459
    """
460 +
448 461
    def _fn(it):
449 462
        if isinstance(it, Col):
450 463
            itcopy = Col([x for x in it._opstrings])
451 464
            return itcopy._custom(fn)
452 465
        return fn(it)
466 +
453 467
    return _fn
454 468
455 469
@@ -472,6 +486,7 @@
Loading
472 486
    >>> list(Map(x.a).transform([{"a": 1}]))
473 487
    [1]
474 488
    """
489 +
475 490
    def _fn(self, X):
476 491
        fn = self.args[0]
477 492
        if isinstance(fn, (tuple, list)):
@@ -521,6 +536,7 @@
Loading
521 536
    >>> Filter(x.id > 100).transform([{"id": 99, "a": 1}, {"id": 199, "a": 2}])
522 537
    [{'id': 199, 'a': 2}]
523 538
    """
539 +
524 540
    def _fn(self, X):
525 541
        fn = self.args[0]
526 542
        if isinstance(fn, Col):
@@ -538,6 +554,7 @@
Loading
538 554
    >>> Last().transform([{"id": 99, "a": 1}, {"id": 199, "a": 2}])
539 555
    [{'id': 199, 'a': 2}]
540 556
    """
557 +
541 558
    def _fn(self, X):
542 559
        """Show last (N) items"""
543 560
        shown = 1
@@ -560,6 +577,7 @@
Loading
560 577
    >>> Sorted(x.a, reverse=True).transform([{"id": 99, "a": 1}, {"id": 199, "a": 2}])
561 578
    [{'id': 199, 'a': 2}, {'id': 99, 'a': 1}]
562 579
    """
580 +
563 581
    def _fn(self, X):
564 582
        keyget = None
565 583
        if len(self.args) == 1:
@@ -581,13 +599,14 @@
Loading
581 599
    >>> Print().transform([1, 2, 3, 4])
582 600
    [1, 2, 3, 4]
583 601
    """
602 +
584 603
    def _fn(self, arr):
585 604
        n = 1
586 605
        if len(self.args) > 0:
587 606
            n = self.args[0]
588 607
        arr = list(arr)
589 608
        for it in islice(arr, 0, n):
590 -
            sys.stderr.write(json.dumps(it)+"\n")
609 +
            sys.stderr.write(json.dumps(it) + "\n")
591 610
        return arr
592 611
593 612
@@ -598,6 +617,7 @@
Loading
598 617
    A pipeline in this context is a list of transformations that are applied, in order,
599 618
    to the input data stream.
600 619
    """
620 +
601 621
    def __init__(self, *transformations):
602 622
        if len(transformations) == 1 and isinstance(transformations[0], list):
603 623
            transformations = transformations[0]

@@ -152,7 +152,10 @@
Loading
152 152
        if import_from:
153 153
            sys.path.append(os.path.dirname(import_from))
154 154
        globalscope.update(
155 -
            {imp: process.fn_mod(importlib.import_module(imp)) for imp in imports.split(",")}
155 +
            {
156 +
                imp: process.fn_mod(importlib.import_module(imp))
157 +
                for imp in imports.split(",")
158 +
            }
156 159
        )
157 160
158 161
    try:

@@ -4,6 +4,7 @@
Loading
4 4
import datetime
5 5
from jf.process import JFTransformation
6 6
7 +
7 8
def json_encodings(obj):
8 9
    if isinstance(obj, (datetime.datetime, datetime.date)):
9 10
        obj.isoformat()
@@ -18,8 +19,7 @@
Loading
18 19
        from flask import Flask, request, Response
19 20
        from flask_cors import CORS
20 21
21 -
22 -
        base_path = ''
22 +
        base_path = ""
23 23
        if len(self.args) > 0:
24 24
            base_path = self.args[0]
25 25
        model = next(iter(arr))
@@ -67,4 +67,4 @@
Loading
67 67
                # prediction = list(model.predict(data))
68 68
69 69
        CORS(app)
70 -
        app.run(port=self.kwargs.get('port', 5002))
70 +
        app.run(port=self.kwargs.get("port", 5002))

@@ -133,7 +133,9 @@
Loading
133 133
        import openpyxl
134 134
        import pandas
135 135
136 -
        for val in pandas.read_excel(fn, engine='openpyxl').to_dict("records", into=OrderedDict):
136 +
        for val in pandas.read_excel(fn, engine="openpyxl").to_dict(
137 +
            "records", into=OrderedDict
138 +
        ):
137 139
            yield val
138 140
        return
139 141
    elif ext == "csv":
@@ -215,7 +217,7 @@
Loading
215 217
        import openpyxl
216 218
        import pandas
217 219
218 -
        for val in pandas.read_excel(args.files[0], engine='openpyxl').to_dict(
220 +
        for val in pandas.read_excel(args.files[0], engine="openpyxl").to_dict(
219 221
            "records", into=OrderedDict
220 222
        ):
221 223
            yield val
@@ -277,4 +279,5 @@
Loading
277 279
def yield_json_and_json_lines(inp):
278 280
    """Yield  json and json lines"""
279 281
    from jf import jsonlgen  # cpython from jsonlgen.cc
282 +
280 283
    return jsonlgen.gen(iter(inp))

@@ -14,6 +14,7 @@
Loading
14 14
    def transform(self, X, y=None):
15 15
        if isinstance(X, (tuple, list)):
16 16
            import pandas as pd
17 +
17 18
            X = pd.DataFrame(X)
18 19
        # Add selected columns to dataframe if needed
19 20
        if isinstance(self.column, (tuple, list)):
@@ -29,6 +30,7 @@
Loading
29 30
class transform(jf.process.JFTransformation):
30 31
    def _fn(self, arr):
31 32
        import numpy as np
33 +
32 34
        params = self.args[0]
33 35
        model = params
34 36
@@ -49,6 +51,7 @@
Loading
49 51
class shuffle(jf.process.JFTransformation):
50 52
    def _fn(self, arr):
51 53
        import numpy as np
54 +
52 55
        arr = list(arr)
53 56
        np.random.shuffle(arr)
54 57
        yield from arr
@@ -82,7 +85,7 @@
Loading
82 85
83 86
        ifn = self.args[0]
84 87
        ifn = ifn.replace("__JFESCAPED__", "")
85 -
        yield pickle.load(open(ifn, 'rb'))
88 +
        yield pickle.load(open(ifn, "rb"))
86 89
87 90
88 91
class persistent_trainer(jf.process.JFTransformation):

@@ -9,8 +9,11 @@
Loading
9 9
    """
10 10
    Baseclass for JF transformations
11 11
    """
12 +
12 13
    def __init__(self, *args, fn=None, **kwargs):
13 -
        self.args = [x.replace("__JFESCAPED__", "") if isinstance(x, str) else x for x in args]
14 +
        self.args = [
15 +
            x.replace("__JFESCAPED__", "") if isinstance(x, str) else x for x in args
16 +
        ]
14 17
        self.gen = False
15 18
        self.kwargs = kwargs
16 19
        if fn is not None:

@@ -6,10 +6,11 @@
Loading
6 6
7 7
mods = []
8 8
9 +
9 10
def load_sklearn_modules():
10 -
    sklearn = importlib.import_module('sklearn')
11 -
    globals()['sklearn'] = sklearn
12 -
    
11 +
    sklearn = importlib.import_module("sklearn")
12 +
    globals()["sklearn"] = sklearn
13 +
13 14
    classes = "pipeline,feature_extraction,calibration,cluster,compose,covariance,cross_decomposition,datasets,decomposition,experimental,ensemble,feature_selection,gaussian_process,impute,inspection,linear_model,manifold,metrics,mixture,model_selection,neighbors,neural_network,preprocessing,svm,tree,util,base".split(
14 15
        ","
15 16
    )

@@ -30,6 +30,7 @@
Loading
30 30
    outfmt = json.dumps
31 31
    if args.yaml and not args.json:
32 32
        from ruamel import yaml
33 +
33 34
        yaml.RoundTripDumper.add_representer(
34 35
            OrderedDict, yaml.RoundTripRepresenter.represent_dict
35 36
        )
@@ -126,10 +127,13 @@
Loading
126 127
127 128
class pandasWriter(JFTransformation):
128 129
    """General pandas writer"""
130 +
129 131
    def _fn(self, arr):
130 132
        import pandas as pd
131 133
132 -
        logger.info("Writing excel with args: %s and kwargs: %s", self.args, self.kwargs)
134 +
        logger.info(
135 +
            "Writing excel with args: %s and kwargs: %s", self.args, self.kwargs
136 +
        )
133 137
        df = pd.DataFrame(list(map(result_cleaner, arr)))
134 138
        if len(self.args) > 0:
135 139
            fn = self.args[0]
@@ -137,11 +141,12 @@
Loading
137 141
            yield f"data written to {fn}"
138 142
        else:
139 143
            import tempfile
144 +
140 145
            f = tempfile.NamedTemporaryFile(delete=False)
141 146
            f.close()
142 147
            fn = f.name
143 148
            getattr(df, self.writefn)(fn, *self.args[1:], **self.kwargs)
144 -
            with open(f.name, 'rb') as f:
149 +
            with open(f.name, "rb") as f:
145 150
                content = f.read()
146 151
            yield content
147 152
@@ -152,6 +157,7 @@
Loading
152 157
    >>> list(parquet("/tmp/test.parq").transform([{'a': 1}, {'a': 3}]))
153 158
    ['data written to /tmp/test.parq']
154 159
    """
160 +
155 161
    def __init__(self, *args, **kwargs):
156 162
        super(parquet, self).__init__(*args, **kwargs)
157 163
@@ -166,6 +172,7 @@
Loading
166 172
    >>> list(excel("/tmp/test.xlsx").transform([{'a': 1}, {'a': 3}]))
167 173
    ['data written to /tmp/test.xlsx']
168 174
    """
175 +
169 176
    def __init__(self, *args, **kwargs):
170 177
        super(excel, self).__init__(*args, **kwargs)
171 178
        self.writefn = "to_excel"
@@ -174,6 +181,7 @@
Loading
174 181
class browser(JFTransformation):
175 182
    """ Send output to browser
176 183
    """
184 +
177 185
    def _fn(self, arr):
178 186
        import webbrowser
179 187
        import tempfile
@@ -195,6 +203,7 @@
Loading
195 203
    1  |  2
196 204
    2  |  3
197 205
    """
206 +
198 207
    def _fn(self, arr):
199 208
        from csvtomd import md_table
200 209
        from math import isnan
@@ -211,7 +220,10 @@
Loading
211 220
                table.append([str(v) if v else "" for v in row.keys()])
212 221
                first = False
213 222
            table.append(
214 -
                [str(v) if isinstance(v, str) or not isnan(v) else "" for v in row.values()]
223 +
                [
224 +
                    str(v) if isinstance(v, str) or not isnan(v) else ""
225 +
                    for v in row.values()
226 +
                ]
215 227
            )
216 228
        if len(args):
217 229
            args[0].write(md_table(table, **self.kwargs) + "\n")
@@ -227,6 +239,7 @@
Loading
227 239
    1,2
228 240
    2,3
229 241
    """
242 +
230 243
    def _fn(self, arr):
231 244
        from csv import writer as cvs_writer
232 245
@@ -246,14 +259,15 @@
Loading
246 259
247 260
class ipy(JFTransformation):
248 261
    """Start ipython with data-variable"""
262 +
249 263
    def _fn(self, data):
250 264
        from IPython import embed
251 265
252 266
        banner = self.args[0]
253 267
        fakerun = False
254 -
        if 'fakerun' in self.kwargs:
268 +
        if "fakerun" in self.kwargs:
255 269
            fakerun = True
256 -
            del self.kwargs['fakerun']
270 +
            del self.kwargs["fakerun"]
257 271
        if not isinstance(banner, str):
258 272
            banner = ""
259 273
        banner += "\nJf instance is now dropping into IPython\n"
Files Coverage
jf 72.44%
Project Totals (10 files) 72.44%
298.3
TRAVIS_PYTHON_VERSION=3.8
TRAVIS_OS_NAME=linux
298.1
TRAVIS_PYTHON_VERSION=3.6
TRAVIS_OS_NAME=linux

No yaml found.

Create your codecov.yml to customize your Codecov experience

Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading