1
""" a module to convert between the old (Python script) plugin format,
2
and the new (JSON) one
3
"""
4 1
from typing import Dict, Tuple  # noqa: F401
5 1
import ast
6 1
import json
7

8

9 1
def assess_syntax(path):
10

11 1
    with open(path) as file_obj:
12 1
        content = file_obj.read()
13

14 1
    syntax_tree = ast.parse(content)
15

16 1
    docstring = ""  # docstring = ast.get_docstring(syntaxTree)
17 1
    unknowns = []
18 1
    imported = {}
19 1
    assignments = {}
20 1
    for i, child in enumerate(ast.iter_child_nodes(syntax_tree)):
21 1
        if i == 0 and isinstance(child, ast.Expr) and isinstance(child.value, ast.Str):
22 1
            docstring = child.value.s
23 1
        elif isinstance(child, ast.ImportFrom):
24 1
            module = child.module
25 1
            for n in child.names:
26 1
                import_pth = module + "." + n.name
27 1
                imported[n.name if n.asname is None else n.asname] = import_pth
28 1
        elif isinstance(child, ast.Assign):
29 1
            targets = child.targets
30 1
            if len(targets) > 1:
31 0
                raise IOError(
32
                    "cannot handle expansion assignments " "(e.g. `a, b = [1, 2]`)"
33
                )
34 1
            target = child.targets[0]  # type: ast.Name
35 1
            assignments[target.id] = child.value
36
        else:
37 0
            unknowns.append(child)
38

39 1
    if unknowns:
40 0
        print(
41
            "Warning this script can only handle 'ImportFrom' and 'Assign' "
42
            "syntax, found additional items: {}".format(unknowns)
43
        )
44

45 1
    return docstring, imported, assignments
46

47

48 1
def ast_to_json(item, imported, assignments):
49
    """recursively convert ast items to json friendly values"""
50 1
    value = None
51 1
    if item in ["True", "False", "None"]:  # python 2.7
52 0
        value = {"True": True, "False": False, "None": None}[item]
53 1
    elif hasattr(ast, "NameConstant") and isinstance(item, ast.NameConstant):
54 1
        value = item.value
55 1
    elif isinstance(item, ast.Str):
56 1
        value = item.s
57 1
    elif isinstance(item, ast.Num):
58 0
        value = item.n
59 1
    elif isinstance(item, ast.Name):
60 1
        if item.id in imported:
61 1
            value = imported[item.id]
62 1
        elif item.id in assignments:
63 1
            value = ast_to_json(assignments[item.id], imported, assignments)
64 0
        elif item.id in ["True", "False", "None"]:  # python 2.7
65 0
            value = {"True": True, "False": False, "None": None}[item.id]
66
        else:
67 0
            raise ValueError("could not find assignment '{}' in config".format(item.id))
68 1
    elif isinstance(item, (ast.List, ast.Tuple, ast.Set)):
69 1
        value = [ast_to_json(i, imported, assignments) for i in item.elts]
70 1
    elif isinstance(item, ast.Dict):
71 1
        value = convert_dict(item, imported, assignments)
72
    else:
73 0
        raise ValueError("could not handle ast item: {}".format(item))
74

75 1
    return value
76

77

78 1
def convert_dict(dct, imported, assignments):
79
    # type: (ast.Dict, Dict[str, str], dict) -> dict
80
    """recurse through and replace keys"""
81 1
    out_dict = {}
82 1
    for key, val in zip(dct.keys, dct.values):
83 1
        if not isinstance(key, ast.Str):
84 0
            raise ValueError("expected key to be a Str; {}".format(key))
85 1
        out_dict[key.s] = ast_to_json(val, imported, assignments)
86

87 1
    return out_dict
88

89

90 1
def convert_oformat(oformat):
91

92 1
    if oformat == "Notebook":
93 0
        outline = None  # TODO do notebooks need template (they have currently)
94 0
        exporter = "nbconvert.exporters.NotebookExporter"
95 1
    elif oformat == "Latex":
96 1
        exporter = "nbconvert.exporters.LatexExporter"
97 1
        outline = {
98
            "module": "ipypublish.templates.outline_schemas",
99
            "file": "latex_outline.latex.j2",
100
        }
101 1
    elif oformat == "HTML":
102 1
        exporter = "nbconvert.exporters.HTMLExporter"
103 1
        outline = {
104
            "module": "ipypublish.templates.outline_schemas",
105
            "file": "html_outline.html.j2",
106
        }
107 0
    elif oformat == "Slides":
108 0
        exporter = "nbconvert.exporters.SlidesExporter"
109 0
        outline = {
110
            "module": "ipypublish.templates.outline_schemas",
111
            "file": "html_outline.html.j2",
112
        }
113
    else:
114 0
        raise ValueError(
115
            "expected oformat to be: " "'Notebook', 'Latex', 'HTML' or 'Slides'"
116
        )
117 1
    return exporter, outline
118

119

120 1
def convert_config(config, exporter_class, allow_other):
121
    # type: (dict, str) -> dict
122
    """convert config into required exporter format"""
123 1
    filters = {}
124 1
    preprocs = {}
125 1
    other = {}
126
    # first parse
127 1
    for key, val in config.items():
128
        # TODO Exporter.filters and TemplateExporter.filters always the same?
129 1
        if key in ["Exporter.filters", "TemplateExporter.filters"]:
130 1
            filters.update(config[key])
131 1
        if key in ["Exporter.preprocessors", "TemplateExporter.preprocessors"]:
132 1
            if preprocs:
133 0
                raise ValueError(
134
                    "'config' contains both Exporter.preprocessors and "
135
                    "TemplateExporter.preprocessors"
136
                )
137 1
            for p in val:
138 1
                pname = p.split(".")[-1]
139 1
                preprocs[pname] = {"class": p, "args": {}}
140
                # TODO move these special cases to seperate input/function
141 1
                if pname in ["LatexDocLinks", "LatexDocHTML"]:
142 1
                    preprocs[pname]["args"]["metapath"] = "${meta_path}"
143 1
                    preprocs[pname]["args"]["filesfolder"] = "${files_path}"
144

145
    # second parse
146 1
    for key, val in config.items():
147 1
        if key in [
148
            "Exporter.filters",
149
            "TemplateExporter.filters",
150
            "Exporter.preprocessors",
151
            "TemplateExporter.preprocessors",
152
        ]:
153 1
            continue
154 1
        if key.split(".")[0] in preprocs:
155 1
            preprocs[key.split(".")[0]]["args"][".".join(key.split(".")[1:])] = val
156
        else:
157 0
            other[key] = val
158

159 1
    if other and not allow_other:
160 0
        print("Warning: ignoring other args: {}".format(other))
161 0
        other = {}
162

163 1
    output = {
164
        "class": exporter_class,
165
        "filters": filters,
166
        "preprocessors": list(preprocs.values()),
167
        "other_args": other,
168
    }
169 1
    return output
170

171

172 1
def replace_template_path(path):
173
    """ replace original template path with new dict """
174 1
    segments = path.split(".")
175 1
    module = ".".join(segments[0:-1])
176 1
    name = segments[-1]
177 1
    if module == "ipypublish.html.ipypublish":
178 1
        return {
179
            "module": "ipypublish.templates.segments",
180
            "file": "ipy-{0}.html-tplx.json".format(name),
181
        }
182 1
    elif module == "ipypublish.html.standard":
183 1
        return {
184
            "module": "ipypublish.templates.segments",
185
            "file": "std-{0}.html-tplx.json".format(name),
186
        }
187 1
    elif module == "ipypublish.latex.standard":
188 1
        return {
189
            "module": "ipypublish.templates.segments",
190
            "file": "std-{0}.latex-tpl.json".format(name),
191
        }
192 1
    elif module == "ipypublish.latex.ipypublish":
193 1
        return {
194
            "module": "ipypublish.templates.segments",
195
            "file": "ipy-{0}.latex-tpl.json".format(name),
196
        }
197
    else:
198 0
        print("Warning: unknown template path: {}".format(path))
199 0
        return {"module": module, "file": "{0}.json".format(name)}
200

201

202 1
def create_json(docstring, imported, assignments, allow_other=True):
203
    #  type: (str, Dict[str, str], dict, bool) -> dict
204
    """Set docstring here.
205

206
    Parameters
207
    ----------
208
    docstring: str
209
        the doc string of the module
210
    imported: dict
211
        imported classes
212
    assignments: dict
213
        assigned values (i.e. 'a = b')
214
    allow_other: bool
215
        whether to allow arguments in config,
216
        which do not relate to preprocessors
217

218
    Returns
219
    -------
220

221
    """
222

223 1
    oformat = None
224 1
    config = None
225 1
    template = None
226 1
    for value, expr in assignments.items():
227 1
        if value == "oformat":
228 1
            if not isinstance(expr, ast.Str):
229 0
                raise ValueError("expected 'oformat' to be a Str; {}".format(expr))
230 1
            oformat = expr.s
231 1
        elif value == "config":
232 1
            if not isinstance(expr, ast.Dict):
233 0
                raise ValueError("expected 'config' to be a Dict; {}".format(expr))
234 1
            config = convert_dict(expr, imported, assignments)
235 1
        elif value == "template":
236 1
            if not isinstance(expr, ast.Call):
237 0
                raise ValueError("expected 'config' to be a call to create_tpl(x)")
238
            # func = expr.func  # TODO make sure func name is create_tpl/tplx
239 1
            args = expr.args
240 1
            keywords = expr.keywords
241 1
            if len(args) != 1 or len(keywords) > 0:
242 0
                raise ValueError("expected create_tpl(x) to have one argument")
243 1
            seg_list = args[0]
244 1
            if isinstance(seg_list, ast.ListComp):
245 1
                seg_list = seg_list.generators[0].iter
246 1
            if not isinstance(seg_list, ast.List):
247 0
                raise ValueError(
248
                    "expected create_tpl(x) arg to be a List; {}".format(seg_list)
249
                )
250 1
            segments = []
251 1
            for seg in seg_list.elts:
252 1
                if isinstance(seg, ast.Attribute):
253 1
                    seg_name = seg.value.id
254 1
                elif isinstance(seg, ast.Name):
255 1
                    seg_name = seg.id
256
                else:
257 0
                    raise ValueError(
258
                        "expected seg in template to be an Attribute; "
259
                        + "{1}".format(seg)
260
                    )
261

262 1
                if seg_name not in imported:
263 0
                    raise ValueError("segment '{}' not found".format(seg_name))
264 1
                segments.append(imported[seg_name])
265 1
            template = segments
266

267 1
    if oformat is None:
268 0
        raise ValueError("could not find 'oformat' assignment")
269 1
    if config is None:
270 0
        raise ValueError("could not find 'config' assignment")
271 1
    if template is None:
272 0
        raise ValueError("could not find 'template' assignment")
273

274 1
    exporter_class, outline = convert_oformat(oformat)
275 1
    exporter = convert_config(config, exporter_class, allow_other)
276

277 1
    if any(["biblio_natbib" in s for s in template]):
278 1
        exporter["filters"]["strip_ext"] = "ipypublish.filters.filters.strip_ext"
279

280 1
    return {
281
        "description": docstring.splitlines(),
282
        "exporter": exporter,
283
        "template": None
284
        if outline is None
285
        else {
286
            "outline": outline,
287
            "segments": [replace_template_path(s) for s in template],
288
        },
289
    }
290

291

292 1
def convert_to_json(path, outpath=None, ignore_other=False):
293
    """Set docstring here.
294

295
    Parameters
296
    ----------
297
    path: str
298
        input module path
299
    outpath=None: str or None
300
        if set, output json to this path
301
    ignore_other: bool
302
        whether to ignore arguments in config,
303
        which do not relate to preprocessors
304
    Returns
305
    -------
306

307
    """
308 1
    _docstring, _imported, _assignments = assess_syntax(path)
309
    # print(_docstring)
310
    # print()
311
    # print(_imported)
312
    # print()
313
    # print(_assignments)
314 1
    output = create_json(_docstring, _imported, _assignments, not ignore_other)
315 1
    if outpath:
316 0
        with open(outpath, "w") as file_obj:
317 0
            json.dump(output, file_obj, indent=2)
318 1
    return json.dumps(output, indent=2)
319

320

321 1
if __name__ == "__main__":
322

323
    if False:
324
        import glob
325
        import os
326

327
        for path in glob.glob(
328
            "/Users/cjs14/GitHub/ipypublish" "/ipypublish/export_plugins/*.py"
329
        ):
330
            dirname = os.path.dirname(path)
331
            name = os.path.splitext(os.path.basename(path))[0]
332
            try:
333
                convert_to_json(
334
                    path, os.path.join(dirname, name + ".json"), ignore_other=True
335
                )
336
            except ValueError as err:
337
                print("{0} failed: {1}".format(path, err))
338

339 0
    convert_to_json(
340
        "/Users/cjs14/GitHub/ipypublish" "/ipypublish_plugins/example_new_plugin.py",
341
        "/Users/cjs14/GitHub/ipypublish" "/ipypublish_plugins/example_new_plugin.json",
342
    )

Read our documentation on viewing source code .

Loading