chrisjsewell / ipypublish
1
""" a module to convert between the old (Python script) plugin format,
2
and the new (JSON) one
3
"""
4 3
from typing import Dict, Tuple  # noqa: F401
5 3
import ast
6 3
import json
7

8

9 3
def assess_syntax(path):
10

11 3
    with open(path) as file_obj:
12 3
        content = file_obj.read()
13

14 3
    syntax_tree = ast.parse(content)
15

16 3
    docstring = ""  # docstring = ast.get_docstring(syntaxTree)
17 3
    unknowns = []
18 3
    imported = {}
19 3
    assignments = {}
20 3
    for i, child in enumerate(ast.iter_child_nodes(syntax_tree)):
21 3
        if i == 0 and isinstance(child, ast.Expr) and isinstance(child.value, ast.Str):
22 3
            docstring = child.value.s
23 3
        elif isinstance(child, ast.ImportFrom):
24 3
            module = child.module
25 3
            for n in child.names:
26 3
                import_pth = module + "." + n.name
27 3
                imported[n.name if n.asname is None else n.asname] = import_pth
28 3
        elif isinstance(child, ast.Assign):
29 3
            targets = child.targets
30 3
            if len(targets) > 1:
31 0
                raise IOError(
32
                    "cannot handle expansion assignments " "(e.g. `a, b = [1, 2]`)"
33
                )
34 3
            target = child.targets[0]  # type: ast.Name
35 3
            assignments[target.id] = child.value
36
        else:
37 0
            unknowns.append(child)
38

39 3
    if unknowns:
40 0
        print(
41
            "Warning this script can only handle 'ImportFrom' and 'Assign' "
42
            "syntax, found additional items: {}".format(unknowns)
43
        )
44

45 3
    return docstring, imported, assignments
46

47

48 3
def ast_to_json(item, imported, assignments):
49
    """recursively convert ast items to json friendly values"""
50 3
    value = None
51 3
    if item in ["True", "False", "None"]:  # python 2.7
52 0
        value = {"True": True, "False": False, "None": None}[item]
53 3
    elif hasattr(ast, "NameConstant") and isinstance(item, ast.NameConstant):
54 3
        value = item.value
55 3
    elif isinstance(item, ast.Str):
56 3
        value = item.s
57 3
    elif isinstance(item, ast.Num):
58 0
        value = item.n
59 3
    elif isinstance(item, ast.Name):
60 3
        if item.id in imported:
61 3
            value = imported[item.id]
62 3
        elif item.id in assignments:
63 3
            value = ast_to_json(assignments[item.id], imported, assignments)
64 0
        elif item.id in ["True", "False", "None"]:  # python 2.7
65 0
            value = {"True": True, "False": False, "None": None}[item.id]
66
        else:
67 0
            raise ValueError("could not find assignment '{}' in config".format(item.id))
68 3
    elif isinstance(item, (ast.List, ast.Tuple, ast.Set)):
69 3
        value = [ast_to_json(i, imported, assignments) for i in item.elts]
70 3
    elif isinstance(item, ast.Dict):
71 3
        value = convert_dict(item, imported, assignments)
72
    else:
73 0
        raise ValueError("could not handle ast item: {}".format(item))
74

75 3
    return value
76

77

78 3
def convert_dict(dct, imported, assignments):
79
    # type: (ast.Dict, Dict[str, str], dict) -> dict
80
    """recurse through and replace keys"""
81 3
    out_dict = {}
82 3
    for key, val in zip(dct.keys, dct.values):
83 3
        if not isinstance(key, ast.Str):
84 0
            raise ValueError("expected key to be a Str; {}".format(key))
85 3
        out_dict[key.s] = ast_to_json(val, imported, assignments)
86

87 3
    return out_dict
88

89

90 3
def convert_oformat(oformat):
91

92 3
    if oformat == "Notebook":
93 0
        outline = None  # TODO do notebooks need template (they have currently)
94 0
        exporter = "nbconvert.exporters.NotebookExporter"
95 3
    elif oformat == "Latex":
96 3
        exporter = "nbconvert.exporters.LatexExporter"
97 3
        outline = {
98
            "module": "ipypublish.templates.outline_schemas",
99
            "file": "latex_outline.latex.j2",
100
        }
101 3
    elif oformat == "HTML":
102 3
        exporter = "nbconvert.exporters.HTMLExporter"
103 3
        outline = {
104
            "module": "ipypublish.templates.outline_schemas",
105
            "file": "html_outline.html.j2",
106
        }
107 0
    elif oformat == "Slides":
108 0
        exporter = "nbconvert.exporters.SlidesExporter"
109 0
        outline = {
110
            "module": "ipypublish.templates.outline_schemas",
111
            "file": "html_outline.html.j2",
112
        }
113
    else:
114 0
        raise ValueError(
115
            "expected oformat to be: " "'Notebook', 'Latex', 'HTML' or 'Slides'"
116
        )
117 3
    return exporter, outline
118

119

120 3
def convert_config(config, exporter_class, allow_other):
121
    # type: (dict, str) -> dict
122
    """convert config into required exporter format"""
123 3
    filters = {}
124 3
    preprocs = {}
125 3
    other = {}
126
    # first parse
127 3
    for key, val in config.items():
128
        # TODO Exporter.filters and TemplateExporter.filters always the same?
129 3
        if key in ["Exporter.filters", "TemplateExporter.filters"]:
130 3
            filters.update(config[key])
131 3
        if key in ["Exporter.preprocessors", "TemplateExporter.preprocessors"]:
132 3
            if preprocs:
133 0
                raise ValueError(
134
                    "'config' contains both Exporter.preprocessors and "
135
                    "TemplateExporter.preprocessors"
136
                )
137 3
            for p in val:
138 3
                pname = p.split(".")[-1]
139 3
                preprocs[pname] = {"class": p, "args": {}}
140
                # TODO move these special cases to seperate input/function
141 3
                if pname in ["LatexDocLinks", "LatexDocHTML"]:
142 3
                    preprocs[pname]["args"]["metapath"] = "${meta_path}"
143 3
                    preprocs[pname]["args"]["filesfolder"] = "${files_path}"
144

145
    # second parse
146 3
    for key, val in config.items():
147 3
        if key in [
148
            "Exporter.filters",
149
            "TemplateExporter.filters",
150
            "Exporter.preprocessors",
151
            "TemplateExporter.preprocessors",
152
        ]:
153 3
            continue
154 3
        if key.split(".")[0] in preprocs:
155 3
            preprocs[key.split(".")[0]]["args"][".".join(key.split(".")[1:])] = val
156
        else:
157 0
            other[key] = val
158

159 3
    if other and not allow_other:
160 0
        print("Warning: ignoring other args: {}".format(other))
161 0
        other = {}
162

163 3
    output = {
164
        "class": exporter_class,
165
        "filters": filters,
166
        "preprocessors": list(preprocs.values()),
167
        "other_args": other,
168
    }
169 3
    return output
170

171

172 3
def replace_template_path(path):
173
    """ replace original template path with new dict """
174 3
    segments = path.split(".")
175 3
    module = ".".join(segments[0:-1])
176 3
    name = segments[-1]
177 3
    if module == "ipypublish.html.ipypublish":
178 3
        return {
179
            "module": "ipypublish.templates.segments",
180
            "file": "ipy-{0}.html-tplx.json".format(name),
181
        }
182 3
    elif module == "ipypublish.html.standard":
183 3
        return {
184
            "module": "ipypublish.templates.segments",
185
            "file": "std-{0}.html-tplx.json".format(name),
186
        }
187 3
    elif module == "ipypublish.latex.standard":
188 3
        return {
189
            "module": "ipypublish.templates.segments",
190
            "file": "std-{0}.latex-tpl.json".format(name),
191
        }
192 3
    elif module == "ipypublish.latex.ipypublish":
193 3
        return {
194
            "module": "ipypublish.templates.segments",
195
            "file": "ipy-{0}.latex-tpl.json".format(name),
196
        }
197
    else:
198 0
        print("Warning: unknown template path: {}".format(path))
199 0
        return {"module": module, "file": "{0}.json".format(name)}
200

201

202 3
def create_json(docstring, imported, assignments, allow_other=True):
203
    #  type: (str, Dict[str, str], dict, bool) -> dict
204
    """Set docstring here.
205

206
    Parameters
207
    ----------
208
    docstring: str
209
        the doc string of the module
210
    imported: dict
211
        imported classes
212
    assignments: dict
213
        assigned values (i.e. 'a = b')
214
    allow_other: bool
215
        whether to allow arguments in config,
216
        which do not relate to preprocessors
217

218
    Returns
219
    -------
220

221
    """
222

223 3
    oformat = None
224 3
    config = None
225 3
    template = None
226 3
    for value, expr in assignments.items():
227 3
        if value == "oformat":
228 3
            if not isinstance(expr, ast.Str):
229 0
                raise ValueError("expected 'oformat' to be a Str; {}".format(expr))
230 3
            oformat = expr.s
231 3
        elif value == "config":
232 3
            if not isinstance(expr, ast.Dict):
233 0
                raise ValueError("expected 'config' to be a Dict; {}".format(expr))
234 3
            config = convert_dict(expr, imported, assignments)
235 3
        elif value == "template":
236 3
            if not isinstance(expr, ast.Call):
237 0
                raise ValueError("expected 'config' to be a call to create_tpl(x)")
238
            # func = expr.func  # TODO make sure func name is create_tpl/tplx
239 3
            args = expr.args
240 3
            keywords = expr.keywords
241 3
            if len(args) != 1 or len(keywords) > 0:
242 0
                raise ValueError("expected create_tpl(x) to have one argument")
243 3
            seg_list = args[0]
244 3
            if isinstance(seg_list, ast.ListComp):
245 3
                seg_list = seg_list.generators[0].iter
246 3
            if not isinstance(seg_list, ast.List):
247 0
                raise ValueError(
248
                    "expected create_tpl(x) arg to be a List; {}".format(seg_list)
249
                )
250 3
            segments = []
251 3
            for seg in seg_list.elts:
252 3
                if isinstance(seg, ast.Attribute):
253 3
                    seg_name = seg.value.id
254 3
                elif isinstance(seg, ast.Name):
255 3
                    seg_name = seg.id
256
                else:
257 0
                    raise ValueError(
258
                        "expected seg in template to be an Attribute; "
259
                        + "{1}".format(seg)
260
                    )
261

262 3
                if seg_name not in imported:
263 0
                    raise ValueError("segment '{}' not found".format(seg_name))
264 3
                segments.append(imported[seg_name])
265 3
            template = segments
266

267 3
    if oformat is None:
268 0
        raise ValueError("could not find 'oformat' assignment")
269 3
    if config is None:
270 0
        raise ValueError("could not find 'config' assignment")
271 3
    if template is None:
272 0
        raise ValueError("could not find 'template' assignment")
273

274 3
    exporter_class, outline = convert_oformat(oformat)
275 3
    exporter = convert_config(config, exporter_class, allow_other)
276

277 3
    if any(["biblio_natbib" in s for s in template]):
278 3
        exporter["filters"]["strip_ext"] = "ipypublish.filters.filters.strip_ext"
279

280 3
    return {
281
        "description": docstring.splitlines(),
282
        "exporter": exporter,
283
        "template": None
284
        if outline is None
285
        else {
286
            "outline": outline,
287
            "segments": [replace_template_path(s) for s in template],
288
        },
289
    }
290

291

292 3
def convert_to_json(path, outpath=None, ignore_other=False):
293
    """Set docstring here.
294

295
    Parameters
296
    ----------
297
    path: str
298
        input module path
299
    outpath=None: str or None
300
        if set, output json to this path
301
    ignore_other: bool
302
        whether to ignore arguments in config,
303
        which do not relate to preprocessors
304
    Returns
305
    -------
306

307
    """
308 3
    _docstring, _imported, _assignments = assess_syntax(path)
309
    # print(_docstring)
310
    # print()
311
    # print(_imported)
312
    # print()
313
    # print(_assignments)
314 3
    output = create_json(_docstring, _imported, _assignments, not ignore_other)
315 3
    if outpath:
316 0
        with open(outpath, "w") as file_obj:
317 0
            json.dump(output, file_obj, indent=2)
318 3
    return json.dumps(output, indent=2)
319

320

321 3
if __name__ == "__main__":
322

323
    if False:
324
        import glob
325
        import os
326

327
        for path in glob.glob(
328
            "/Users/cjs14/GitHub/ipypublish" "/ipypublish/export_plugins/*.py"
329
        ):
330
            dirname = os.path.dirname(path)
331
            name = os.path.splitext(os.path.basename(path))[0]
332
            try:
333
                convert_to_json(
334
                    path, os.path.join(dirname, name + ".json"), ignore_other=True
335
                )
336
            except ValueError as err:
337
                print("{0} failed: {1}".format(path, err))
338

339 0
    convert_to_json(
340
        "/Users/cjs14/GitHub/ipypublish" "/ipypublish_plugins/example_new_plugin.py",
341
        "/Users/cjs14/GitHub/ipypublish" "/ipypublish_plugins/example_new_plugin.json",
342
    )

Read our documentation on viewing source code .

Loading