1
import Core: Expr
2

3
# Terminals
4
function julia_normalization_map(c::Int32, x::Ptr{Nothing})::Int32
5 23
    return c == 0x00B5 ? 0x03BC : # micro sign -> greek small letter mu
6
           c == 0x025B ? 0x03B5 : # latin small letter open e -> greek small letter
7
           c
8
end
9

10
# Note: This code should be in julia base
11
function utf8proc_map_custom(str::String, options, func)
12 23
    norm_func = @cfunction $func Int32 (Int32, Ptr{Nothing})
13 23
    nwords = ccall(:utf8proc_decompose_custom, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Int, Cint, Ptr{Nothing}, Ptr{Nothing}),
14
                   str, sizeof(str), C_NULL, 0, options, norm_func, C_NULL)
15 23
    nwords < 0 && Base.Unicode.utf8proc_error(nwords)
16 21
    buffer = Base.StringVector(nwords * 4)
17 23
    nwords = ccall(:utf8proc_decompose_custom, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Int, Cint, Ptr{Nothing}, Ptr{Nothing}),
18
                   str, sizeof(str), buffer, nwords, options, norm_func, C_NULL)
19 23
    nwords < 0 && Base.Unicode.utf8proc_error(nwords)
20 23
    nbytes = ccall(:utf8proc_reencode, Int, (Ptr{UInt8}, Int, Cint), buffer, nwords, options)
21 23
    nbytes < 0 && Base.Unicode.utf8proc_error(nbytes)
22 23
    return String(resize!(buffer, nbytes))
23
end
24

25
function normalize_julia_identifier(str::AbstractString)
26 0
    options = Base.Unicode.UTF8PROC_STABLE | Base.Unicode.UTF8PROC_COMPOSE
27 23
    utf8proc_map_custom(String(str), options, julia_normalization_map)
28
end
29

30

31
function sized_uint_literal(s::AbstractString, b::Integer)
32
    # We know integers are all ASCII, so we can use sizeof to compute
33
    # the length of ths string more quickly
34 23
    l = (sizeof(s) - 2) * b
35 23
    l <= 8   && return Base.parse(UInt8,   s)
36 23
    l <= 16  && return Base.parse(UInt16,  s)
37 23
    l <= 32  && return Base.parse(UInt32,  s)
38 23
    l <= 64  && return Base.parse(UInt64,  s)
39
    # l <= 128 && return Base.parse(UInt128, s)
40 23
    l <= 128 && return Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), nothing, s)
41 0
    return Base.parse(BigInt, s)
42
end
43

44
function sized_uint_oct_literal(s::AbstractString)
45 23
    s[3] == 0 && return sized_uint_literal(s, 3)
46 21
    len = sizeof(s)
47 23
    (len < 5  || (len == 5  && s <= "0o377")) && return Base.parse(UInt8, s)
48 23
    (len < 8  || (len == 8  && s <= "0o177777")) && return Base.parse(UInt16, s)
49 0
    (len < 13 || (len == 13 && s <= "0o37777777777")) && return Base.parse(UInt32, s)
50 0
    (len < 24 || (len == 24 && s <= "0o1777777777777777777777")) && return Base.parse(UInt64, s)
51
    # (len < 45 || (len == 45 && s <= "0o3777777777777777777777777777777777777777777")) && return Base.parse(UInt128, s)
52
    # return Base.parse(BigInt, s)
53 0
    (len < 45 || (len == 45 && s <= "0o3777777777777777777777777777777777777777777")) && return Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), nothing, s)
54 0
    return flisp_parse(s)
55
end
56

57
function _literal_expr(x)
58 23
    if kindof(x) === Tokens.TRUE
59 23
        return true
60 23
    elseif kindof(x) === Tokens.FALSE
61 23
        return false
62 23
    elseif is_nothing(x)
63 23
        return nothing
64 23
    elseif kindof(x) === Tokens.INTEGER || kindof(x) === Tokens.BIN_INT || kindof(x) === Tokens.HEX_INT || kindof(x) === Tokens.OCT_INT
65 23
        return Expr_int(x)
66 23
    elseif kindof(x) === Tokens.FLOAT
67 23
        return Expr_float(x)
68 23
    elseif kindof(x) === Tokens.CHAR
69 23
        return Expr_char(x)
70 23
    elseif kindof(x) === Tokens.MACRO
71 0
        return Symbol(valof(x))
72 23
    elseif kindof(x) === Tokens.STRING
73 23
        return valof(x)
74 23
    elseif kindof(x) === Tokens.TRIPLE_STRING
75 23
        return valof(x)
76 23
    elseif kindof(x) === Tokens.CMD
77 23
        return Expr_cmd(x)
78 23
    elseif kindof(x) === Tokens.TRIPLE_CMD
79 23
        return Expr_tcmd(x)
80
    end
81
end
82

83
const TYPEMAX_INT64_STR = string(typemax(Int))
84
const TYPEMAX_INT128_STR = string(typemax(Int128))
85
function Expr_int(x)
86 19
    is_hex = is_oct = is_bin = false
87 23
    val = replace(valof(x), "_" => "")
88 23
    if sizeof(val) > 2 && val[1] == '0'
89 23
        c = val[2]
90 23
        c == 'x' && (is_hex = true)
91 23
        c == 'o' && (is_oct = true)
92 23
        c == 'b' && (is_bin = true)
93
    end
94 23
    is_hex && return sized_uint_literal(val, 4)
95 23
    is_oct && return sized_uint_oct_literal(val)
96 23
    is_bin && return sized_uint_literal(val, 1)
97
    # sizeof(val) <= sizeof(TYPEMAX_INT64_STR) && return Base.parse(Int64, val)
98 23
    return flisp_parse(val)
99
    # # val < TYPEMAX_INT64_STR && return Base.parse(Int64, val)
100
    # sizeof(val) <= sizeof(TYPEMAX_INTval < TYPEMAX_INT128_STR128_STR) && return Base.parse(Int128, val)
101
    # # val < TYPEMAX_INT128_STR && return Base.parse(Int128, val)
102
    # Base.parse(BigInt, val)
103
end
104

105
function Expr_float(x)
106 23
    if !startswith(valof(x), "0x") && 'f' in valof(x)
107 23
        return Base.parse(Float32, replace(valof(x), 'f' => 'e'))
108
    end
109 23
    Base.parse(Float64, replace(valof(x), "_" => ""))
110
end
111
function Expr_char(x)
112 23
    val = _unescape_string(valof(x)[2:prevind(valof(x), lastindex(valof(x)))])
113
    # one byte e.g. '\xff' maybe not valid UTF-8
114
    # but we want to use the raw value as a codepoint in this case
115 23
    sizeof(val) == 1 && return Char(codeunit(val, 1))
116 23
    length(val) == 1 || error("Invalid character literal: $(Vector{UInt8}(valof(x)))")
117 23
    val[1]
118
end
119

120

121
# Expressions
122

123
# Fallback
124
function Expr(x::EXPR)
125 23
    if isidentifier(x)
126 23
        if typof(x) === NONSTDIDENTIFIER
127 12
            Symbol(normalize_julia_identifier(valof(x.args[2])))
128
        else
129 23
            return Symbol(normalize_julia_identifier(valof(x)))
130
        end
131 23
    elseif iskw(x)
132 23
        if kindof(x) === Tokens.BREAK
133 23
            return Expr(:break)
134 23
        elseif kindof(x) === Tokens.CONTINUE
135 23
            return Expr(:continue)
136
        else
137 23
            return Symbol(lowercase(string(kindof(x))))
138
        end
139 23
    elseif isoperator(x)
140 23
        ret = x.val isa String ? Symbol(valof(x)) : UNICODE_OPS_REVERSE[kindof(x)]
141 23
        return x.dot ? Symbol(:., ret) : ret
142 23
    elseif ispunctuation(x)
143 0
        return string(kindof(x))
144 23
    elseif isliteral(x)
145 23
        return _literal_expr(x)
146 23
    elseif isunarycall(x)
147 23
        return _unary_expr(x)
148 23
    elseif isbinarycall(x)
149 23
        return _binary_expr(x)
150 23
    elseif iswherecall(x)
151 23
        return _where_expr(x)
152 23
    elseif typof(x) === ConditionalOpCall
153 23
        return Expr(:if, Expr(x.args[1]), Expr(x.args[3]), Expr(x.args[5]))
154 23
    elseif typof(x) === ErrorToken
155 0
        ret = Expr(:error)
156 0
        if x.args !== nothing
157 0
            for a in x.args
158 0
                if !(ispunctuation(a))
159 0
                    push!(ret.args, Expr(a))
160
                end
161
            end
162
        end
163 0
        return ret
164 23
    elseif typof(x) === ChainOpCall
165 23
        ret = Expr(:call, Expr(x.args[2]))
166 23
        for i = 1:length(x.args)
167 23
            if isodd(i)
168 23
                push!(ret.args, Expr(x.args[i]))
169
            end
170
        end
171 23
        return ret
172 23
    elseif typof(x) === Comparison
173 21
        ret = Expr(:comparison)
174 23
        for a in x.args
175 23
            if !(ispunctuation(a))
176 23
                push!(ret.args, Expr(a))
177
            end
178
        end
179 23
        return ret
180 23
    elseif typof(x) === ColonOpCall
181 23
        return Expr(:call, :(:), Expr(x.args[1]), Expr(x.args[3]), Expr(x.args[5]))
182 23
    elseif typof(x) === TopLevel
183 21
        ret = Expr(:toplevel)
184 23
        for a in x.args
185 23
            if !(ispunctuation(a))
186 23
                push!(ret.args, Expr(a))
187
            end
188
        end
189 23
        return ret
190 23
    elseif typof(x) === MacroName
191 23
        if isidentifier(x.args[2])
192 23
            if valof(x.args[2]) == "."
193 0
                return Symbol("@", "__dot__")
194
            else
195 23
                return Symbol("@", valof(x.args[2]))
196
            end
197 0
        elseif isoperator(x.args[2])
198 0
            return Symbol("@", Expr(x.args[2]))
199

200
        else
201 0
            return Symbol("@")
202
        end
203 23
    elseif typof(x) === MacroCall
204 21
        ret = Expr(:macrocall)
205 23
        for a in x.args
206 23
            if !(ispunctuation(a))
207 23
                push!(ret.args, Expr(a))
208
            end
209
        end
210 23
        insert!(ret.args, 2, nothing)
211 23
        if ret.args[1] isa Expr && ret.args[1].head == :. && string(ret.args[1].args[2].value)[1] != '@'
212 21
            clear_at!(ret.args[1])
213 21
            ret.args[1].args[2] = QuoteNode(Symbol(string('@', ret.args[1].args[2].value)))
214
        end
215 23
        ret
216 23
    elseif typof(x) === x_Str
217 23
        if isbinarycall(x.args[1]) && issyntaxcall(x.args[1].args[2])
218 23
            mname = Expr(x.args[1])
219 23
            mname.args[2] = QuoteNode(Symbol("@", mname.args[2].value, "_str"))
220 23
            ret = Expr(:macrocall, mname, nothing)
221
        else
222 23
            ret = Expr(:macrocall, Symbol("@", valof(x.args[1]), "_str"), nothing)
223
        end
224 23
        for i = 2:length(x.args)
225 23
            push!(ret.args, valof(x.args[i]))
226
        end
227 23
        return ret
228 23
    elseif typof(x) === x_Cmd
229 23
        ret = Expr(:macrocall, Symbol("@", valof(x.args[1]), "_cmd"), nothing)
230 23
        for i = 2:length(x.args)
231 23
            push!(ret.args, valof(x.args[i]))
232
        end
233 23
        return ret
234 23
    elseif typof(x) === Quotenode
235 23
        return QuoteNode(Expr(x.args[end]))
236 23
    elseif typof(x) === Call
237 23
        if kindof(x.args[1]) === Tokens.ISSUBTYPE || kindof(x.args[1]) === Tokens.ISSUPERTYPE
238 23
            ret = Expr(Expr(x.args[1]))
239 23
            for i in 2:length(x.args)
240 23
                a = x.args[i]
241 23
                if typof(a) === Parameters
242 0
                    insert!(ret.args, 2, Expr(a))
243 23
                elseif !(ispunctuation(a))
244 23
                    push!(ret.args, Expr(a))
245
                end
246
            end
247 23
            return ret
248
        end
249 21
        ret = Expr(:call)
250 23
        for a in x.args
251 23
            if typof(a) === Parameters
252 23
                insert!(ret.args, 2, Expr(a))
253 23
            elseif !(ispunctuation(a))
254 23
                push!(ret.args, Expr(a))
255
            end
256
        end
257 23
        return ret
258 23
    elseif typof(x) === Braces
259 21
        ret = Expr(:braces)
260 23
        for a in x.args
261 23
            if typof(a) === Parameters
262 23
                insert!(ret.args, 1, Expr(a))
263 23
            elseif !(ispunctuation(a))
264 23
                push!(ret.args, Expr(a))
265
            end
266
        end
267 23
        return ret
268 23
    elseif typof(x) === BracesCat
269 21
        ret = Expr(:bracescat)
270 23
        for a in x.args
271 23
            if typof(a) === Parameters
272 0
                insert!(ret.args, 1, Expr(a))
273 23
            elseif !(ispunctuation(a))
274 23
                push!(ret.args, Expr(a))
275
            end
276
        end
277 23
        return ret
278 23
    elseif typof(x) === Struct
279 23
        return Expr(:struct, false, Expr(x.args[2]), Expr(x.args[3]))
280 23
    elseif typof(x) === Mutable
281 23
        return length(x.args) == 4 ? Expr(:struct, true, Expr(x.args[2]), Expr(x.args[3])) : Expr(:struct, true, Expr(x.args[3]), Expr(x.args[4]))
282 23
    elseif typof(x) === Abstract
283 23
        return length(x.args) == 2 ? Expr(:abstract, Expr(x.args[2])) : Expr(:abstract, Expr(x.args[3]))
284 23
    elseif typof(x) === Primitive
285 23
        return Expr(:primitive, Expr(x.args[3]), Expr(x.args[4]))
286 23
    elseif typof(x) === FunctionDef
287 21
        ret = Expr(:function)
288 23
        for a in x.args
289 23
            if !(ispunctuation(a) || iskw(a))
290 23
                push!(ret.args, Expr(a))
291
            end
292
        end
293 23
        return ret
294 23
    elseif typof(x) === Macro
295 23
        if length(x.args) == 3
296 23
            Expr(:macro, Expr(x.args[2]))
297
        else
298 23
            Expr(:macro, Expr(x.args[2]), Expr(x.args[3]))
299
        end
300 23
    elseif typof(x) === ModuleH
301 23
        return Expr(:module, true, Expr(x.args[2]), Expr(x.args[3]))
302 23
    elseif typof(x) === BareModule
303 23
        return Expr(:module, false, Expr(x.args[2]), Expr(x.args[3]))
304 23
    elseif typof(x) === If
305 23
        return _if_expr(x)
306 23
    elseif typof(x) === Try
307 21
        ret = Expr(:try)
308 23
        for a in x.args
309 23
            if !(ispunctuation(a) || iskw(a))
310 23
                push!(ret.args, Expr(a))
311
            end
312
        end
313 23
        return ret
314 23
    elseif typof(x) === Let
315 23
        return _let_expr(x)
316 23
    elseif typof(x) === Do
317 23
        return Expr(:do, Expr(x.args[1]), Expr(:->, Expr(x.args[3]), Expr(x.args[4])))
318 23
    elseif typof(x) === Outer
319 23
        return Expr(:outer, Expr(x.args[2]))
320 23
    elseif typof(x) === For
321 21
        ret = Expr(:for)
322 23
        if typof(x.args[2]) === Block
323 21
            arg = Expr(:block)
324 23
            for a in x.args[2].args
325 23
                if !(ispunctuation(a))
326 23
                    push!(arg.args, fix_range(a))
327
                end
328
            end
329 23
            push!(ret.args, arg)
330
        else
331 23
            push!(ret.args, fix_range(x.args[2]))
332
        end
333 23
        push!(ret.args, Expr(x.args[3]))
334 23
        return ret
335 23
    elseif typof(x) === While
336 21
        ret = Expr(:while)
337 23
        for a in x.args
338 23
            if !(ispunctuation(a) || iskw(a))
339 23
                push!(ret.args, Expr(a))
340
            end
341
        end
342 23
        return ret
343 23
    elseif istuple(x)
344 21
        ret = Expr(:tuple)
345 23
        for a in x.args
346 23
            if typof(a) == Parameters
347 23
                insert!(ret.args, 1, Expr(a))
348 23
            elseif !(ispunctuation(a))
349 23
                push!(ret.args, Expr(a))
350
            end
351
        end
352 23
        return ret
353 23
    elseif typof(x) === Curly
354 21
        ret = Expr(:curly)
355 23
        for a in x.args
356 23
            if typof(a) === Parameters
357 23
                insert!(ret.args, 2, Expr(a))
358 23
            elseif !(ispunctuation(a))
359 23
                push!(ret.args, Expr(a))
360
            end
361
        end
362 23
        return ret
363 23
    elseif typof(x) === Vect
364 21
        ret = Expr(:vect)
365 23
        for a in x.args
366 23
            if typof(a) === Parameters
367 23
                pushfirst!(ret.args, Expr(a))
368 23
            elseif !(ispunctuation(a))
369 23
                push!(ret.args, Expr(a))
370
            end
371
        end
372 23
        return ret
373 23
    elseif typof(x) === Row
374 21
        ret = Expr(:row)
375 23
        for a in x.args
376 23
            if !(ispunctuation(a))
377 23
                push!(ret.args, Expr(a))
378
            end
379
        end
380 23
        return ret
381 23
    elseif typof(x) === Hcat
382 21
        ret = Expr(:hcat)
383 23
        for a in x.args
384 23
            if !(ispunctuation(a))
385 23
                push!(ret.args, Expr(a))
386
            end
387
        end
388 23
        return ret
389 23
    elseif typof(x) === Vcat
390 21
        ret = Expr(:vcat)
391 23
        for a in x.args
392 23
            if !(ispunctuation(a))
393 23
                push!(ret.args, Expr(a))
394
            end
395
        end
396 23
        return ret
397 23
    elseif typof(x) === Block
398 21
        ret = Expr(:block)
399 23
        for a in x.args
400 23
            if !(ispunctuation(a))
401 23
                push!(ret.args, Expr(a))
402
            end
403
        end
404 23
        return ret
405 23
    elseif typof(x) === Kw
406 23
        return Expr(:kw, Expr(x.args[1]), Expr(x.args[3]))
407 23
    elseif typof(x) === Parameters
408 21
        ret = Expr(:parameters)
409 23
        for a in x.args
410 23
            if typof(a) === Parameters
411 0
                insert!(ret.args, 2, Expr(a))
412 23
            elseif !(ispunctuation(a))
413 23
                push!(ret.args, Expr(a))
414
            end
415
        end
416 23
        return ret
417 23
    elseif typof(x) === Return
418 21
        ret = Expr(:return)
419 23
        for i = 2:length(x.args)
420 23
            a = x.args[i]
421 23
            push!(ret.args, Expr(a))
422
        end
423 23
        return ret
424 23
    elseif isbracketed(x)
425 23
        return Expr(x.args[2])
426 23
    elseif typof(x) === Begin
427 23
        return Expr(x.args[2])
428 23
    elseif typof(x) === Quote
429 23
        if length(x.args) == 1
430 0
            return Expr(:quote, Expr(x.args[1]))
431 23
        elseif isbracketed(x.args[2]) && (isoperator(x.args[2].args[2]) || isliteral(x.args[2].args[2]) || isidentifier(x.args[2].args[2]))
432 23
            return QuoteNode(Expr(x.args[2]))
433
        else
434 23
            return Expr(:quote, Expr(x.args[2]))
435
        end
436 23
    elseif typof(x) === Global
437 21
        ret = Expr(:global)
438 23
        if typof(x.args[2]) === Const
439 23
            ret = Expr(:const, Expr(:global, Expr(x.args[2].args[2])))
440 23
        elseif length(x.args) == 2 && istuple(x.args[2])
441 23
            for a in x.args[2].args
442 23
                if !(ispunctuation(a))
443 23
                    push!(ret.args, Expr(a))
444
                end
445
            end
446
        else
447 23
            for i = 2:length(x.args)
448 23
                a = x.args[i]
449 23
                push!(ret.args, Expr(a))
450
            end
451
        end
452 23
        return ret
453 23
    elseif typof(x) === Local
454 21
        ret = Expr(:local)
455 23
        if typof(x.args[2]) === Const
456 0
            ret = Expr(:const, Expr(:global, Expr(x.args[2].args[2])))
457 23
        elseif length(x.args) == 2 && istuple(x.args[2])
458 23
            for a in x.args[2].args
459 23
                if !(ispunctuation(a))
460 23
                    push!(ret.args, Expr(a))
461
                end
462
            end
463
        else
464 23
            for i = 2:length(x.args)
465 23
                a = x.args[i]
466 23
                push!(ret.args, Expr(a))
467
            end
468
        end
469 23
        return ret
470 23
    elseif typof(x) === Const
471 21
        ret = Expr(:const)
472 23
        for i = 2:length(x.args)
473 23
            a = x.args[i]
474 23
            push!(ret.args, Expr(a))
475
        end
476 23
        return ret
477 23
    elseif typof(x) === GlobalRefDoc
478 23
        return GlobalRef(Core, Symbol("@doc"))
479 23
    elseif typof(x) === Ref
480 21
        ret = Expr(:ref)
481 23
        for a in x.args
482 23
            if typof(a) === Parameters
483 23
                insert!(ret.args, 2, Expr(a))
484 23
            elseif !(ispunctuation(a))
485 23
                push!(ret.args, Expr(a))
486
            end
487
        end
488 23
        return ret
489 23
    elseif typof(x) === TypedHcat
490 21
        ret = Expr(:typed_hcat)
491 23
        for a in x.args
492 23
            if !(ispunctuation(a))
493 23
                push!(ret.args, Expr(a))
494
            end
495
        end
496 23
        return ret
497 23
    elseif typof(x) === TypedVcat
498 21
        ret = Expr(:typed_vcat)
499 23
        for a in x.args
500 23
            if typof(a) === Parameters
501 0
                insert!(ret.args, 2, Expr(a))
502 23
            elseif !(ispunctuation(a))
503 23
                push!(ret.args, Expr(a))
504
            end
505
        end
506 23
        return ret
507 23
    elseif typof(x) === Comprehension || typof(x) === DictComprehension
508 21
        ret = Expr(:comprehension)
509 23
        for a in x.args
510 23
            if !(ispunctuation(a))
511 23
                push!(ret.args, Expr(a))
512
            end
513
        end
514 23
        return ret
515 23
    elseif typof(x) === Flatten
516 23
        iters, args = get_inner_gen(x)
517 21
        i = popfirst!(iters)
518 23
        ex = Expr(:generator, Expr(args[1]), convert_iter_assign(i[1]))
519 23
        for i in iters
520 23
            if length(i) == 1
521 23
                ex = Expr(:generator, ex, convert_iter_assign(i[1]))
522 23
                ex = Expr(:flatten, ex)
523
            else
524 21
                ex = Expr(:generator, ex)
525 23
                for j in i
526 23
                    push!(ex.args, convert_iter_assign(j))
527
                end
528 23
                ex = Expr(:flatten, ex)
529
            end
530
        end
531 23
        return ex
532 23
    elseif typof(x) === Generator
533 23
        ret = Expr(:generator, Expr(x.args[1]))
534 23
        for i = 3:length(x.args)
535 23
            a = x.args[i]
536 23
            if !(ispunctuation(a))
537 23
                push!(ret.args, convert_iter_assign(a))
538
            end
539
        end
540 23
        return ret
541 23
    elseif typof(x) === Filter
542 21
        ret = Expr(:filter)
543 23
        push!(ret.args, Expr(last(x.args)))
544 23
        for i in 1:length(x.args) - 1
545 23
            a = x.args[i]
546 23
            if !(is_if(a) || ispunctuation(a))
547 23
                push!(ret.args, convert_iter_assign(a))
548
            end
549
        end
550 23
        return ret
551 23
    elseif typof(x) === TypedComprehension
552 21
        ret = Expr(:typed_comprehension)
553 23
        for a in x.args
554 23
            if !(ispunctuation(a))
555 23
                push!(ret.args, Expr(a))
556
            end
557
        end
558 23
        return ret
559 23
    elseif typof(x) === Import
560 23
        return expr_import(x, :import)
561 23
    elseif typof(x) === Using
562 23
        return expr_import(x, :using)
563 23
    elseif typof(x) === Export
564 21
        ret = Expr(:export)
565 23
        for i = 2:length(x.args)
566 23
            a = x.args[i]
567 23
            if !(ispunctuation(a))
568 23
                push!(ret.args, Expr(a))
569
            end
570
        end
571 23
        return ret
572 23
    elseif typof(x) === FileH
573 21
        ret = Expr(:file)
574 23
        for a in x.args
575 23
            push!(ret.args, Expr(a))
576
        end
577 23
        return ret
578 23
    elseif typof(x) === StringH
579 21
        ret = Expr(:string)
580 23
        for (i, a) in enumerate(x.args)
581 23
            if isunarycall(a)
582 23
                a = a.args[2]
583 23
            elseif isliteral(a) && kindof(a) === Tokens.STRING && span(a) == 0 || ((i == 1 || i == length(x.args)) && span(a) == 1) || (valof(a) === nothing || isempty(valof(a)))
584 23
                continue
585 23
            else isliteral(a) && kindof(a) === Tokens.TRIPLE_STRING && span(a) == 0 || ((i == 1 || i == length(x.args)) && span(a) == 3) || (valof(a) === nothing || isempty(valof(a)))
586
            end
587 23
            push!(ret.args, Expr(a))
588
        end
589 23
        return ret
590
    else
591 0
        ret = Expr(:call)
592 0
        for a in x.args
593 0
            if !(ispunctuation(a))
594 0
                push!(ret.args, Expr(a))
595
            end
596
        end
597 0
        return ret
598
    end
599
end
600

601
# Op. expressions
602

603
function _unary_expr(x)
604 23
    if isoperator(x.args[1]) && issyntaxunarycall(x.args[1])
605 23
        Expr(Expr(x.args[1]), Expr(x.args[2]))
606 23
    elseif isoperator(x.args[2]) && issyntaxunarycall(x.args[2])
607 23
        Expr(Expr(x.args[2]), Expr(x.args[1]))
608
    else
609 23
        Expr(:call, Expr(x.args[1]), Expr(x.args[2]))
610
    end
611
end
612
function _binary_expr(x)
613 23
    if issyntaxcall(x.args[2]) && !(kindof(x.args[2]) in (Tokens.COLON,))
614 23
        if kindof(x.args[2]) === Tokens.DOT
615 23
            arg1, arg2 = Expr(x.args[1]), Expr(x.args[3])
616 23
            if arg2 isa Expr && arg2.head === :macrocall && endswith(string(arg2.args[1]), "_cmd")
617 0
                return Expr(:macrocall, Expr(:., arg1, QuoteNode(arg2.args[1])), nothing, arg2.args[3])
618 23
            elseif arg2 isa Expr && arg2.head === :braces
619 23
                return Expr(:., arg1, Expr(:quote, arg2))
620
            end
621
        end
622 23
        Expr(Expr(x.args[2]), Expr(x.args[1]), Expr(x.args[3]))
623
    else
624 23
        Expr(:call, Expr(x.args[2]), Expr(x.args[1]), Expr(x.args[3]))
625
    end
626
end
627

628
function _where_expr(x)
629 23
    ret = Expr(:where, Expr(x.args[1]))
630 23
    for i = 3:length(x.args)
631 23
        a = x.args[i]
632 23
        if typof(a) === Parameters
633 0
            insert!(ret.args, 2, Expr(a))
634 23
        elseif !(ispunctuation(a) || iskw(a))
635 23
            push!(ret.args, Expr(a))
636
        end
637
    end
638 23
    return ret
639
end
640

641

642
# cross compatability for line number insertion in macrocalls
643
if VERSION > v"1.1-"
644 21
    Expr_cmd(x) = Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), nothing, valof(x))
645 21
    Expr_tcmd(x) = Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), nothing, valof(x))
646
else
647 2
    Expr_cmd(x) = Expr(:macrocall, Symbol("@cmd"), nothing, valof(x))
648 2
    Expr_tcmd(x) = Expr(:macrocall, Symbol("@cmd"), nothing, valof(x))
649
end
650

651

652
function clear_at!(x)
653 21
    if x isa Expr && x.head == :.
654 21
        if x.args[2] isa QuoteNode && string(x.args[2].value)[1] == '@'
655 0
            x.args[2].value = Symbol(string(x.args[2].value)[2:end])
656
        end
657 21
        if x.args[1] isa Symbol && string(x.args[1])[1] == '@'
658 21
            x.args[1] = Symbol(string(x.args[1])[2:end])
659
        else
660 0
            clear_at!(x.args[1])
661
        end
662
    end
663
end
664

665

666
"""
667
    remlineinfo!(x)
668
Removes line info expressions. (i.e. Expr(:line, 1))
669
"""
670
function remlineinfo!(x)
671 23
    if isa(x, Expr)
672 23
        if x.head == :macrocall && x.args[2] !== nothing
673 23
            id = findall(map(x -> (isa(x, Expr) && x.head == :line) || (@isdefined(LineNumberNode) && x isa LineNumberNode), x.args))
674 21
            deleteat!(x.args, id)
675 23
            for j in x.args
676 23
                remlineinfo!(j)
677
            end
678 23
            insert!(x.args, 2, nothing)
679
        else
680 23
            id = findall(map(x -> (isa(x, Expr) && x.head == :line) || (@isdefined(LineNumberNode) && x isa LineNumberNode), x.args))
681 21
            deleteat!(x.args, id)
682 23
            for j in x.args
683 23
                remlineinfo!(j)
684
            end
685
        end
686 23
        if x.head == :elseif && x.args[1] isa Expr && x.args[1].head == :block && length(x.args[1].args) == 1
687 23
            x.args[1] = x.args[1].args[1]
688
        end
689
    end
690 23
    x
691
end
692

693
function _if_expr(x)
694 23
    ret = Expr(:if)
695 0
    iselseif = false
696 23
    n = length(x.args)
697 0
    i = 0
698 23
    while i < n
699 21
        i += 1
700 23
        a = x.args[i]
701 23
        if iskw(a) && kindof(a) === Tokens.ELSEIF
702 21
            i += 1
703 23
            r1 = Expr(x.args[i].args[1])
704 23
            push!(ret.args, Expr(:elseif, r1.args...))
705 23
        elseif !(ispunctuation(a) || iskw(a))
706 23
            push!(ret.args, Expr(a))
707
        end
708
    end
709 23
    ret
710
end
711

712
function _let_expr(x)
713 23
    ret = Expr(:let)
714 23
    if length(x.args) == 3
715 21
        push!(ret.args, Expr(:block))
716 23
        push!(ret.args, Expr(x.args[2]))
717 23
        return ret
718 23
    elseif typof(x.args[2]) === Block
719 21
        arg = Expr(:block)
720 23
        for a in x.args[2].args
721 23
            if !(ispunctuation(a))
722 23
                push!(arg.args, fix_range(a))
723
            end
724
        end
725 23
        push!(ret.args, arg)
726
    else
727 23
        push!(ret.args, fix_range(x.args[2]))
728
    end
729 23
    push!(ret.args, Expr(x.args[3]))
730 23
    ret
731
end
732

733
function fix_range(a)
734 23
    if isbinarycall(a) && (is_in(a.args[2]) || is_elof(a.args[2]))
735 23
        Expr(:(=), Expr(a.args[1]), Expr(a.args[3]))
736
    else
737 23
        Expr(a)
738
    end
739
end
740

741
function get_inner_gen(x, iters=[], arg=[])
742 23
    if typof(x) == Flatten
743 23
        get_inner_gen(x.args[1], iters, arg)
744 23
    elseif typof(x) === Generator
745
        # push!(iters, get_iter(x))
746 23
        get_iters(x, iters)
747 23
        if typof(x.args[1]) === Generator || typof(x.args[1]) === Flatten
748 23
            get_inner_gen(x.args[1], iters, arg)
749
        else
750 23
            push!(arg, x.args[1])
751
        end
752
    end
753 23
    return iters, arg
754
end
755

756 0
function get_iter(x)
757 0
    if typof(x) === Generator
758 0
        return x.args[3]
759
    end
760
end
761

762
function get_iters(x, iters)
763 23
    iters1 = []
764 23
    if typof(x) === Generator
765
        # return x.args[3]
766 23
        for i = 3:length(x.args)
767 23
            if typof(x.args[i]) !== PUNCTUATION
768 23
                push!(iters1, x.args[i])
769
            end
770
        end
771
    end
772 23
    push!(iters, iters1)
773
end
774

775
function convert_iter_assign(a)
776 23
    if isbinarycall(a) && (is_in(a.args[2]) || is_elof(a.args[2]))
777 23
        return Expr(:(=), Expr(a.args[1]), Expr(a.args[3]))
778
    else
779 23
        return Expr(a)
780
    end
781
end
782

783 0
function _get_import_block(x, i, ret)
784 0
    while is_dot(x.args[i + 1])
785 0
        i += 1
786 0
        push!(ret.args, :.)
787
    end
788 0
    while i < length(x.args) && !(is_comma(x.args[i + 1]))
789 0
        i += 1
790 0
        a = x.args[i]
791 0
        if !(ispunctuation(a)) && !(is_dot(a) || is_colon(a))
792 0
            push!(ret.args, Expr(a))
793
        end
794
    end
795

796 0
    return i
797
end
798

799
function expr_import(x, kw)
800 23
    col = findall(a -> isoperator(a) && precedence(a) == ColonOp, x.args)
801 23
    comma = findall(is_comma, x.args)
802

803 21
    header = []
804 23
    args = [Expr(:.)]
805 0
    i = 1 # skip keyword
806 23
    while i < length(x.args)
807 21
        i += 1
808 23
        a = x.args[i]
809 23
        if is_colon(a)
810 21
            push!(header, popfirst!(args))
811 23
            push!(args, Expr(:.))
812 23
        elseif is_comma(a)
813 23
            push!(args, Expr(:.))
814 23
        elseif !(ispunctuation(a))
815 23
            push!(last(args).args, Expr(a))
816
        end
817
    end
818 23
    if isempty(header)
819 23
        return Expr(kw, args...)
820
    else
821 23
        return Expr(kw, Expr(:(:), header..., args...))
822
    end
823
end
824

825

826

827
const UNICODE_OPS_REVERSE = Dict{Tokenize.Tokens.Kind,Symbol}()
828 17
for (k, v) in Tokenize.Tokens.UNICODE_OPS
829 21
    UNICODE_OPS_REVERSE[v] = Symbol(k)
830
end
831

832
UNICODE_OPS_REVERSE[Tokens.EQ] = :(=)
833
UNICODE_OPS_REVERSE[Tokens.PLUS_EQ] = :(+=)
834
UNICODE_OPS_REVERSE[Tokens.MINUS_EQ] = :(-=)
835
UNICODE_OPS_REVERSE[Tokens.STAR_EQ] = :(*=)
836
UNICODE_OPS_REVERSE[Tokens.FWD_SLASH_EQ] = :(/=)
837
UNICODE_OPS_REVERSE[Tokens.FWDFWD_SLASH_EQ] = :(//=)
838
UNICODE_OPS_REVERSE[Tokens.OR_EQ] = :(|=)
839
UNICODE_OPS_REVERSE[Tokens.CIRCUMFLEX_EQ] = :(^=)
840
UNICODE_OPS_REVERSE[Tokens.DIVISION_EQ] = :(÷=)
841
UNICODE_OPS_REVERSE[Tokens.REM_EQ] = :(%=)
842
UNICODE_OPS_REVERSE[Tokens.LBITSHIFT_EQ] = :(<<=)
843
UNICODE_OPS_REVERSE[Tokens.RBITSHIFT_EQ] = :(>>=)
844
UNICODE_OPS_REVERSE[Tokens.LBITSHIFT] = :(<<)
845
UNICODE_OPS_REVERSE[Tokens.RBITSHIFT] = :(>>)
846
UNICODE_OPS_REVERSE[Tokens.UNSIGNED_BITSHIFT] = :(>>>)
847
UNICODE_OPS_REVERSE[Tokens.UNSIGNED_BITSHIFT_EQ] = :(>>>=)
848
UNICODE_OPS_REVERSE[Tokens.BACKSLASH_EQ] = :(\=)
849
UNICODE_OPS_REVERSE[Tokens.AND_EQ] = :(&=)
850
UNICODE_OPS_REVERSE[Tokens.COLON_EQ] = :(:=)
851
UNICODE_OPS_REVERSE[Tokens.PAIR_ARROW] = :(=>)
852
UNICODE_OPS_REVERSE[Tokens.APPROX] = :(~)
853
UNICODE_OPS_REVERSE[Tokens.EX_OR_EQ] = :($=)
854
UNICODE_OPS_REVERSE[Tokens.XOR_EQ] = :(=)
855
UNICODE_OPS_REVERSE[Tokens.RIGHT_ARROW] = :(-->)
856
UNICODE_OPS_REVERSE[Tokens.LAZY_OR] = :(||)
857
UNICODE_OPS_REVERSE[Tokens.LAZY_AND] = :(&&)
858
UNICODE_OPS_REVERSE[Tokens.ISSUBTYPE] = :(<:)
859
UNICODE_OPS_REVERSE[Tokens.ISSUPERTYPE] = :(>:)
860
UNICODE_OPS_REVERSE[Tokens.GREATER] = :(>)
861
UNICODE_OPS_REVERSE[Tokens.LESS] = :(<)
862
UNICODE_OPS_REVERSE[Tokens.GREATER_EQ] = :(>=)
863
UNICODE_OPS_REVERSE[Tokens.GREATER_THAN_OR_EQUAL_TO] = :()
864
UNICODE_OPS_REVERSE[Tokens.LESS_EQ] = :(<=)
865
UNICODE_OPS_REVERSE[Tokens.LESS_THAN_OR_EQUAL_TO] = :()
866
UNICODE_OPS_REVERSE[Tokens.EQEQ] = :(==)
867
UNICODE_OPS_REVERSE[Tokens.EQEQEQ] = :(===)
868
UNICODE_OPS_REVERSE[Tokens.IDENTICAL_TO] = :()
869
UNICODE_OPS_REVERSE[Tokens.NOT_EQ] = :(!=)
870
UNICODE_OPS_REVERSE[Tokens.NOT_EQUAL_TO] = :()
871
UNICODE_OPS_REVERSE[Tokens.NOT_IS] = :(!==)
872
UNICODE_OPS_REVERSE[Tokens.NOT_IDENTICAL_TO] = :()
873
UNICODE_OPS_REVERSE[Tokens.IN] = :(in)
874
UNICODE_OPS_REVERSE[Tokens.ISA] = :(isa)
875
UNICODE_OPS_REVERSE[Tokens.LPIPE] = :(<|)
876
UNICODE_OPS_REVERSE[Tokens.RPIPE] = :(|>)
877
UNICODE_OPS_REVERSE[Tokens.COLON] = :(:)
878
UNICODE_OPS_REVERSE[Tokens.DDOT] = :(..)
879
UNICODE_OPS_REVERSE[Tokens.EX_OR] = :($)
880
UNICODE_OPS_REVERSE[Tokens.PLUS] = :(+)
881
UNICODE_OPS_REVERSE[Tokens.MINUS] = :(-)
882
UNICODE_OPS_REVERSE[Tokens.PLUSPLUS] = :(++)
883
UNICODE_OPS_REVERSE[Tokens.OR] = :(|)
884
UNICODE_OPS_REVERSE[Tokens.STAR] = :(*)
885
UNICODE_OPS_REVERSE[Tokens.FWD_SLASH] = :(/)
886
UNICODE_OPS_REVERSE[Tokens.REM] = :(%)
887
UNICODE_OPS_REVERSE[Tokens.BACKSLASH] = :(\)
888
UNICODE_OPS_REVERSE[Tokens.AND] = :(&)
889
UNICODE_OPS_REVERSE[Tokens.FWDFWD_SLASH] = :(//)
890
UNICODE_OPS_REVERSE[Tokens.CIRCUMFLEX_ACCENT] = :(^)
891
UNICODE_OPS_REVERSE[Tokens.DECLARATION] = :(::)
892
UNICODE_OPS_REVERSE[Tokens.CONDITIONAL] = :?
893
UNICODE_OPS_REVERSE[Tokens.DOT] = :(.)
894
UNICODE_OPS_REVERSE[Tokens.NOT] = :(!)
895
UNICODE_OPS_REVERSE[Tokens.PRIME] = Symbol(''')
896
UNICODE_OPS_REVERSE[Tokens.DDDOT] = :(...)
897
UNICODE_OPS_REVERSE[Tokens.TRANSPOSE] = Symbol(".'")
898
UNICODE_OPS_REVERSE[Tokens.ANON_FUNC] = :(->)
899
UNICODE_OPS_REVERSE[Tokens.WHERE] = :where

Read our documentation on viewing source code .

Loading