@@ -51,31 +51,29 @@
Loading
51 51
    # (len < 45 || (len == 45 && s <= "0o3777777777777777777777777777777777777777777")) && return Base.parse(UInt128, s)
52 52
    # return Base.parse(BigInt, s)
53 53
    (len < 45 || (len == 45 && s <= "0o3777777777777777777777777777777777777777777")) && return Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), nothing, s)
54 -
    return flisp_parse(s)
54 +
    return Meta.parse(s)
55 55
end
56 56
57 57
function _literal_expr(x)
58 -
    if kindof(x) === Tokens.TRUE
58 +
    if headof(x) === :TRUE
59 59
        return true
60 -
    elseif kindof(x) === Tokens.FALSE
60 +
    elseif headof(x) === :FALSE
61 61
        return false
62 62
    elseif is_nothing(x)
63 63
        return nothing
64 -
    elseif kindof(x) === Tokens.INTEGER || kindof(x) === Tokens.BIN_INT || kindof(x) === Tokens.HEX_INT || kindof(x) === Tokens.OCT_INT
64 +
    elseif headof(x) === :INTEGER || headof(x) === :BININT || headof(x) === :HEXINT || headof(x) === :OCTINT
65 65
        return Expr_int(x)
66 -
    elseif kindof(x) === Tokens.FLOAT
66 +
    elseif isfloat(x)
67 67
        return Expr_float(x)
68 -
    elseif kindof(x) === Tokens.CHAR
68 +
    elseif ischar(x)
69 69
        return Expr_char(x)
70 -
    elseif kindof(x) === Tokens.MACRO
70 +
    elseif headof(x) === :MACRO
71 71
        return Symbol(valof(x))
72 -
    elseif kindof(x) === Tokens.STRING
72 +
    elseif headof(x) === :STRING || headof(x) === :TRIPLESTRING
73 73
        return valof(x)
74 -
    elseif kindof(x) === Tokens.TRIPLE_STRING
75 -
        return valof(x)
76 -
    elseif kindof(x) === Tokens.CMD
74 +
    elseif headof(x) === :CMD
77 75
        return Expr_cmd(x)
78 -
    elseif kindof(x) === Tokens.TRIPLE_CMD
76 +
    elseif headof(x) === :TRIPLECMD
79 77
        return Expr_tcmd(x)
80 78
    end
81 79
end
@@ -95,7 +93,7 @@
Loading
95 93
    is_oct && return sized_uint_oct_literal(val)
96 94
    is_bin && return sized_uint_literal(val, 1)
97 95
    # sizeof(val) <= sizeof(TYPEMAX_INT64_STR) && return Base.parse(Int64, val)
98 -
    return flisp_parse(val)
96 +
    return Meta.parse(val)
99 97
    # # val < TYPEMAX_INT64_STR && return Base.parse(Int64, val)
100 98
    # sizeof(val) <= sizeof(TYPEMAX_INTval < TYPEMAX_INT128_STR128_STR) && return Base.parse(Int128, val)
101 99
    # # val < TYPEMAX_INT128_STR && return Base.parse(Int128, val)
@@ -120,525 +118,71 @@
Loading
120 118
121 119
# Expressions
122 120
123 -
# Fallback
124 121
function Expr(x::EXPR)
125 122
    if isidentifier(x)
126 -
        if typof(x) === NONSTDIDENTIFIER
123 +
        if headof(x) === :NONSTDIDENTIFIER
127 124
            Symbol(normalize_julia_identifier(valof(x.args[2])))
128 125
        else
129 126
            return Symbol(normalize_julia_identifier(valof(x)))
130 127
        end
131 -
    elseif iskw(x)
132 -
        if kindof(x) === Tokens.BREAK
128 +
    elseif iskeyword(x)
129 +
        if headof(x) === :BREAK
133 130
            return Expr(:break)
134 -
        elseif kindof(x) === Tokens.CONTINUE
131 +
        elseif headof(x) === :CONTINUE
135 132
            return Expr(:continue)
136 133
        else
137 -
            return Symbol(lowercase(string(kindof(x))))
134 +
            return Symbol(lowercase(string(headof(x))))
138 135
        end
139 136
    elseif isoperator(x)
140 -
        ret = x.val isa String ? Symbol(valof(x)) : UNICODE_OPS_REVERSE[kindof(x)]
141 -
        return x.dot ? Symbol(:., ret) : ret
137 +
        return Symbol(valof(x))
142 138
    elseif ispunctuation(x)
143 -
        return string(kindof(x))
139 +
        if headof(x) === :DOT
140 +
            return :(.)
141 +
        else 
142 +
            error()
143 +
        end
144 144
    elseif isliteral(x)
145 145
        return _literal_expr(x)
146 -
    elseif isunarycall(x)
147 -
        return _unary_expr(x)
148 -
    elseif isbinarycall(x)
149 -
        return _binary_expr(x)
150 -
    elseif iswherecall(x)
151 -
        return _where_expr(x)
152 -
    elseif typof(x) === ConditionalOpCall
153 -
        return Expr(:if, Expr(x.args[1]), Expr(x.args[3]), Expr(x.args[5]))
154 -
    elseif typof(x) === ErrorToken
155 -
        ret = Expr(:error)
156 -
        if x.args !== nothing
157 -
            for a in x.args
158 -
                if !(ispunctuation(a))
159 -
                    push!(ret.args, Expr(a))
160 -
                end
161 -
            end
162 -
        end
163 -
        return ret
164 -
    elseif typof(x) === ChainOpCall
165 -
        ret = Expr(:call, Expr(x.args[2]))
166 -
        for i = 1:length(x.args)
167 -
            if isodd(i)
168 -
                push!(ret.args, Expr(x.args[i]))
169 -
            end
170 -
        end
171 -
        return ret
172 -
    elseif typof(x) === Comparison
173 -
        ret = Expr(:comparison)
174 -
        for a in x.args
175 -
            if !(ispunctuation(a))
176 -
                push!(ret.args, Expr(a))
177 -
            end
178 -
        end
179 -
        return ret
180 -
    elseif typof(x) === ColonOpCall
181 -
        return Expr(:call, :(:), Expr(x.args[1]), Expr(x.args[3]), Expr(x.args[5]))
182 -
    elseif typof(x) === TopLevel
183 -
        ret = Expr(:toplevel)
184 -
        for a in x.args
185 -
            if !(ispunctuation(a))
186 -
                push!(ret.args, Expr(a))
187 -
            end
188 -
        end
189 -
        return ret
190 -
    elseif typof(x) === MacroName
191 -
        if isidentifier(x.args[2])
192 -
            if valof(x.args[2]) == "."
193 -
                return Symbol("@", "__dot__")
194 -
            else
195 -
                return Symbol("@", valof(x.args[2]))
196 -
            end
197 -
        elseif isoperator(x.args[2])
198 -
            return Symbol("@", Expr(x.args[2]))
199 -
200 -
        else
201 -
            return Symbol("@")
202 -
        end
203 -
    elseif typof(x) === MacroCall
204 -
        ret = Expr(:macrocall)
205 -
        for a in x.args
206 -
            if !(ispunctuation(a))
207 -
                push!(ret.args, Expr(a))
208 -
            end
209 -
        end
210 -
        insert!(ret.args, 2, nothing)
211 -
        if ret.args[1] isa Expr && ret.args[1].head == :. && string(ret.args[1].args[2].value)[1] != '@'
212 -
            clear_at!(ret.args[1])
213 -
            ret.args[1].args[2] = QuoteNode(Symbol(string('@', ret.args[1].args[2].value)))
214 -
        end
215 -
        ret
216 -
    elseif typof(x) === x_Str
217 -
        if isbinarycall(x.args[1]) && issyntaxcall(x.args[1].args[2])
218 -
            mname = Expr(x.args[1])
219 -
            mname.args[2] = QuoteNode(Symbol("@", mname.args[2].value, "_str"))
220 -
            ret = Expr(:macrocall, mname, nothing)
221 -
        else
222 -
            ret = Expr(:macrocall, Symbol("@", valof(x.args[1]), "_str"), nothing)
223 -
        end
224 -
        for i = 2:length(x.args)
225 -
            push!(ret.args, valof(x.args[i]))
226 -
        end
227 -
        return ret
228 -
    elseif typof(x) === x_Cmd
229 -
        ret = Expr(:macrocall, Symbol("@", valof(x.args[1]), "_cmd"), nothing)
230 -
        for i = 2:length(x.args)
231 -
            push!(ret.args, valof(x.args[i]))
232 -
        end
233 -
        return ret
234 -
    elseif typof(x) === Quotenode
235 -
        return QuoteNode(Expr(x.args[end]))
236 -
    elseif typof(x) === Call
237 -
        if kindof(x.args[1]) === Tokens.ISSUBTYPE || kindof(x.args[1]) === Tokens.ISSUPERTYPE
238 -
            ret = Expr(Expr(x.args[1]))
239 -
            for i in 2:length(x.args)
240 -
                a = x.args[i]
241 -
                if typof(a) === Parameters
242 -
                    insert!(ret.args, 2, Expr(a))
243 -
                elseif !(ispunctuation(a))
244 -
                    push!(ret.args, Expr(a))
245 -
                end
246 -
            end
247 -
            return ret
248 -
        end
249 -
        ret = Expr(:call)
250 -
        for a in x.args
251 -
            if typof(a) === Parameters
252 -
                insert!(ret.args, 2, Expr(a))
253 -
            elseif !(ispunctuation(a))
254 -
                push!(ret.args, Expr(a))
255 -
            end
256 -
        end
257 -
        return ret
258 -
    elseif typof(x) === Braces
259 -
        ret = Expr(:braces)
260 -
        for a in x.args
261 -
            if typof(a) === Parameters
262 -
                insert!(ret.args, 1, Expr(a))
263 -
            elseif !(ispunctuation(a))
264 -
                push!(ret.args, Expr(a))
265 -
            end
266 -
        end
267 -
        return ret
268 -
    elseif typof(x) === BracesCat
269 -
        ret = Expr(:bracescat)
270 -
        for a in x.args
271 -
            if typof(a) === Parameters
272 -
                insert!(ret.args, 1, Expr(a))
273 -
            elseif !(ispunctuation(a))
274 -
                push!(ret.args, Expr(a))
275 -
            end
276 -
        end
277 -
        return ret
278 -
    elseif typof(x) === Struct
279 -
        return Expr(:struct, false, Expr(x.args[2]), Expr(x.args[3]))
280 -
    elseif typof(x) === Mutable
281 -
        return length(x.args) == 4 ? Expr(:struct, true, Expr(x.args[2]), Expr(x.args[3])) : Expr(:struct, true, Expr(x.args[3]), Expr(x.args[4]))
282 -
    elseif typof(x) === Abstract
283 -
        return length(x.args) == 2 ? Expr(:abstract, Expr(x.args[2])) : Expr(:abstract, Expr(x.args[3]))
284 -
    elseif typof(x) === Primitive
285 -
        return Expr(:primitive, Expr(x.args[3]), Expr(x.args[4]))
286 -
    elseif typof(x) === FunctionDef
287 -
        ret = Expr(:function)
288 -
        for a in x.args
289 -
            if !(ispunctuation(a) || iskw(a))
290 -
                push!(ret.args, Expr(a))
291 -
            end
292 -
        end
293 -
        return ret
294 -
    elseif typof(x) === Macro
295 -
        if length(x.args) == 3
296 -
            Expr(:macro, Expr(x.args[2]))
297 -
        else
298 -
            Expr(:macro, Expr(x.args[2]), Expr(x.args[3]))
299 -
        end
300 -
    elseif typof(x) === ModuleH
301 -
        return Expr(:module, true, Expr(x.args[2]), Expr(x.args[3]))
302 -
    elseif typof(x) === BareModule
303 -
        return Expr(:module, false, Expr(x.args[2]), Expr(x.args[3]))
304 -
    elseif typof(x) === If
305 -
        return _if_expr(x)
306 -
    elseif typof(x) === Try
307 -
        ret = Expr(:try)
308 -
        for a in x.args
309 -
            if !(ispunctuation(a) || iskw(a))
310 -
                push!(ret.args, Expr(a))
311 -
            end
312 -
        end
313 -
        return ret
314 -
    elseif typof(x) === Let
315 -
        return _let_expr(x)
316 -
    elseif typof(x) === Do
317 -
        return Expr(:do, Expr(x.args[1]), Expr(:->, Expr(x.args[3]), Expr(x.args[4])))
318 -
    elseif typof(x) === Outer
319 -
        return Expr(:outer, Expr(x.args[2]))
320 -
    elseif typof(x) === For
321 -
        ret = Expr(:for)
322 -
        if typof(x.args[2]) === Block
323 -
            arg = Expr(:block)
324 -
            for a in x.args[2].args
325 -
                if !(ispunctuation(a))
326 -
                    push!(arg.args, fix_range(a))
327 -
                end
328 -
            end
329 -
            push!(ret.args, arg)
330 -
        else
331 -
            push!(ret.args, fix_range(x.args[2]))
332 -
        end
333 -
        push!(ret.args, Expr(x.args[3]))
334 -
        return ret
335 -
    elseif typof(x) === While
336 -
        ret = Expr(:while)
337 -
        for a in x.args
338 -
            if !(ispunctuation(a) || iskw(a))
339 -
                push!(ret.args, Expr(a))
340 -
            end
341 -
        end
342 -
        return ret
343 -
    elseif istuple(x)
344 -
        ret = Expr(:tuple)
345 -
        for a in x.args
346 -
            if typof(a) == Parameters
347 -
                insert!(ret.args, 1, Expr(a))
348 -
            elseif !(ispunctuation(a))
349 -
                push!(ret.args, Expr(a))
350 -
            end
351 -
        end
352 -
        return ret
353 -
    elseif typof(x) === Curly
354 -
        ret = Expr(:curly)
355 -
        for a in x.args
356 -
            if typof(a) === Parameters
357 -
                insert!(ret.args, 2, Expr(a))
358 -
            elseif !(ispunctuation(a))
359 -
                push!(ret.args, Expr(a))
360 -
            end
361 -
        end
362 -
        return ret
363 -
    elseif typof(x) === Vect
364 -
        ret = Expr(:vect)
365 -
        for a in x.args
366 -
            if typof(a) === Parameters
367 -
                pushfirst!(ret.args, Expr(a))
368 -
            elseif !(ispunctuation(a))
369 -
                push!(ret.args, Expr(a))
370 -
            end
371 -
        end
372 -
        return ret
373 -
    elseif typof(x) === Row
374 -
        ret = Expr(:row)
375 -
        for a in x.args
376 -
            if !(ispunctuation(a))
377 -
                push!(ret.args, Expr(a))
378 -
            end
379 -
        end
380 -
        return ret
381 -
    elseif typof(x) === Hcat
382 -
        ret = Expr(:hcat)
383 -
        for a in x.args
384 -
            if !(ispunctuation(a))
385 -
                push!(ret.args, Expr(a))
386 -
            end
387 -
        end
388 -
        return ret
389 -
    elseif typof(x) === Vcat
390 -
        ret = Expr(:vcat)
391 -
        for a in x.args
392 -
            if !(ispunctuation(a))
393 -
                push!(ret.args, Expr(a))
394 -
            end
395 -
        end
396 -
        return ret
397 -
    elseif typof(x) === Block
398 -
        ret = Expr(:block)
399 -
        for a in x.args
400 -
            if !(ispunctuation(a))
401 -
                push!(ret.args, Expr(a))
402 -
            end
403 -
        end
404 -
        return ret
405 -
    elseif typof(x) === Kw
406 -
        return Expr(:kw, Expr(x.args[1]), Expr(x.args[3]))
407 -
    elseif typof(x) === Parameters
408 -
        ret = Expr(:parameters)
409 -
        for a in x.args
410 -
            if typof(a) === Parameters
411 -
                insert!(ret.args, 2, Expr(a))
412 -
            elseif !(ispunctuation(a))
413 -
                push!(ret.args, Expr(a))
414 -
            end
415 -
        end
416 -
        return ret
417 -
    elseif typof(x) === Return
418 -
        ret = Expr(:return)
419 -
        for i = 2:length(x.args)
420 -
            a = x.args[i]
421 -
            push!(ret.args, Expr(a))
422 -
        end
423 -
        return ret
424 146
    elseif isbracketed(x)
425 -
        return Expr(x.args[2])
426 -
    elseif typof(x) === Begin
427 -
        return Expr(x.args[2])
428 -
    elseif typof(x) === Quote
429 -
        if length(x.args) == 1
430 -
            return Expr(:quote, Expr(x.args[1]))
431 -
        elseif isbracketed(x.args[2]) && (isoperator(x.args[2].args[2]) || isliteral(x.args[2].args[2]) || isidentifier(x.args[2].args[2]))
432 -
            return QuoteNode(Expr(x.args[2]))
147 +
        return Expr(x.args[1])
148 +
    elseif x.head isa EXPR
149 +
        Expr(Expr(x.head), Expr.(x.args)...)
150 +
    elseif x.head === :quotenode
151 +
        QuoteNode(Expr(x.args[1]))
152 +
    elseif x.head === :globalrefdoc
153 +
        GlobalRef(Core, Symbol("@doc"))
154 +
    elseif x.head === :globalrefcmd
155 +
        if VERSION > v"1.1"
156 +
            GlobalRef(Core, Symbol("@cmd"))
433 157
        else
434 -
            return Expr(:quote, Expr(x.args[2]))
435 -
        end
436 -
    elseif typof(x) === Global
437 -
        ret = Expr(:global)
438 -
        if typof(x.args[2]) === Const
439 -
            ret = Expr(:const, Expr(:global, Expr(x.args[2].args[2])))
440 -
        elseif length(x.args) == 2 && istuple(x.args[2])
441 -
            for a in x.args[2].args
442 -
                if !(ispunctuation(a))
443 -
                    push!(ret.args, Expr(a))
444 -
                end
445 -
            end
446 -
        else
447 -
            for i = 2:length(x.args)
448 -
                a = x.args[i]
449 -
                push!(ret.args, Expr(a))
450 -
            end
451 -
        end
452 -
        return ret
453 -
    elseif typof(x) === Local
454 -
        ret = Expr(:local)
455 -
        if typof(x.args[2]) === Const
456 -
            ret = Expr(:const, Expr(:global, Expr(x.args[2].args[2])))
457 -
        elseif length(x.args) == 2 && istuple(x.args[2])
458 -
            for a in x.args[2].args
459 -
                if !(ispunctuation(a))
460 -
                    push!(ret.args, Expr(a))
461 -
                end
462 -
            end
463 -
        else
464 -
            for i = 2:length(x.args)
465 -
                a = x.args[i]
466 -
                push!(ret.args, Expr(a))
467 -
            end
468 -
        end
469 -
        return ret
470 -
    elseif typof(x) === Const
471 -
        ret = Expr(:const)
472 -
        for i = 2:length(x.args)
473 -
            a = x.args[i]
474 -
            push!(ret.args, Expr(a))
475 -
        end
476 -
        return ret
477 -
    elseif typof(x) === GlobalRefDoc
478 -
        return GlobalRef(Core, Symbol("@doc"))
479 -
    elseif typof(x) === Ref
480 -
        ret = Expr(:ref)
481 -
        for a in x.args
482 -
            if typof(a) === Parameters
483 -
                insert!(ret.args, 2, Expr(a))
484 -
            elseif !(ispunctuation(a))
485 -
                push!(ret.args, Expr(a))
486 -
            end
487 -
        end
488 -
        return ret
489 -
    elseif typof(x) === TypedHcat
490 -
        ret = Expr(:typed_hcat)
491 -
        for a in x.args
492 -
            if !(ispunctuation(a))
493 -
                push!(ret.args, Expr(a))
494 -
            end
495 -
        end
496 -
        return ret
497 -
    elseif typof(x) === TypedVcat
498 -
        ret = Expr(:typed_vcat)
499 -
        for a in x.args
500 -
            if typof(a) === Parameters
501 -
                insert!(ret.args, 2, Expr(a))
502 -
            elseif !(ispunctuation(a))
503 -
                push!(ret.args, Expr(a))
504 -
            end
505 -
        end
506 -
        return ret
507 -
    elseif typof(x) === Comprehension || typof(x) === DictComprehension
508 -
        ret = Expr(:comprehension)
509 -
        for a in x.args
510 -
            if !(ispunctuation(a))
511 -
                push!(ret.args, Expr(a))
512 -
            end
513 -
        end
514 -
        return ret
515 -
    elseif typof(x) === Flatten
516 -
        iters, args = get_inner_gen(x)
517 -
        i = popfirst!(iters)
518 -
        ex = Expr(:generator, Expr(args[1]), convert_iter_assign(i[1]))
519 -
        for i in iters
520 -
            if length(i) == 1
521 -
                ex = Expr(:generator, ex, convert_iter_assign(i[1]))
522 -
                ex = Expr(:flatten, ex)
523 -
            else
524 -
                ex = Expr(:generator, ex)
525 -
                for j in i
526 -
                    push!(ex.args, convert_iter_assign(j))
527 -
                end
528 -
                ex = Expr(:flatten, ex)
529 -
            end
530 -
        end
531 -
        return ex
532 -
    elseif typof(x) === Generator
533 -
        ret = Expr(:generator, Expr(x.args[1]))
534 -
        for i = 3:length(x.args)
535 -
            a = x.args[i]
536 -
            if !(ispunctuation(a))
537 -
                push!(ret.args, convert_iter_assign(a))
538 -
            end
539 -
        end
540 -
        return ret
541 -
    elseif typof(x) === Filter
542 -
        ret = Expr(:filter)
543 -
        push!(ret.args, Expr(last(x.args)))
544 -
        for i in 1:length(x.args) - 1
545 -
            a = x.args[i]
546 -
            if !(is_if(a) || ispunctuation(a))
547 -
                push!(ret.args, convert_iter_assign(a))
548 -
            end
549 -
        end
550 -
        return ret
551 -
    elseif typof(x) === TypedComprehension
552 -
        ret = Expr(:typed_comprehension)
553 -
        for a in x.args
554 -
            if !(ispunctuation(a))
555 -
                push!(ret.args, Expr(a))
556 -
            end
557 -
        end
558 -
        return ret
559 -
    elseif typof(x) === Import
560 -
        return expr_import(x, :import)
561 -
    elseif typof(x) === Using
562 -
        return expr_import(x, :using)
563 -
    elseif typof(x) === Export
564 -
        ret = Expr(:export)
565 -
        for i = 2:length(x.args)
566 -
            a = x.args[i]
567 -
            if !(ispunctuation(a))
568 -
                push!(ret.args, Expr(a))
569 -
            end
570 -
        end
571 -
        return ret
572 -
    elseif typof(x) === FileH
573 -
        ret = Expr(:file)
574 -
        for a in x.args
575 -
            push!(ret.args, Expr(a))
576 -
        end
577 -
        return ret
578 -
    elseif typof(x) === StringH
579 -
        ret = Expr(:string)
580 -
        for (i, a) in enumerate(x.args)
581 -
            if isunarycall(a)
582 -
                a = a.args[2]
583 -
            elseif isliteral(a) && kindof(a) === Tokens.STRING && span(a) == 0 || ((i == 1 || i == length(x.args)) && span(a) == 1) || (valof(a) === nothing || isempty(valof(a)))
584 -
                continue
585 -
            else isliteral(a) && kindof(a) === Tokens.TRIPLE_STRING && span(a) == 0 || ((i == 1 || i == length(x.args)) && span(a) == 3) || (valof(a) === nothing || isempty(valof(a)))
586 -
            end
587 -
            push!(ret.args, Expr(a))
588 -
        end
589 -
        return ret
158 +
            Symbol("@cmd")
159 +
        end
160 +
    elseif x.head === :macrocall && is_getfield_w_quotenode(x.args[1]) && !ismacroname(x.args[1].args[2].args[1])
161 +
        # Shift '@' to the right
162 +
        new_name = Expr(:., remove_at(x.args[1].args[1]), QuoteNode(Symbol("@", valof(x.args[1].args[2].args[1]))))
163 +
        Expr(:macrocall, new_name, Expr.(x.args[2:end])...)
164 +
    elseif x.head === :macrocall && isidentifier(x.args[1]) && valof(x.args[1]) == "@."
165 +
        Expr(:macrocall, Symbol("@__dot__"), Expr.(x.args[2:end])...)
166 +
    elseif x.head === :string && length(x.args) > 0 && (x.args[1].head === :STRING || x.args[1].head === :TRIPLESTRING) && isempty(valof(x.args[1]))
167 +
        # Special conversion needed - the initial text section is treated as empty for the represented string following lowest-common-prefix adjustments, but exists in the source.
168 +
        Expr(:string, Expr.(x.args[2:end])...)
169 +
    elseif x.args === nothing
170 +
        Expr(Symbol(lowercase(String(x.head))))
590 171
    else
591 -
        ret = Expr(:call)
592 -
        for a in x.args
593 -
            if !(ispunctuation(a))
594 -
                push!(ret.args, Expr(a))
595 -
            end
596 -
        end
597 -
        return ret
172 +
        Expr(Symbol(lowercase(String(x.head))), Expr.(x.args)...)
598 173
    end
599 174
end
600 175
601 -
# Op. expressions
602 -
603 -
function _unary_expr(x)
604 -
    if isoperator(x.args[1]) && issyntaxunarycall(x.args[1])
605 -
        Expr(Expr(x.args[1]), Expr(x.args[2]))
606 -
    elseif isoperator(x.args[2]) && issyntaxunarycall(x.args[2])
607 -
        Expr(Expr(x.args[2]), Expr(x.args[1]))
176 +
function remove_at(x)
177 +
    if isidentifier(x) && valof(x) !== nothing && first(valof(x)) == '@'
178 +
        return Symbol(valof(x)[2:end])
179 +
    elseif is_getfield_w_quotenode(x)
180 +
        Expr(:., remove_at(x.args[1]), QuoteNode(remove_at(x.args[2].args[1])))
608 181
    else
609 -
        Expr(:call, Expr(x.args[1]), Expr(x.args[2]))
610 -
    end
611 -
end
612 -
function _binary_expr(x)
613 -
    if issyntaxcall(x.args[2]) && !(kindof(x.args[2]) in (Tokens.COLON,))
614 -
        if kindof(x.args[2]) === Tokens.DOT
615 -
            arg1, arg2 = Expr(x.args[1]), Expr(x.args[3])
616 -
            if arg2 isa Expr && arg2.head === :macrocall && endswith(string(arg2.args[1]), "_cmd")
617 -
                return Expr(:macrocall, Expr(:., arg1, QuoteNode(arg2.args[1])), nothing, arg2.args[3])
618 -
            elseif arg2 isa Expr && arg2.head === :braces
619 -
                return Expr(:., arg1, Expr(:quote, arg2))
620 -
            end
621 -
        end
622 -
        Expr(Expr(x.args[2]), Expr(x.args[1]), Expr(x.args[3]))
623 -
    else
624 -
        Expr(:call, Expr(x.args[2]), Expr(x.args[1]), Expr(x.args[3]))
182 +
        Expr(x)
625 183
    end
626 184
end
627 185
628 -
function _where_expr(x)
629 -
    ret = Expr(:where, Expr(x.args[1]))
630 -
    for i = 3:length(x.args)
631 -
        a = x.args[i]
632 -
        if typof(a) === Parameters
633 -
            insert!(ret.args, 2, Expr(a))
634 -
        elseif !(ispunctuation(a) || iskw(a))
635 -
            push!(ret.args, Expr(a))
636 -
        end
637 -
    end
638 -
    return ret
639 -
end
640 -
641 -
642 186
# cross compatability for line number insertion in macrocalls
643 187
if VERSION > v"1.1-"
644 188
    Expr_cmd(x) = Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), nothing, valof(x))
@@ -690,210 +234,4 @@
Loading
690 234
    x
691 235
end
692 236
693 -
function _if_expr(x)
694 -
    ret = Expr(:if)
695 -
    iselseif = false
696 -
    n = length(x.args)
697 -
    i = 0
698 -
    while i < n
699 -
        i += 1
700 -
        a = x.args[i]
701 -
        if iskw(a) && kindof(a) === Tokens.ELSEIF
702 -
            i += 1
703 -
            r1 = Expr(x.args[i].args[1])
704 -
            push!(ret.args, Expr(:elseif, r1.args...))
705 -
        elseif !(ispunctuation(a) || iskw(a))
706 -
            push!(ret.args, Expr(a))
707 -
        end
708 -
    end
709 -
    ret
710 -
end
711 -
712 -
function _let_expr(x)
713 -
    ret = Expr(:let)
714 -
    if length(x.args) == 3
715 -
        push!(ret.args, Expr(:block))
716 -
        push!(ret.args, Expr(x.args[2]))
717 -
        return ret
718 -
    elseif typof(x.args[2]) === Block
719 -
        arg = Expr(:block)
720 -
        for a in x.args[2].args
721 -
            if !(ispunctuation(a))
722 -
                push!(arg.args, fix_range(a))
723 -
            end
724 -
        end
725 -
        push!(ret.args, arg)
726 -
    else
727 -
        push!(ret.args, fix_range(x.args[2]))
728 -
    end
729 -
    push!(ret.args, Expr(x.args[3]))
730 -
    ret
731 -
end
732 -
733 -
function fix_range(a)
734 -
    if isbinarycall(a) && (is_in(a.args[2]) || is_elof(a.args[2]))
735 -
        Expr(:(=), Expr(a.args[1]), Expr(a.args[3]))
736 -
    else
737 -
        Expr(a)
738 -
    end
739 -
end
740 -
741 -
function get_inner_gen(x, iters=[], arg=[])
742 -
    if typof(x) == Flatten
743 -
        get_inner_gen(x.args[1], iters, arg)
744 -
    elseif typof(x) === Generator
745 -
        # push!(iters, get_iter(x))
746 -
        get_iters(x, iters)
747 -
        if typof(x.args[1]) === Generator || typof(x.args[1]) === Flatten
748 -
            get_inner_gen(x.args[1], iters, arg)
749 -
        else
750 -
            push!(arg, x.args[1])
751 -
        end
752 -
    end
753 -
    return iters, arg
754 -
end
755 -
756 -
function get_iter(x)
757 -
    if typof(x) === Generator
758 -
        return x.args[3]
759 -
    end
760 -
end
761 -
762 -
function get_iters(x, iters)
763 -
    iters1 = []
764 -
    if typof(x) === Generator
765 -
        # return x.args[3]
766 -
        for i = 3:length(x.args)
767 -
            if typof(x.args[i]) !== PUNCTUATION
768 -
                push!(iters1, x.args[i])
769 -
            end
770 -
        end
771 -
    end
772 -
    push!(iters, iters1)
773 -
end
774 -
775 -
function convert_iter_assign(a)
776 -
    if isbinarycall(a) && (is_in(a.args[2]) || is_elof(a.args[2]))
777 -
        return Expr(:(=), Expr(a.args[1]), Expr(a.args[3]))
778 -
    else
779 -
        return Expr(a)
780 -
    end
781 -
end
782 -
783 -
function _get_import_block(x, i, ret)
784 -
    while is_dot(x.args[i + 1])
785 -
        i += 1
786 -
        push!(ret.args, :.)
787 -
    end
788 -
    while i < length(x.args) && !(is_comma(x.args[i + 1]))
789 -
        i += 1
790 -
        a = x.args[i]
791 -
        if !(ispunctuation(a)) && !(is_dot(a) || is_colon(a))
792 -
            push!(ret.args, Expr(a))
793 -
        end
794 -
    end
795 -
796 -
    return i
797 -
end
798 -
799 -
function expr_import(x, kw)
800 -
    col = findall(a -> isoperator(a) && precedence(a) == ColonOp, x.args)
801 -
    comma = findall(is_comma, x.args)
802 -
803 -
    header = []
804 -
    args = [Expr(:.)]
805 -
    i = 1 # skip keyword
806 -
    while i < length(x.args)
807 -
        i += 1
808 -
        a = x.args[i]
809 -
        if is_colon(a)
810 -
            push!(header, popfirst!(args))
811 -
            push!(args, Expr(:.))
812 -
        elseif is_comma(a)
813 -
            push!(args, Expr(:.))
814 -
        elseif !(ispunctuation(a))
815 -
            push!(last(args).args, Expr(a))
816 -
        end
817 -
    end
818 -
    if isempty(header)
819 -
        return Expr(kw, args...)
820 -
    else
821 -
        return Expr(kw, Expr(:(:), header..., args...))
822 -
    end
823 -
end
824 -
825 -
826 -
827 -
const UNICODE_OPS_REVERSE = Dict{Tokenize.Tokens.Kind,Symbol}()
828 -
for (k, v) in Tokenize.Tokens.UNICODE_OPS
829 -
    UNICODE_OPS_REVERSE[v] = Symbol(k)
830 -
end
831 237
832 -
UNICODE_OPS_REVERSE[Tokens.EQ] = :(=)
833 -
UNICODE_OPS_REVERSE[Tokens.PLUS_EQ] = :(+=)
834 -
UNICODE_OPS_REVERSE[Tokens.MINUS_EQ] = :(-=)
835 -
UNICODE_OPS_REVERSE[Tokens.STAR_EQ] = :(*=)
836 -
UNICODE_OPS_REVERSE[Tokens.FWD_SLASH_EQ] = :(/=)
837 -
UNICODE_OPS_REVERSE[Tokens.FWDFWD_SLASH_EQ] = :(//=)
838 -
UNICODE_OPS_REVERSE[Tokens.OR_EQ] = :(|=)
839 -
UNICODE_OPS_REVERSE[Tokens.CIRCUMFLEX_EQ] = :(^=)
840 -
UNICODE_OPS_REVERSE[Tokens.DIVISION_EQ] = :(÷=)
841 -
UNICODE_OPS_REVERSE[Tokens.REM_EQ] = :(%=)
842 -
UNICODE_OPS_REVERSE[Tokens.LBITSHIFT_EQ] = :(<<=)
843 -
UNICODE_OPS_REVERSE[Tokens.RBITSHIFT_EQ] = :(>>=)
844 -
UNICODE_OPS_REVERSE[Tokens.LBITSHIFT] = :(<<)
845 -
UNICODE_OPS_REVERSE[Tokens.RBITSHIFT] = :(>>)
846 -
UNICODE_OPS_REVERSE[Tokens.UNSIGNED_BITSHIFT] = :(>>>)
847 -
UNICODE_OPS_REVERSE[Tokens.UNSIGNED_BITSHIFT_EQ] = :(>>>=)
848 -
UNICODE_OPS_REVERSE[Tokens.BACKSLASH_EQ] = :(\=)
849 -
UNICODE_OPS_REVERSE[Tokens.AND_EQ] = :(&=)
850 -
UNICODE_OPS_REVERSE[Tokens.COLON_EQ] = :(:=)
851 -
UNICODE_OPS_REVERSE[Tokens.PAIR_ARROW] = :(=>)
852 -
UNICODE_OPS_REVERSE[Tokens.APPROX] = :(~)
853 -
UNICODE_OPS_REVERSE[Tokens.EX_OR_EQ] = :($=)
854 -
UNICODE_OPS_REVERSE[Tokens.XOR_EQ] = :(⊻=)
855 -
UNICODE_OPS_REVERSE[Tokens.RIGHT_ARROW] = :(-->)
856 -
UNICODE_OPS_REVERSE[Tokens.LAZY_OR] = :(||)
857 -
UNICODE_OPS_REVERSE[Tokens.LAZY_AND] = :(&&)
858 -
UNICODE_OPS_REVERSE[Tokens.ISSUBTYPE] = :(<:)
859 -
UNICODE_OPS_REVERSE[Tokens.ISSUPERTYPE] = :(>:)
860 -
UNICODE_OPS_REVERSE[Tokens.GREATER] = :(>)
861 -
UNICODE_OPS_REVERSE[Tokens.LESS] = :(<)
862 -
UNICODE_OPS_REVERSE[Tokens.GREATER_EQ] = :(>=)
863 -
UNICODE_OPS_REVERSE[Tokens.GREATER_THAN_OR_EQUAL_TO] = :(≥)
864 -
UNICODE_OPS_REVERSE[Tokens.LESS_EQ] = :(<=)
865 -
UNICODE_OPS_REVERSE[Tokens.LESS_THAN_OR_EQUAL_TO] = :(≤)
866 -
UNICODE_OPS_REVERSE[Tokens.EQEQ] = :(==)
867 -
UNICODE_OPS_REVERSE[Tokens.EQEQEQ] = :(===)
868 -
UNICODE_OPS_REVERSE[Tokens.IDENTICAL_TO] = :(≡)
869 -
UNICODE_OPS_REVERSE[Tokens.NOT_EQ] = :(!=)
870 -
UNICODE_OPS_REVERSE[Tokens.NOT_EQUAL_TO] = :(≠)
871 -
UNICODE_OPS_REVERSE[Tokens.NOT_IS] = :(!==)
872 -
UNICODE_OPS_REVERSE[Tokens.NOT_IDENTICAL_TO] = :(≢)
873 -
UNICODE_OPS_REVERSE[Tokens.IN] = :(in)
874 -
UNICODE_OPS_REVERSE[Tokens.ISA] = :(isa)
875 -
UNICODE_OPS_REVERSE[Tokens.LPIPE] = :(<|)
876 -
UNICODE_OPS_REVERSE[Tokens.RPIPE] = :(|>)
877 -
UNICODE_OPS_REVERSE[Tokens.COLON] = :(:)
878 -
UNICODE_OPS_REVERSE[Tokens.DDOT] = :(..)
879 -
UNICODE_OPS_REVERSE[Tokens.EX_OR] = :($)
880 -
UNICODE_OPS_REVERSE[Tokens.PLUS] = :(+)
881 -
UNICODE_OPS_REVERSE[Tokens.MINUS] = :(-)
882 -
UNICODE_OPS_REVERSE[Tokens.PLUSPLUS] = :(++)
883 -
UNICODE_OPS_REVERSE[Tokens.OR] = :(|)
884 -
UNICODE_OPS_REVERSE[Tokens.STAR] = :(*)
885 -
UNICODE_OPS_REVERSE[Tokens.FWD_SLASH] = :(/)
886 -
UNICODE_OPS_REVERSE[Tokens.REM] = :(%)
887 -
UNICODE_OPS_REVERSE[Tokens.BACKSLASH] = :(\)
888 -
UNICODE_OPS_REVERSE[Tokens.AND] = :(&)
889 -
UNICODE_OPS_REVERSE[Tokens.FWDFWD_SLASH] = :(//)
890 -
UNICODE_OPS_REVERSE[Tokens.CIRCUMFLEX_ACCENT] = :(^)
891 -
UNICODE_OPS_REVERSE[Tokens.DECLARATION] = :(::)
892 -
UNICODE_OPS_REVERSE[Tokens.CONDITIONAL] = :?
893 -
UNICODE_OPS_REVERSE[Tokens.DOT] = :(.)
894 -
UNICODE_OPS_REVERSE[Tokens.NOT] = :(!)
895 -
UNICODE_OPS_REVERSE[Tokens.PRIME] = Symbol(''')
896 -
UNICODE_OPS_REVERSE[Tokens.DDDOT] = :(...)
897 -
UNICODE_OPS_REVERSE[Tokens.TRANSPOSE] = Symbol(".'")
898 -
UNICODE_OPS_REVERSE[Tokens.ANON_FUNC] = :(->)
899 -
UNICODE_OPS_REVERSE[Tokens.WHERE] = :where

@@ -1,44 +1,44 @@
Loading
1 1
function accept_rparen(ps)
2 2
    if kindof(ps.nt) === Tokens.RPAREN
3 -
        return mPUNCTUATION(next(ps))
3 +
        return EXPR(next(ps))
4 4
    else
5 -
        return mErrorToken(ps, mPUNCTUATION(Tokens.RPAREN, 0, 0), UnexpectedToken)
5 +
        return mErrorToken(ps, EXPR(:RPAREN, 0, 0), UnexpectedToken)
6 6
    end
7 7
end
8 8
accept_rparen(ps::ParseState, args) = push!(args, accept_rparen(ps))
9 9
10 10
function accept_rsquare(ps)
11 11
    if kindof(ps.nt) === Tokens.RSQUARE
12 -
        return mPUNCTUATION(next(ps))
12 +
        return EXPR(next(ps))
13 13
    else
14 -
        return mErrorToken(ps, mPUNCTUATION(Tokens.RSQUARE, 0, 0), UnexpectedToken)
14 +
        return mErrorToken(ps, EXPR(:RSQUARE, 0, 0), UnexpectedToken)
15 15
    end
16 16
end
17 17
accept_rsquare(ps::ParseState, args) = push!(args, accept_rsquare(ps))
18 18
19 19
function accept_rbrace(ps)
20 20
    if kindof(ps.nt) === Tokens.RBRACE
21 -
        return mPUNCTUATION(next(ps))
21 +
        return EXPR(next(ps))
22 22
    else
23 -
        return mErrorToken(ps, mPUNCTUATION(Tokens.RBRACE, 0, 0), UnexpectedToken)
23 +
        return mErrorToken(ps, EXPR(:RBRACE, 0, 0), UnexpectedToken)
24 24
    end
25 25
end
26 26
accept_rbrace(ps::ParseState, args) = push!(args, accept_rbrace(ps))
27 27
28 28
function accept_end(ps::ParseState)
29 29
    if kindof(ps.nt) === Tokens.END
30 -
        return mKEYWORD(next(ps))
30 +
        return EXPR(next(ps))
31 31
    else
32 -
        return mErrorToken(ps, mKEYWORD(Tokens.END, 0, 0), UnexpectedToken)
32 +
        return mErrorToken(ps, EXPR(:END, 0, 0), UnexpectedToken)
33 33
    end
34 34
end
35 35
accept_end(ps::ParseState, args) = push!(args, accept_end(ps))
36 36
37 37
function accept_comma(ps)
38 38
    if iscomma(ps.nt)
39 -
        return mPUNCTUATION(next(ps))
39 +
        return EXPR(next(ps))
40 40
    else
41 -
        return mErrorToken(mPUNCTUATION(Tokens.COMMA, 0, 0), UnexpectedToken)
41 +
        return EXPR(:RPAREN, 0, 0)
42 42
    end
43 43
end
44 44
accept_comma(ps::ParseState, args) = push!(args, accept_comma(ps))
@@ -52,9 +52,9 @@
Loading
52 52
end
53 53
54 54
function requires_no_ws(x, ps)
55 -
    if !(ps.nt.kind === Tokens.RPAREN ||
56 -
        ps.nt.kind === Tokens.RBRACE ||
57 -
        ps.nt.kind === Tokens.RSQUARE) && x.span != x.fullspan
55 +
    if !(kindof(ps.nt) === Tokens.RPAREN ||
56 +
        kindof(ps.nt) === Tokens.RBRACE ||
57 +
        kindof(ps.nt) === Tokens.RSQUARE) && x.span != x.fullspan
58 58
        return mErrorToken(ps, x, UnexpectedWhiteSpace)
59 59
    else
60 60
        return x

@@ -1,95 +1,165 @@
Loading
1 +
# terminals
2 +
isidentifier(x::EXPR) = headof(x) === :IDENTIFIER || headof(x) === :NONSTDIDENTIFIER
3 +
isoperator(x::EXPR) = headof(x) === :OPERATOR
4 +
isnonstdid(x::EXPR) = headof(x) === :NONSTDIDENTIFIER
5 +
iskeyword(x::EXPR) = headof(x) in (:ABSTRACT, :BAREMODULE, :BEGIN, :BREAK, :CATCH, :CONST, :CONTINUE, :DO, :ELSE, :ELSEIF, :END, :EXPORT, :FINALLY, :FOR, :FUNCTION, :GLOBAL, :IF, :IMPORT, :importall, :LET, :LOCAL, :MACRO, :MODULE, :MUTABLE, :NEW, :OUTER, :PRIMITIVE, :QUOTE, :RETURN, :STRUCT, :TRY, :TYPE, :USING, :WHILE)
6 +
isliteral(x::EXPR) = isstringliteral(x) || iscmd(x) || ischar(x) || headof(x) in (:INTEGER, :BININT, :HEXINT, :OCTINT, :FLOAT,  :NOTHING, :TRUE, :FALSE)
7 +
ispunctuation(x::EXPR) = is_comma(x) || is_lparen(x) || is_rparen(x) || is_lsquare(x) || is_rsquare(x) || is_lbrace(x) || is_rbrace(x) || headof(x) === :ATSIGN  || headof(x) === :DOT
8 +
isstringliteral(x) = headof(x) === :STRING || headof(x) === :TRIPLESTRING
9 +
isstring(x) = headof(x) === :string || isstringliteral(x)
10 +
iscmd(x) = headof(x) === :CMD || headof(x) === :TRIPLECMD
11 +
ischar(x) = headof(x) === :CHAR
12 +
isinteger(x) = headof(x) === :INTEGER
13 +
isfloat(x) = headof(x) === :FLOAT
14 +
isnumber(x) = isinteger(x) || isfloat(x)
15 +
is_nothing(x) = headof(x) === :NOTHING
16 +
is_either_id_op_interp(x::EXPR) = isidentifier(x) || isoperator(x) || isinterpolant(x)
17 +
is_id_or_macroname(x::EXPR) = isidentifier(x) || ismacroname(x)
18 +
19 +
# expressions
20 +
iscall(x::EXPR) = headof(x) === :call
21 +
isunarycall(x::EXPR) = (headof(x) === :call && length(x) == 2 && (isoperator(x.args[1]) || isoperator(x.args[2])))
22 +
isunarysyntax(x::EXPR) =  (isoperator(x.head) && length(x.args) == 1)
23 +
isbinarycall(x::EXPR) = headof(x) === :call && length(x) == 3 && isoperator(x.args[1])
24 +
isbinarycall(x::EXPR, op) = headof(x) === :call && length(x) == 3 && isoperator(x.args[1]) && valof(x.args[1]) == op
25 +
isbinarysyntax(x::EXPR) =  (isoperator(x.head) && length(x.args) == 2)
26 +
ischainedcall(x::EXPR) = headof(x) === :call && isoperator(x.args[1]) && (valof(x.args[1]) == "+" || valof(x.args[1]) == "*") && hastrivia(x) && isoperator(first(x.trivia))
27 +
iswhere(x::EXPR) = headof(x) === :where
28 +
istuple(x::EXPR) = headof(x) === :tuple
29 +
ismacrocall(x::EXPR) = headof(x) === :macrocall
30 +
ismacroname(x::EXPR) = (isidentifier(x) && valof(x) !== nothing && !isempty(valof(x)) && first(valof(x)) == '@') || (is_getfield_w_quotenode(x) && (ismacroname(unquotenode(rhs_getfield(x))) || ismacroname(x.args[1])))
31 +
32 +
iskwarg(x::EXPR) = headof(x) === :kw
33 +
isparameters(x::EXPR) = headof(x) === :parameters
34 +
iscurly(x::EXPR) = headof(x) === :curly
35 +
isbracketed(x::EXPR) = headof(x) === :brackets
36 +
37 +
38 +
isassignment(x::EXPR) = isbinarysyntax(x) && valof(x.head) == "="
39 +
isdeclaration(x::EXPR) = isbinarysyntax(x) && valof(x.head) == "::"
40 +
isinterpolant(x::EXPR) = isunarysyntax(x) && valof(x.head) == "\$"
41 +
issplat(x::EXPR) = isunarysyntax(x) && valof(x.head) == "..."
42 +
43 +
44 +
isbeginorblock(x::EXPR) = headof(x) === :begin || headof(unwrapbracket(x)) == :block
45 +
46 +
47 +
"""
48 +
    is_getfield(x::EXPR)
49 +
50 +
Is this an expression of the form `a.b`.
51 +
"""
52 +
is_getfield(x::EXPR) = x.head isa EXPR && isoperator(x.head) && valof(x.head) == "." && length(x.args) == 2
53 +
is_getfield_w_quotenode(x) = is_getfield(x) && headof(x.args[2]) === :quotenode && length(x.args[2].args) > 0
54 +
rhs_getfield(x) = x.args[2]
55 +
unquotenode(x) = x.args[1]
56 +
57 +
function get_rhs_of_getfield(ret::EXPR)
58 +
    if headof(ret.args[2]) === :quotenode || headof(ret.args[2]) === :quote
59 +
        ret.args[2].args[1]
60 +
    else
61 +
        ret.args[2]
62 +
    end
63 +
end
64 +
65 +
66 +
"""
67 +
    is_wrapped_assignment(x::EXPR)
68 +
    
69 +
Is `x` an assignment expression, ignoring any surrounding parentheses.
70 +
"""
71 +
is_wrapped_assignment(x::EXPR) = isassignment(x) || (isbracketed(x) && is_wrapped_assignment(x.args[1]))
72 +
73 +
1 74
function is_func_call(x::EXPR)
2 -
    if typof(x) === Call
75 +
    if isoperator(x.head)
76 +
        if length(x.args) == 2
77 +
            return is_decl(x.head) && is_func_call(x.args[1])
78 +
        elseif length(x.args) == 1
79 +
            return !(is_exor(x.head) || is_decl(x.head))
80 +
        end
81 +
    elseif x.head === :call
3 82
        return true
4 -
    elseif iswherecall(x)
83 +
    elseif x.head === :where || isbracketed(x)
5 84
        return is_func_call(x.args[1])
6 -
    elseif isbracketed(x)
7 -
        return is_func_call(x.args[2])
8 -
    elseif isunarycall(x)
9 -
        return !(isoperator(x.args[1]) && (kindof(x.args[1]) === Tokens.EX_OR || kindof(x.args[1]) === Tokens.DECLARATION))
10 -
    elseif isbinarycall(x)
11 -
        if issyntaxcall(x.args[2])
12 -
            if is_decl(x.args[2])
13 -
                return is_func_call(x.args[1])
14 -
            else
15 -
                return false
16 -
            end
17 -
        else
18 -
            true
19 -
        end
20 85
    else
21 86
        return false
22 87
    end
23 88
end
24 89
25 -
is_assignment(x) = isbinarycall(x) && kindof(x.args[2]) === Tokens.EQ
90 +
91 +
fcall_name(x::EXPR) = iscall(x) && length(x.args) > 0 && valof(x.args[1])
92 +
hasparent(x::EXPR) = parentof(x) isa EXPR
26 93
27 94
# OPERATOR
28 -
is_exor(x) = isoperator(x) && kindof(x) === Tokens.EX_OR && x.dot == false
29 -
is_decl(x) = isoperator(x) && kindof(x) === Tokens.DECLARATION
30 -
is_issubt(x) = isoperator(x) && kindof(x) === Tokens.ISSUBTYPE
31 -
is_issupt(x) = isoperator(x) && kindof(x) === Tokens.ISSUPERTYPE
32 -
is_and(x) = isoperator(x) && kindof(x) === Tokens.AND && x.dot == false
33 -
is_not(x) = isoperator(x) && kindof(x) === Tokens.NOT && x.dot == false
34 -
is_plus(x) = isoperator(x) && kindof(x) === Tokens.PLUS && x.dot == false
35 -
is_minus(x) = isoperator(x) && kindof(x) === Tokens.MINUS && x.dot == false
36 -
is_star(x) = isoperator(x) && kindof(x) === Tokens.STAR && x.dot == false
37 -
is_eq(x) = isoperator(x) && kindof(x) === Tokens.EQ && x.dot == false
38 -
is_dot(x) = isoperator(x) && kindof(x) === Tokens.DOT
39 -
is_ddot(x) = isoperator(x) && kindof(x) === Tokens.DDOT
40 -
is_dddot(x) = isoperator(x) && kindof(x) === Tokens.DDDOT
41 -
is_pairarrow(x) = isoperator(x) && kindof(x) === Tokens.PAIR_ARROW && x.dot == false
42 -
is_in(x) = isoperator(x) && kindof(x) === Tokens.IN && x.dot == false
43 -
is_elof(x) = isoperator(x) && kindof(x) === Tokens.ELEMENT_OF && x.dot == false
44 -
is_colon(x) = isoperator(x) && kindof(x) === Tokens.COLON
45 -
is_prime(x) = isoperator(x) && kindof(x) === Tokens.PRIME
46 -
is_cond(x) = isoperator(x) && kindof(x) === Tokens.CONDITIONAL
47 -
is_where(x) = isoperator(x) && kindof(x) === Tokens.WHERE
48 -
is_anon_func(x) = isoperator(x) && kindof(x) === Tokens.ANON_FUNC
49 -
50 -
is_comma(x) = ispunctuation(x) && kindof(x) === Tokens.COMMA
51 -
is_lparen(x) = ispunctuation(x) && kindof(x) === Tokens.LPAREN
52 -
is_rparen(x) = ispunctuation(x) && kindof(x) === Tokens.RPAREN
53 -
is_lbrace(x) = ispunctuation(x) && kindof(x) === Tokens.LBRACE
54 -
is_rbrace(x) = ispunctuation(x) && kindof(x) === Tokens.RBRACE
55 -
is_lsquare(x) = ispunctuation(x) && kindof(x) === Tokens.LSQUARE
56 -
is_rsquare(x) = ispunctuation(x) && kindof(x) === Tokens.RSQUARE
95 +
is_approx(x::EXPR) = isoperator(x) && valof(x) == "~"
96 +
is_exor(x) = isoperator(x) && valof(x) == "\$"
97 +
is_decl(x) = isoperator(x) && valof(x) == "::"
98 +
is_issubt(x) = isoperator(x) && valof(x) == "<:"
99 +
is_issupt(x) = isoperator(x) && valof(x) == ">:"
100 +
is_and(x) = isoperator(x) && valof(x) == "&"
101 +
is_not(x) = isoperator(x) && valof(x) == "!"
102 +
is_plus(x) = isoperator(x) && valof(x) == "+"
103 +
is_minus(x) = isoperator(x) && valof(x) == "-"
104 +
is_star(x) = isoperator(x) && valof(x) == "*"
105 +
is_eq(x) = isoperator(x) && valof(x) == "="
106 +
is_dot(x) = isoperator(x) && valof(x) == "."
107 +
is_ddot(x) = isoperator(x) && valof(x) == ".."
108 +
is_dddot(x) = isoperator(x) && valof(x) == "..."
109 +
is_pairarrow(x) = isoperator(x) && valof(x) == "=>"
110 +
is_in(x) = isoperator(x) && valof(x) == "in"
111 +
is_elof(x) = isoperator(x) && valof(x) == "∈"
112 +
is_colon(x) = isoperator(x) && valof(x) == ":"
113 +
is_prime(x) = isoperator(x) && valof(x) == "'"
114 +
is_cond(x) = isoperator(x) && valof(x) == "?"
115 +
is_where(x) = isoperator(x) && valof(x) == "where"
116 +
is_anon_func(x) = isoperator(x) && valof(x) == "->"
117 +
118 +
is_comma(x) = headof(x) === :COMMA
119 +
is_lparen(x) = headof(x) === :LPAREN
120 +
is_rparen(x) = headof(x) === :RPAREN
121 +
is_lbrace(x) = headof(x) === :LBRACE
122 +
is_rbrace(x) = headof(x) === :RBRACE
123 +
is_lsquare(x) = headof(x) === :LSQUARE
124 +
is_rsquare(x) = headof(x) === :RSQUARE
57 125
58 126
# KEYWORD
59 -
is_if(x) = iskw(x) && kindof(x) === Tokens.IF
60 -
is_import(x) = iskw(x) && kindof(x) === Tokens.IMPORT
127 +
is_if(x) = iskeyword(x) && headof(x) === :IF
128 +
is_import(x) = iskeyword(x) && headof(x) === :IMPORT
61 129
62 130
63 131
# Literals
64 -
is_lit_string(x) = kindof(x) === Tokens.STRING || kindof(x) === Tokens.TRIPLE_STRING
65 -
66 -
issubtypedecl(x::EXPR) = isbinarycall(x) && is_issubt(x.args[2])
67 -
68 -
rem_subtype(x::EXPR) = issubtypedecl(x) ? x[1] : x
69 -
rem_decl(x::EXPR) = isdeclaration(x) ? x[1] : x
70 -
rem_curly(x::EXPR) = typof(x) === Curly ? x.args[1] : x
71 -
rem_call(x::EXPR) = typof(x) === Call ? x[1] : x
72 -
rem_where(x::EXPR) = iswherecall(x) ? x[1] : x
73 -
rem_wheres(x::EXPR) = iswherecall(x) ? rem_wheres(x[1]) : x
74 -
rem_where_subtype(x::EXPR) = (iswherecall(x) || issubtypedecl(x)) ? x[1] : x
75 -
rem_where_decl(x::EXPR) = (iswherecall(x) || isdeclaration(x)) ? x[1] : x
76 -
rem_invis(x::EXPR) = isbracketed(x) ? rem_invis(x[2]) : x
77 -
rem_dddot(x::EXPR) = is_splat(x) ? x[1] : x
132 +
133 +
issubtypedecl(x::EXPR) = isoperator(x.head) && valof(x.head) == "<:"
134 +
135 +
rem_subtype(x::EXPR) = issubtypedecl(x) ? x.args[1] : x
136 +
rem_decl(x::EXPR) = isdeclaration(x) ? x.args[1] : x
137 +
rem_curly(x::EXPR) = headof(x) === :curly ? x.args[1] : x
138 +
rem_call(x::EXPR) = headof(x) === :call ? x.args[1] : x
139 +
rem_where(x::EXPR) = iswhere(x) ? x.args[1] : x
140 +
rem_wheres(x::EXPR) = iswhere(x) ? rem_wheres(x.args[1]) : x
141 +
rem_where_subtype(x::EXPR) = (iswhere(x) || issubtypedecl(x)) ? x.args[1] : x
142 +
rem_wheres_subtypes(x::EXPR) = (iswhere(x) || issubtypedecl(x)) ? rem_wheres_subtypes(x.args[1]) : x
143 +
rem_where_decl(x::EXPR) = (iswhere(x) || isdeclaration(x)) ? x.args[1] : x
144 +
rem_wheres_decls(x::EXPR) = (iswhere(x) || isdeclaration(x)) ? rem_wheres_decls(x.args[1]) : x
145 +
rem_invis(x::EXPR) = isbracketed(x) ? rem_invis(x.args[1]) : x
146 +
rem_dddot(x::EXPR) = issplat(x) ? x.args[1] : x
78 147
const rem_splat = rem_dddot
79 -
rem_kw(x::EXPR) = typof(x) === Kw ? x[1] : x
148 +
rem_kw(x::EXPR) = headof(x) === :kw ? x.args[1] : x
149 +
unwrapbracket(x::EXPR) = isbracketed(x) ? unwrapbracket(x.args[1]) : x
80 150
81 -
is_some_call(x) = typof(x) === Call || isunarycall(x) || (isbinarycall(x) && !(kindof(x.args[2]) === Tokens.DOT || issyntaxcall(x.args[2])))
82 -
is_eventually_some_call(x) = is_some_call(x) || ((isdeclaration(x) || iswherecall(x)) && is_eventually_some_call(x[1]))
151 +
is_some_call(x) = headof(x) === :call || isunarycall(x)
152 +
is_eventually_some_call(x) = is_some_call(x) || ((isdeclaration(x) || iswhere(x)) && is_eventually_some_call(x.args[1]))
83 153
84 -
defines_function(x::EXPR) = typof(x) === FunctionDef || (is_assignment(x) && is_eventually_some_call(x[1]))
85 -
defines_macro(x) = typof(x) == Macro
154 +
defines_function(x::EXPR) = headof(x) === :function || (isassignment(x) && is_eventually_some_call(x.args[1]))
155 +
defines_macro(x) = headof(x) == :macro
86 156
defines_datatype(x) = defines_struct(x) || defines_abstract(x) || defines_primitive(x)
87 -
defines_struct(x) = typof(x) === Struct || defines_mutable(x)
88 -
defines_mutable(x) = typof(x) === Mutable
89 -
defines_abstract(x) = typof(x) === Abstract
90 -
defines_primitive(x) = typof(x) === Primitive
91 -
defines_module(x) = typof(x) === ModuleH || typof(x) === BareModule
92 -
defines_anon_function(x) = isbinarycall(x) && is_anon_func(x.args[2])
157 +
defines_struct(x) = headof(x) === :struct
158 +
defines_mutable(x) = defines_struct(x) && x.args[1].head == :TRUE
159 +
defines_abstract(x) = headof(x) === :abstract
160 +
defines_primitive(x) = headof(x) === :primitive
161 +
defines_module(x) = headof(x) === :module
162 +
defines_anon_function(x) = isoperator(x.head) && valof(x.head) == "->"
93 163
94 164
has_sig(x::EXPR) = defines_datatype(x) || defines_function(x) || defines_macro(x) || defines_anon_function(x)
95 165
@@ -100,46 +170,39 @@
Loading
100 170
Should only be called when has_sig(x) == true.
101 171
"""
102 172
function get_sig(x::EXPR)
103 -
    if isbinarycall(x)
173 +
    if headof(x) isa EXPR # headof(headof(x)) === :OPERATOR valof(headof(x)) == "="
104 174
        return x.args[1]
105 -
    elseif typof(x) === Struct || typof(x) === FunctionDef || typof(x) === Macro
175 +
    elseif headof(x) === :struct || headof(x) === :mutable 
106 176
        return x.args[2]
107 -
    elseif typof(x) === Mutable || typof(x) === Abstract || typof(x) === Primitive
108 -
        return x.args[3]
177 +
    elseif  headof(x) === :abstract || headof(x) === :primitive || headof(x) === :function || headof(x) === :macro
178 +
        return x.args[1]
109 179
    end
110 180
end
111 181
112 182
function get_name(x::EXPR)
113 -
    if typof(x) === Struct || typof(x) === Mutable || typof(x) === Abstract || typof(x) === Primitive
183 +
    if defines_datatype(x)
114 184
        sig = get_sig(x)
115 185
        sig = rem_subtype(sig)
116 186
        sig = rem_wheres(sig)
117 187
        sig = rem_subtype(sig)
118 188
        sig = rem_curly(sig)
119 -
    elseif typof(x) === ModuleH || typof(x) === BareModule
189 +
    elseif defines_module(x)
120 190
        sig = x.args[2]
121 -
    elseif typof(x) === FunctionDef || typof(x) === Macro
191 +
    elseif defines_function(x) || defines_macro(x)
122 192
        sig = get_sig(x)
123 193
        sig = rem_wheres(sig)
124 194
        sig = rem_decl(sig)
125 195
        sig = rem_call(sig)
126 196
        sig = rem_curly(sig)
127 197
        sig = rem_invis(sig)
128 -
        if isbinarycall(sig) && kindof(sig.args[2]) === Tokens.DOT
129 -
            if length(sig.args) > 2 && sig.args[3].args isa Vector{EXPR} && length(sig.args[3].args) > 0
130 -
                sig = sig.args[3].args[1]
131 -
            end
198 +
        # if isbinarysyntax(sig) && is_dot(sig.head)
199 +
        if is_getfield_w_quotenode(sig)
200 +
            sig = sig.args[2].args[1]
132 201
        end
133 202
        return sig
203 +
    elseif is_getfield_w_quotenode(x)
204 +
        return x.args[2].args[1]
134 205
    elseif isbinarycall(x)
135 -
        length(x.args) < 2 && return x
136 -
        if kindof(x.args[2]) === Tokens.DOT
137 -
            if length(x.args) > 2 && typof(x.args[3]) === Quotenode && x.args[3].args isa Vector{EXPR} && length(x.args[3].args) > 0
138 -
                return get_name(x.args[3].args[1])
139 -
            else
140 -
                return x
141 -
            end
142 -
        end
143 206
        sig = x.args[1]
144 207
        if isunarycall(sig)
145 208
            return get_name(sig.args[1])
@@ -150,6 +213,10 @@
Loading
150 213
        sig = rem_curly(sig)
151 214
        sig = rem_invis(sig)
152 215
        return get_name(sig)
216 +
    elseif isbinarysyntax(x) && valof(x.head) == "<:"
217 +
        return get_name(x.args[1])
218 +
    elseif isunarysyntax(x) && valof(x.head) == "..."
219 +
        return get_name(x.args[1])
153 220
    else
154 221
        sig = x
155 222
        if isunarycall(sig)

@@ -96,6 +96,7 @@
Loading
96 96
97 97
Base.position(ps::ParseState) = ps.nt.startbyte
98 98
99 +
current_line(ps::ParseState) = ps.nt.startpos[1]
99 100
"""
100 101
    lex_ws_comment(l::Lexer, c)
101 102
@@ -178,14 +179,14 @@
Loading
178 179
@inline val(token::AbstractToken, ps::ParseState) = String(ps.l.io.data[token.startbyte + 1:token.endbyte + 1])
179 180
both_symbol_and_op(t::AbstractToken) = kindof(t) === Tokens.WHERE || kindof(t) === Tokens.IN || kindof(t) === Tokens.ISA
180 181
isprefixableliteral(t::AbstractToken) = (kindof(t) === Tokens.STRING || kindof(t) === Tokens.TRIPLE_STRING || kindof(t) === Tokens.CMD || kindof(t) === Tokens.TRIPLE_CMD)
181 -
isassignment(t::AbstractToken) = Tokens.begin_assignments < kindof(t) < Tokens.end_assignments
182 +
isassignmentop(t::AbstractToken) = Tokens.begin_assignments < kindof(t) < Tokens.end_assignments
182 183
183 184
isidentifier(t::AbstractToken) = kindof(t) === Tokens.IDENTIFIER
184 185
isliteral(t::AbstractToken) = Tokens.begin_literal < kindof(t) < Tokens.end_literal
185 186
isbool(t::AbstractToken) =  Tokens.TRUE ≤ kindof(t) ≤ Tokens.FALSE
186 187
iscomma(t::AbstractToken) =  kindof(t) === Tokens.COMMA
187 188
iscolon(t::AbstractToken) =  kindof(t) === Tokens.COLON
188 -
iskw(t::AbstractToken) = Tokens.iskeyword(kindof(t))
189 -
isinstance(t::AbstractToken) = isidentifier(t) || isliteral(t) || isbool(t) || iskw(t)
189 +
iskeyword(t::AbstractToken) = Tokens.iskeyword(kindof(t))
190 +
isinstance(t::AbstractToken) = isidentifier(t) || isliteral(t) || isbool(t) || iskeyword(t)
190 191
ispunctuation(t::AbstractToken) = iscomma(t) || kindof(t) === Tokens.END || Tokens.LSQUARE ≤ kindof(t) ≤ Tokens.RPAREN || kindof(t) === Tokens.AT_SIGN
191 192

@@ -7,13 +7,12 @@
Loading
7 7
const ComparisonOp  = 6
8 8
const PipeOp        = 7
9 9
const ColonOp       = 8
10 +
const PlusOp        = 9
10 11
@static if Base.operator_precedence(:<<) == 12
11 -
    const PlusOp        = 9
12 12
    const BitShiftOp    = 10
13 13
    const TimesOp       = 11
14 14
    const RationalOp    = 12
15 15
else
16 -
    const PlusOp        = 9
17 16
    const TimesOp       = 10
18 17
    const RationalOp    = 11
19 18
    const BitShiftOp    = 12
@@ -26,77 +25,6 @@
Loading
26 25
const DddotOp       = 7
27 26
const AnonFuncOp    = 14
28 27
29 -
@enum(Head,IDENTIFIER,
30 -
NONSTDIDENTIFIER,
31 -
PUNCTUATION,
32 -
OPERATOR,
33 -
KEYWORD,
34 -
LITERAL,
35 -
NoHead,
36 -
Call,
37 -
UnaryOpCall,
38 -
BinaryOpCall,
39 -
WhereOpCall,
40 -
ConditionalOpCall,
41 -
ChainOpCall,
42 -
ColonOpCall,
43 -
Abstract,
44 -
Begin,
45 -
Block,
46 -
Braces,
47 -
BracesCat,
48 -
Const,
49 -
Comparison,
50 -
Curly,
51 -
Do,
52 -
Filter,
53 -
Flatten,
54 -
For,
55 -
FunctionDef,
56 -
Generator,
57 -
Global,
58 -
GlobalRefDoc,
59 -
If,
60 -
Kw,
61 -
Let,
62 -
Local,
63 -
Macro,
64 -
MacroCall,
65 -
MacroName,
66 -
Mutable,
67 -
Outer,
68 -
Parameters,
69 -
Primitive,
70 -
Quote,
71 -
Quotenode,
72 -
InvisBrackets,
73 -
StringH,
74 -
Struct,
75 -
Try,
76 -
TupleH,
77 -
FileH,
78 -
Return,
79 -
While,
80 -
x_Cmd,
81 -
x_Str,
82 -
ModuleH,
83 -
BareModule,
84 -
TopLevel,
85 -
Export,
86 -
Import,
87 -
Using,
88 -
Comprehension,
89 -
DictComprehension,
90 -
TypedComprehension,
91 -
Hcat,
92 -
TypedHcat,
93 -
Ref,
94 -
Row,
95 -
Vcat,
96 -
TypedVcat,
97 -
Vect,
98 -
ErrorToken)
99 -
100 28
@enum(ErrorKind,
101 29
    UnexpectedToken,
102 30
    CannotJuxtapose,
@@ -110,62 +38,57 @@
Loading
110 38
    InvalidIterator,
111 39
    StringInterpolationWithTrailingWhitespace,
112 40
    TooLongChar,
113 -
    Unknown)
114 -
115 -
const NoKind = Tokenize.Tokens.begin_keywords
41 +
    Unknown,
42 +
    SignatureOfFunctionDefIsNotACall)
116 43
117 44
mutable struct EXPR
118 -
    typ::Head
45 +
    head::Union{Symbol,EXPR}
119 46
    args::Union{Nothing,Vector{EXPR}}
47 +
    trivia::Union{Nothing,Vector{EXPR}}
120 48
    fullspan::Int
121 49
    span::Int
122 50
    val::Union{Nothing,String}
123 -
    kind::Tokenize.Tokens.Kind
124 -
    dot::Bool
125 51
    parent::Union{Nothing,EXPR}
126 52
    meta
127 53
end
128 54
129 -
function EXPR(T::Head, args::Vector{EXPR}, fullspan::Int, span::Int)
130 -
    ex = EXPR(T, args, fullspan, span, nothing, NoKind, false, nothing, nothing)
55 +
function EXPR(head::Union{Symbol,EXPR}, args::Vector{EXPR}, trivia::Union{Vector{EXPR},Nothing}, fullspan::Int, span::Int)
56 +
    ex = EXPR(head, args, trivia, fullspan, span, nothing, nothing, nothing)
57 +
    if head isa EXPR
58 +
        setparent!(head, ex)
59 +
    end
131 60
    for c in args
132 61
        setparent!(c, ex)
133 62
    end
63 +
    if trivia isa Vector{EXPR}
64 +
        for c in trivia
65 +
            setparent!(c, ex)
66 +
        end
67 +
    end
134 68
    ex
135 69
end
136 70
137 -
function EXPR(T::Head, args::Vector{EXPR})
138 -
    ret = EXPR(T, args, 0, 0)
71 +
function EXPR(head::Union{Symbol,EXPR}, args::Vector{EXPR}, trivia::Union{Vector{EXPR},Nothing} = EXPR[])
72 +
    ret = EXPR(head, args, trivia, 0, 0)
139 73
    update_span!(ret)
140 74
    ret
141 75
end
142 76
77 +
# These methods are for terminal/childless expressions.
78 +
@noinline EXPR(head::Union{Symbol,EXPR}, fullspan::Int, span::Int, val = nothing) = EXPR(head, nothing, nothing, fullspan, span, val, nothing, nothing)
79 +
@noinline EXPR(head::Union{Symbol,EXPR}, ps::ParseState) = EXPR(head, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps))
80 +
@noinline EXPR(ps::ParseState) = EXPR(tokenkindtoheadmap(kindof(ps.t)), ps)
143 81
144 -
145 -
146 -
@noinline mIDENTIFIER(ps::ParseState) = EXPR(IDENTIFIER, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, nothing)
147 -
148 -
mPUNCTUATION(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(PUNCTUATION, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
149 -
@noinline mPUNCTUATION(ps::ParseState) = EXPR(PUNCTUATION, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)
150 -
151 -
mOPERATOR(fullspan::Int, span::Int, kind::Tokens.Kind, dotop::Bool) = EXPR(OPERATOR, nothing, fullspan, span, nothing, kind, dotop, nothing, nothing)
152 -
@noinline mOPERATOR(ps::ParseState) = EXPR(OPERATOR, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, ps.t.suffix ? val(ps.t, ps) : nothing, kindof(ps.t), ps.t.dotop, nothing, nothing)
153 -
154 -
mKEYWORD(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(KEYWORD, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
155 -
@noinline mKEYWORD(ps::ParseState) = EXPR(KEYWORD, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)
156 -
157 -
mLITERAL(fullspan::Int, span::Int, val::String, kind::Tokens.Kind) = EXPR(LITERAL, nothing, fullspan, span, val, kind, false, nothing, nothing)
158 82
@noinline function mLITERAL(ps::ParseState)
159 -
160 83
    if kindof(ps.t) === Tokens.STRING || kindof(ps.t) === Tokens.TRIPLE_STRING ||
161 84
        kindof(ps.t) === Tokens.CMD || kindof(ps.t) === Tokens.TRIPLE_CMD
162 85
        return parse_string_or_cmd(ps)
163 86
    else
164 87
        v = val(ps.t, ps)
165 88
        if kindof(ps.t) === Tokens.CHAR && length(v) > 3 && !(v[2] == '\\' && valid_escaped_seq(v[2:prevind(v, length(v))]))
166 -
            return mErrorToken(ps, mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\''), kindof(ps.t)), TooLongChar)
89 +
            return mErrorToken(ps, EXPR(:CHAR, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\'')), TooLongChar)
167 90
        end
168 -
        return mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v, kindof(ps.t))
91 +
        return EXPR(literalmap(kindof(ps.t)), ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v)
169 92
    end
170 93
end
171 94
@@ -174,12 +97,28 @@
Loading
174 97
span(x::EXPR) = x.span
175 98
176 99
function update_span!(x::EXPR)
177 -
    (x.args isa Nothing || isempty(x.args)) && return
100 +
    (x.args isa Nothing || isempty(x.args)) && !hastrivia(x) && return
178 101
    x.fullspan = 0
179 102
    for i = 1:length(x.args)
180 103
        x.fullspan += x.args[i].fullspan
181 104
    end
182 -
    x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
105 +
    if hastrivia(x)
106 +
        for i = 1:length(x.trivia)
107 +
            x.fullspan += x.trivia[i].fullspan
108 +
        end
109 +
    end
110 +
    if x.head isa EXPR
111 +
        x.fullspan += x.head.fullspan
112 +
        # TODO: special case for trailing unary ops?
113 +
    end
114 +
    if x.head isa EXPR && isoperator(x.head) && (is_dddot(x.head) || is_prime(x.head))
115 +
        # trailing unary operator
116 +
        x.span  = x.fullspan - x.head.fullspan + x.head.span
117 +
    elseif lastchildistrivia(x)
118 +
        x.span = x.fullspan - last(x.trivia).fullspan + last(x.trivia).span
119 +
    elseif !isempty(x.args)
120 +
        x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
121 +
    end
183 122
    return
184 123
end
185 124
@@ -190,6 +129,13 @@
Loading
190 129
    push!(e.args, arg)
191 130
end
192 131
132 +
function pushtotrivia!(e::EXPR, arg::EXPR)
133 +
    e.span = e.fullspan + arg.span
134 +
    e.fullspan += arg.fullspan
135 +
    setparent!(arg, e)
136 +
    push!(e.trivia, arg)
137 +
end
138 +
193 139
function Base.pushfirst!(e::EXPR, arg::EXPR)
194 140
    e.fullspan += arg.fullspan
195 141
    setparent!(arg, e)
@@ -227,69 +173,37 @@
Loading
227 173
228 174
function INSTANCE(ps::ParseState)
229 175
    if isidentifier(ps.t)
230 -
        return mIDENTIFIER(ps)
176 +
        return EXPR(:IDENTIFIER, ps)
231 177
    elseif isliteral(ps.t)
232 178
        return mLITERAL(ps)
233 -
    elseif iskw(ps.t)
234 -
        return mKEYWORD(ps)
179 +
    elseif iskeyword(ps.t)
180 +
        return EXPR(ps)
235 181
    elseif isoperator(ps.t)
236 -
        return mOPERATOR(ps)
182 +
        return EXPR(:OPERATOR, ps)
237 183
    elseif ispunctuation(ps.t)
238 -
        return mPUNCTUATION(ps)
184 +
        return EXPR(ps)
239 185
    elseif kindof(ps.t) === Tokens.ERROR
240 186
        ps.errored = true
241 -
        return EXPR(ErrorToken, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, Unknown)
187 +
        return EXPR(:errortoken, nothing, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), nothing, Unknown)
242 188
    else
243 189
        return mErrorToken(ps, Unknown)
244 190
    end
245 191
end
246 192
247 -
function mUnaryOpCall(op::EXPR, arg::EXPR)
248 -
    fullspan = op.fullspan + arg.fullspan
249 -
    ex = EXPR(UnaryOpCall, EXPR[op, arg], fullspan, fullspan - arg.fullspan + arg.span)
250 -
    setparent!(op, ex)
251 -
    setparent!(op, ex)
252 -
    return ex
253 -
end
254 -
function mBinaryOpCall(arg1::EXPR, op::EXPR, arg2::EXPR)
255 -
    fullspan = arg1.fullspan + op.fullspan + arg2.fullspan
256 -
    ex = EXPR(BinaryOpCall, EXPR[arg1, op, arg2], fullspan, fullspan - arg2.fullspan + arg2.span)
257 -
    setparent!(arg1, ex)
258 -
    setparent!(op, ex)
259 -
    setparent!(arg2, ex)
260 -
    return ex
261 -
end
262 -
function mWhereOpCall(arg1::EXPR, op::EXPR, args::Vector{EXPR})
263 -
    ex = EXPR(WhereOpCall, EXPR[arg1; op; args], arg1.fullspan + op.fullspan, 0)
264 -
    setparent!(arg1, ex)
265 -
    setparent!(op, ex)
266 -
    for a in args
267 -
        ex.fullspan += a.fullspan
268 -
        setparent!(a, ex)
269 -
    end
270 -
    ex.span = ex.fullspan - last(args).fullspan + last(args).span
271 -
    return ex
272 -
end
273 -
274 193
function mErrorToken(ps::ParseState, k::ErrorKind)
275 194
    ps.errored = true
276 -
    return EXPR(ErrorToken, EXPR[], 0, 0, nothing, NoKind, false, nothing, k)
195 +
    return EXPR(:errortoken, EXPR[], nothing, 0, 0, nothing, nothing, k)
277 196
end
278 197
function mErrorToken(ps::ParseState, x::EXPR, k)
279 198
    ps.errored = true
280 -
    ret = EXPR(ErrorToken, EXPR[x], x.fullspan, x.span, nothing, NoKind, false, nothing, k)
281 -
    setparent!(ret[1], ret)
199 +
    ret = EXPR(:errortoken, EXPR[x], nothing, x.fullspan, x.span, nothing, nothing, k)
200 +
    setparent!(ret.args[1], ret)
282 201
    return ret
283 202
end
284 203
285 -
TRUE() = mLITERAL(0, 0, "", Tokens.TRUE)
286 -
FALSE() = mLITERAL(0, 0, "", Tokens.FALSE)
287 -
NOTHING() = mLITERAL(0, 0, "", Tokens.NOTHING)
288 -
GlobalRefDOC() = EXPR(GlobalRefDoc, EXPR[])
289 204
290 -
typof(x::EXPR) = x.typ
205 +
headof(x::EXPR) = x.head
291 206
valof(x::EXPR) = x.val
292 -
kindof(x::EXPR) = x.kind
293 207
kindof(t::Tokens.AbstractToken) = t.kind
294 208
parentof(x::EXPR) = x.parent
295 209
errorof(x::EXPR) = errorof(x.meta)
@@ -299,14 +213,164 @@
Loading
299 213
    c.parent = p
300 214
    return c
301 215
end
216 +
hastrivia(x::EXPR) = x.trivia !== nothing && length(x.trivia) > 0
217 +
218 +
function lastchildistrivia(x::EXPR)
219 +
    return hastrivia(x) && (last(x.trivia).head in (:END, :RPAREN, :RSQUARE, :RBRACE) || (x.head in (:parameters, :tuple) && length(x.args) <= length(x.trivia)))
220 +
end
221 +
222 +
function Base.length(x::EXPR) 
223 +
    headof(x) === :NONSTDIDENTIFIER && return 0 
224 +
    n = x.args isa Nothing ? 0 : length(x.args)
225 +
    n += hastrivia(x) ? length(x.trivia) : 0
226 +
    x.head isa EXPR && !(x.head.span === 0) && (n += 1)
227 +
    return n
228 +
end
229 +
230 +
function literalmap(k::Tokens.Kind)
231 +
    if k === Tokens.INTEGER
232 +
        return :INTEGER
233 +
    elseif k === Tokens.BIN_INT
234 +
        return :BININT
235 +
    elseif k === Tokens.HEX_INT
236 +
        return :HEXINT
237 +
    elseif k === Tokens.OCT_INT
238 +
        return :OCTINT
239 +
    elseif k === Tokens.FLOAT
240 +
        return :FLOAT
241 +
    elseif k === Tokens.STRING
242 +
        return :STRING
243 +
    elseif k === Tokens.TRIPLE_STRING
244 +
        return :TRIPLESTRING
245 +
    elseif k === Tokens.CHAR
246 +
        return :CHAR
247 +
    elseif k === Tokens.CMD
248 +
        return :CMD
249 +
    elseif k === Tokens.TRIPLE_CMD
250 +
        return :TRIPLECMD
251 +
    elseif k === Tokens.TRUE
252 +
        return :TRUE
253 +
    elseif k === Tokens.FALSE
254 +
        return :FALSE
255 +
    end
256 +
end
302 257
303 258
304 -
Base.iterate(x::EXPR) = length(x) == 0 ? nothing : (x.args[1], 1)
305 -
Base.iterate(x::EXPR, s) = s < length(x) ? (x.args[s + 1], s + 1) : nothing
306 -
Base.length(x::EXPR) = x.args isa Nothing ? 0 : length(x.args)
307 -
Base.firstindex(x::EXPR) = 1
308 -
Base.lastindex(x::EXPR) = x.args === nothing ? 0 : lastindex(x.args)
309 -
Base.getindex(x::EXPR, i) = x.args[i]
310 -
Base.setindex!(x::EXPR, val, i) = Base.setindex!(x.args, val, i)
311 -
Base.first(x::EXPR) = x.args === nothing ? nothing : first(x.args)
312 -
Base.last(x::EXPR) = x.args === nothing ? nothing : last(x.args)
259 +
260 +
function tokenkindtoheadmap(k::Tokens.Kind)
261 +
    if k == Tokens.COMMA
262 +
        :COMMA
263 +
    elseif k == Tokens.LPAREN
264 +
        :LPAREN
265 +
    elseif k == Tokens.RPAREN
266 +
        :RPAREN
267 +
    elseif k == Tokens.LSQUARE
268 +
        :LSQUARE
269 +
    elseif k == Tokens.RSQUARE
270 +
        :RSQUARE
271 +
    elseif k == Tokens.LBRACE
272 +
        :LBRACE
273 +
    elseif k == Tokens.RBRACE
274 +
        :RBRACE
275 +
    elseif k == Tokens.AT_SIGN
276 +
        :ATSIGN
277 +
    elseif k == Tokens.DOT
278 +
        :DOT
279 +
    elseif k === Tokens.ABSTRACT
280 +
        return :ABSTRACT
281 +
    elseif k === Tokens.BAREMODULE
282 +
        return :BAREMODULE
283 +
    elseif k === Tokens.BEGIN
284 +
        return :BEGIN
285 +
    elseif k === Tokens.BREAK
286 +
        return :BREAK
287 +
    elseif k === Tokens.CATCH
288 +
        return :CATCH
289 +
    elseif k === Tokens.CONST
290 +
        return :CONST
291 +
    elseif k === Tokens.CONTINUE
292 +
        return :CONTINUE
293 +
    elseif k === Tokens.DO
294 +
        return :DO
295 +
    elseif k === Tokens.ELSE
296 +
        return :ELSE
297 +
    elseif k === Tokens.ELSEIF
298 +
        return :ELSEIF
299 +
    elseif k === Tokens.END
300 +
        return :END
301 +
    elseif k === Tokens.EXPORT
302 +
        return :EXPORT
303 +
    elseif k === Tokens.FINALLY
304 +
        return :FINALLY
305 +
    elseif k === Tokens.FOR
306 +
        return :FOR
307 +
    elseif k === Tokens.FUNCTION
308 +
        return :FUNCTION
309 +
    elseif k === Tokens.GLOBAL
310 +
        return :GLOBAL
311 +
    elseif k === Tokens.IF
312 +
        return :IF
313 +
    elseif k === Tokens.IMPORT
314 +
        return :IMPORT
315 +
    elseif k === Tokens.IMPORTALL
316 +
        return :importall
317 +
    elseif k === Tokens.LET
318 +
        return :LET
319 +
    elseif k === Tokens.LOCAL
320 +
        return :LOCAL
321 +
    elseif k === Tokens.MACRO
322 +
        return :MACRO
323 +
    elseif k === Tokens.MODULE
324 +
        return :MODULE
325 +
    elseif k === Tokens.MUTABLE
326 +
        return :MUTABLE
327 +
    elseif k === Tokens.NEW
328 +
        return :NEW
329 +
    elseif k === Tokens.OUTER
330 +
        return :OUTER
331 +
    elseif k === Tokens.PRIMITIVE
332 +
        return :PRIMITIVE
333 +
    elseif k === Tokens.QUOTE
334 +
        return :QUOTE
335 +
    elseif k === Tokens.RETURN
336 +
        return :RETURN
337 +
    elseif k === Tokens.STRUCT
338 +
        return :STRUCT
339 +
    elseif k === Tokens.TRY
340 +
        return :TRY
341 +
    elseif k === Tokens.TYPE
342 +
        return :TYPE
343 +
    elseif k === Tokens.USING
344 +
        return :USING
345 +
    elseif k === Tokens.WHILE
346 +
        return :WHILE
347 +
    elseif k === Tokens.INTEGER
348 +
        return :INTEGER
349 +
    elseif k === Tokens.BIN_INT
350 +
        return :BININT
351 +
    elseif k === Tokens.HEX_INT
352 +
        return :HEXINT
353 +
    elseif k === Tokens.OCT_INT
354 +
        return :OCTINT
355 +
    elseif k === Tokens.FLOAT
356 +
        return :FLOAT
357 +
    elseif k === Tokens.STRING
358 +
        return :STRING
359 +
    elseif k === Tokens.TRIPLE_STRING
360 +
        return :TRIPLESTRING
361 +
    elseif k === Tokens.CHAR
362 +
        return :CHAR
363 +
    elseif k === Tokens.CMD
364 +
        return :CMD
365 +
    elseif k === Tokens.TRIPLE_CMD
366 +
        return :TRIPLECMD
367 +
    elseif k === Tokens.TRUE
368 +
        return :TRUE
369 +
    elseif k === Tokens.FALSE
370 +
        return :FALSE
371 +
    elseif k === Tokens.ENDMARKER
372 +
        return :errortoken
373 +
    else
374 +
        error("")
375 +
    end
376 +
end

@@ -74,30 +74,39 @@
Loading
74 74
        end
75 75
    end
76 76
77 -
    # there are interpolations in the string
78 -
    if prefixed != false || iscmd
79 -
        t_str = val(ps.t, ps)
77 +
    isinterpolated = false
78 +
79 +
    t_str = val(ps.t, ps)
80 +
    if istrip && length(t_str) == 6
81 +
        if iscmd
82 +
            return wrapwithcmdmacro(EXPR(:TRIPLESTRING , sfullspan, sspan, ""))
83 +
        else
84 +
            return EXPR(:TRIPLESTRING, sfullspan, sspan, "")
85 +
        end
86 +
    elseif length(t_str) == 2
87 +
        if iscmd
88 +
            return wrapwithcmdmacro(EXPR(:STRING , sfullspan, sspan, ""))
89 +
        else
90 +
            return EXPR(:STRING, sfullspan, sspan, "")
91 +
        end
92 +
    elseif prefixed != false || iscmd
80 93
        _val = istrip ? t_str[4:prevind(t_str, sizeof(t_str), 3)] : t_str[2:prevind(t_str, sizeof(t_str))]
81 94
        if iscmd
82 95
            _val = replace(_val, "\\\\" => "\\")
83 96
            _val = replace(_val, "\\`" => "`")
84 97
        else
85 -
            if endswith(_val, "\\\\")
86 -
                _val = _val[1:end - 1]
87 -
            end
88 -
            _val = replace(_val, "\\\"" => "\"")
98 +
            _val = unescape_prefixed(_val)
89 99
        end
90 -
        expr = mLITERAL(sfullspan, sspan, _val, kindof(ps.t))
100 +
        expr = EXPR(istrip ? :TRIPLESTRING : :STRING, sfullspan, sspan, _val)
91 101
        if istrip
92 102
            adjust_lcp(expr)
93 -
            ret = EXPR(StringH, EXPR[expr], sfullspan, sspan)
103 +
            ret = EXPR(:string, EXPR[expr], nothing, sfullspan, sspan)
94 104
        else
95 -
            return expr
105 +
            return iscmd ? wrapwithcmdmacro(expr) : expr
96 106
        end
97 107
    else
98 -
        ret = EXPR(StringH, EXPR[], sfullspan, sspan)
99 -
        str2 = val(ps.t, ps)
100 -
        input = IOBuffer(str2)
108 +
        ret = EXPR(:string, EXPR[], EXPR[], sfullspan, sspan)
109 +
        input = IOBuffer(t_str)
101 110
        startbytes = istrip ? 3 : 1
102 111
        seek(input, startbytes)
103 112
        b = IOBuffer()
@@ -109,38 +118,48 @@
Loading
109 118
                write(b, c)
110 119
                write(b, read(input, Char))
111 120
            elseif c == '$'
121 +
                isinterpolated = true
112 122
                lspan = position(b)
113 123
                str = tostr(b)
114 -
                ex = mLITERAL(lspan + startbytes, lspan + startbytes, str, Tokens.STRING)
115 -
                push!(ret, ex)
124 +
                ex = EXPR(:STRING, lspan + startbytes, lspan + startbytes, str)
125 +
                if position(input) == (istrip ? 3 : 1) + 1
126 +
                    # Need to add empty :STRING at start to account for \"
127 +
                    pushtotrivia!(ret, ex)
128 +
                elseif !isempty(str)
129 +
                    push!(ret, ex)
130 +
                end
116 131
                istrip && adjust_lcp(ex)
117 132
                startbytes = 0
118 -
                op = mOPERATOR(1, 1, Tokens.EX_OR, false)
133 +
                op = EXPR(:OPERATOR, 1, 1, "\$")
119 134
                if peekchar(input) == '('
120 135
                    skip(input, 1) # skip past '('
121 136
                    lpfullspan = -position(input)
122 137
                    if iswhitespace(peekchar(input)) || peekchar(input) === '#'
123 138
                        read_ws_comment(input, readchar(input))
124 139
                    end
125 -
                    lparen = mPUNCTUATION(Tokens.LPAREN, lpfullspan + position(input) + 1, 1)
126 -
                    rparen = mPUNCTUATION(Tokens.RPAREN, 1, 1)
140 +
                    lparen = EXPR(:LPAREN, lpfullspan + position(input) + 1, 1)
141 +
                    rparen = EXPR(:RPAREN, 1, 1)
127 142
128 143
                    ps1 = ParseState(input)
129 144
130 145
                    if kindof(ps1.nt) === Tokens.RPAREN
131 -
                        call = mUnaryOpCall(op, EXPR(InvisBrackets, EXPR[lparen, rparen]))
132 -
                        push!(ret, call)
146 +
                        push!(ret, EXPR(:ERRORTOKEN, EXPR[], nothing))
147 +
                        pushtotrivia!(ret, op)
148 +
                        pushtotrivia!(ret, lparen)
149 +
                        pushtotrivia!(ret, rparen)
133 150
                        skip(input, 1)
134 151
                    else
135 -
                        interp = @closer ps1 :paren parse_expression(ps1)
136 -
                        call = mUnaryOpCall(op, EXPR(InvisBrackets, EXPR[lparen, interp, rparen]))
137 -
                        push!(ret, call)
152 +
                        interp_val = @closer ps1 :paren parse_expression(ps1)
153 +
                        push!(ret, interp_val)
154 +
                        pushtotrivia!(ret, op)
155 +
                        pushtotrivia!(ret, lparen)
156 +
                        pushtotrivia!(ret, rparen)
138 157
                        seek(input, ps1.nt.startbyte + 1)
139 158
                    end
140 159
                    # Compared to flisp/JuliaParser, we have an extra lookahead token,
141 160
                    # so we need to back up one here
142 161
                elseif Tokenize.Lexers.iswhitespace(peekchar(input)) || peekchar(input) === '#'
143 -
                    push!(ret, mErrorToken(ps, op, StringInterpolationWithTrailingWhitespace))
162 +
                    pushtotrivia!(ret, mErrorToken(ps, op, StringInterpolationWithTrailingWhitespace))
144 163
                else
145 164
                    pos = position(input)
146 165
                    ps1 = ParseState(input)
@@ -152,8 +171,8 @@
Loading
152 171
                    end
153 172
                    # Attribute trailing whitespace to the string
154 173
                    t = adjustspan(t)
155 -
                    call = mUnaryOpCall(op, t)
156 -
                    push!(ret, call)
174 +
                    push!(ret, t)
175 +
                    pushtotrivia!(ret, op)
157 176
                    seek(input, pos + t.fullspan)
158 177
                end
159 178
            else
@@ -166,23 +185,27 @@
Loading
166 185
        lspan = position(b)
167 186
        if b.size == 0
168 187
            ex = mErrorToken(ps, Unknown)
188 +
            push!(ret, ex)
169 189
        else
170 190
            str = tostr(b)
171 191
            if istrip
172 192
                str = str[1:prevind(str, lastindex(str), 3)]
173 193
                # only mark non-interpolated triple strings
174 -
                ex = mLITERAL(lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str, length(ret) == 0 ? Tokens.TRIPLE_STRING : Tokens.STRING)
194 +
                ex = EXPR(length(ret) == 0 ? :TRIPLESTRING : :STRING, lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str)
175 195
                adjust_lcp(ex, true)
176 196
            else
177 197
                str = str[1:prevind(str, lastindex(str))]
178 -
                ex = mLITERAL(lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str, Tokens.STRING)
198 +
                ex = EXPR(:STRING, lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str)
199 +
            end
200 +
            if isempty(str)
201 +
                pushtotrivia!(ret, ex)
202 +
            else
203 +
                push!(ret, ex)
179 204
            end
180 205
        end
181 -
        push!(ret, ex)
182 -
183 206
    end
184 207
185 -
    single_string_T = (Tokens.STRING, kindof(ps.t))
208 +
    single_string_T = (:STRING, :TRIPLESTRING, literalmap(kindof(ps.t)))
186 209
    if istrip
187 210
        if lcp !== nothing && !isempty(lcp)
188 211
            for expr in exprs_to_adjust
@@ -195,18 +218,18 @@
Loading
195 218
            end
196 219
        end
197 220
        # Drop leading newline
198 -
        if isliteral(ret.args[1]) && kindof(ret.args[1]) in single_string_T &&
221 +
        if !isempty(ret.args) && isliteral(ret.args[1]) && headof(ret.args[1]) in single_string_T &&
199 222
                !isempty(valof(ret.args[1])) && valof(ret.args[1])[1] == '\n'
200 223
            ret.args[1] = dropleadlingnewline(ret.args[1])
201 224
        end
202 225
    end
203 226
204 -
    if (length(ret.args) == 1 && isliteral(ret.args[1]) && kindof(ret.args[1]) in single_string_T)
227 +
    if (length(ret.args) == 1 && isliteral(ret.args[1]) && headof(ret.args[1]) in single_string_T) && !isinterpolated
205 228
        ret = ret.args[1]
206 229
    end
207 230
    update_span!(ret)
208 231
209 -
    return ret
232 +
    return iscmd ? wrapwithcmdmacro(ret) : ret
210 233
end
211 234
212 235
function adjustspan(x::EXPR)
@@ -214,4 +237,70 @@
Loading
214 237
    return x
215 238
end
216 239
217 -
dropleadlingnewline(x::EXPR) = mLITERAL(x.fullspan, x.span, valof(x)[2:end], kindof(x))
240 +
dropleadlingnewline(x::EXPR) = EXPR(headof(x), x.fullspan, x.span, valof(x)[2:end])
241 +
242 +
wrapwithcmdmacro(x) =EXPR(:macrocall, EXPR[EXPR(:globalrefcmd, 0, 0), EXPR(:NOTHING, 0, 0), x])
243 +
244 +
"""
245 +
    parse_prefixed_string_cmd(ps::ParseState, ret::EXPR)
246 +
247 +
Parse prefixed strings and commands such as `pre"text"`.
248 +
"""
249 +
function parse_prefixed_string_cmd(ps::ParseState, ret::EXPR)
250 +
    arg = parse_string_or_cmd(next(ps), ret)
251 +
252 +
    if ret.head === :IDENTIFIER && valof(ret) == "var" && isstringliteral(arg) && VERSION > v"1.3.0-"
253 +
        return EXPR(:NONSTDIDENTIFIER, EXPR[ret, arg], nothing)
254 +
    elseif headof(arg) === :macrocall && headof(arg.args[1]) === :globalrefcmd
255 +
        mname = EXPR(:IDENTIFIER, ret.fullspan, ret.span, string("@", valof(ret), "_cmd")) # NOTE: sizeof(valof(mname)) != mname.span
256 +
        return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0), arg.args[3]], nothing)
257 +
    elseif is_getfield(ret)
258 +
        if headof(ret.args[2]) === :quote || headof(ret.args[2]) === :quotenode
259 +
            str_type = valof(ret.args[2].args[1]) isa String ? valof(ret.args[2].args[1]) : "" # to handle some malformed case
260 +
            ret.args[2].args[1] = setparent!(EXPR(:IDENTIFIER, ret.args[2].args[1].fullspan, ret.args[2].args[1].span, string("@", str_type, "_str")), ret.args[2])
261 +
        else
262 +
            str_type = valof(ret.args[2]) isa String ? valof(ret.args[2]) : "" # to handle some malformed case
263 +
            ret.args[2] = EXPR(:IDENTIFIER, ret.args[2].fullspan, ret.args[2].span, string("@", str_type, "_str"))
264 +
        end
265 +
266 +
        return EXPR(:macrocall, EXPR[ret, EXPR(:NOTHING, 0, 0), arg], nothing)
267 +
    else
268 +
        return EXPR(:macrocall, EXPR[EXPR(:IDENTIFIER, ret.fullspan, ret.span, string("@", valof(ret), "_str")), EXPR(:NOTHING, 0, 0), arg], nothing)
269 +
    end
270 +
end
271 +
272 +
function unescape_prefixed(str)
273 +
    edits = UnitRange{Int}[]
274 +
    start = -1
275 +
    for (i, c) in enumerate(str)
276 +
        if start == -1 && c === '\\'
277 +
            start = i
278 +
        end
279 +
        if start > -1
280 +
            if c === '\\'
281 +
            elseif c === '\"'
282 +
                push!(edits, start:i)
283 +
                start = -1
284 +
            else 
285 +
                start = -1
286 +
            end
287 +
        end
288 +
    end
289 +
    
290 +
    if !isempty(edits) || start > -1
291 +
        str1 = deepcopy(str)
292 +
        if start > -1
293 +
            # slashes preceding closing '"'
294 +
            n = div(length(start:length(str)), 2) - 1
295 +
            str1 = string(str1[1:prevind(str1, start)], repeat("\\", n + 1))
296 +
        end
297 +
298 +
        for e in reverse(edits)
299 +
            n = div(length(e), 2) - 1
300 +
            str1 = string(str1[1:prevind(str1, first(e))], string(repeat("\\", n), "\""), str1[nextind(str1, last(e)):lastindex(str1)])
301 +
            
302 +
        end
303 +
        return str1
304 +
    end
305 +
    return str
306 +
end

@@ -13,15 +13,16 @@
Loading
13 13
    (ps.closer.inwhere && kindof(ps.nt) === Tokens.WHERE) ||
14 14
    (ps.closer.inwhere && ps.closer.ws && kindof(ps.t) === Tokens.RPAREN && isoperator(ps.nt) && precedence(ps.nt) < DeclarationOp) ||
15 15
    (ps.closer.precedence > WhereOp && (
16 -
        (kindof(ps.nt) === Tokens.LPAREN && !(ps.t.kind === Tokens.EX_OR)) ||
16 +
        (kindof(ps.nt) === Tokens.LPAREN && !(kindof(ps.t) === Tokens.EX_OR)) ||
17 17
        kindof(ps.nt) === Tokens.LBRACE ||
18 18
        kindof(ps.nt) === Tokens.LSQUARE ||
19 19
        (kindof(ps.nt) === Tokens.STRING && isemptyws(ps.ws)) ||
20 20
        ((kindof(ps.nt) === Tokens.RPAREN || kindof(ps.nt) === Tokens.RSQUARE) && isidentifier(ps.nt))
21 21
    )) ||
22 -
    (iscomma(ps.nt) && ps.closer.precedence > 0) ||
22 +
    (iscomma(ps.nt) && ps.closer.precedence > AssignmentOp) ||
23 +
    kindof(ps.nt) === Tokens.ENDMARKER ||
23 24
    (ps.closer.comma && iscomma(ps.nt)) ||
24 -
    (ps.closer.tuple && (iscomma(ps.nt) || isassignment(ps.nt))) ||
25 +
    (ps.closer.tuple && (iscomma(ps.nt) || isassignmentop(ps.nt))) ||
25 26
    (kindof(ps.nt) === Tokens.FOR && ps.closer.precedence > -1) ||
26 27
    (ps.closer.block && kindof(ps.nt) === Tokens.END) ||
27 28
    (ps.closer.paren && kindof(ps.nt) === Tokens.RPAREN) ||
@@ -41,7 +42,7 @@
Loading
41 42
        !(kindof(ps.nt) === Tokens.DO) &&
42 43
        !(
43 44
            (isbinaryop(ps.nt) && !(ps.closer.wsop && isemptyws(ps.nws) && isunaryop(ps.nt) && precedence(ps.nt) > 7)) ||
44 -
            (isunaryop(ps.t) && kindof(ps.ws) == WS && ps.lt.kind !== CSTParser.Tokens.COLON)
45 +
            (isunaryop(ps.t) && kindof(ps.ws) == WS && kindof(ps.lt) !== CSTParser.Tokens.COLON)
45 46
        )) ||
46 47
    (ps.closer.unary && (kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT, Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE) && isidentifier(ps.nt)))
47 48
end
@@ -201,31 +202,8 @@
Loading
201 202
end
202 203
203 204
204 -
205 -
isidentifier(x::EXPR) = typof(x) === IDENTIFIER || typof(x) === NONSTDIDENTIFIER
206 -
207 -
isunarycall(x::EXPR) = typof(x) === UnaryOpCall
208 -
isbinarycall(x::EXPR) = typof(x) === BinaryOpCall
209 -
iswherecall(x::EXPR) = typof(x) === WhereOpCall
210 -
isdeclaration(x::EXPR) = isbinarycall(x) && is_decl(x[2])
211 -
isinterpolant(x::EXPR) = isunarycall(x) && is_exor(x[1])
212 -
istuple(x::EXPR) = typof(x) === TupleH
213 -
is_either_id_op_interp(x::EXPR) = isidentifier(x) || isoperator(x) || isinterpolant(x)
214 -
is_splat(x::EXPR) = isunarycall(x) && is_dddot(x[2])
215 -
216 -
217 -
isliteral(x::EXPR) = typof(x) === LITERAL
218 -
iskw(x::EXPR) = typof(x) === KEYWORD # TODO: should change to `iskeyword`
219 -
ispunctuation(x::EXPR) = typof(x) === PUNCTUATION
220 -
221 -
isstring(x) = typof(x) === StringH || (isliteral(x) && (kindof(x) === Tokens.STRING || kindof(x) === Tokens.TRIPLE_STRING))
222 -
is_integer(x) = isliteral(x) && kindof(x) === Tokens.INTEGER
223 -
is_float(x) = isliteral(x) && kindof(x) === Tokens.FLOAT
224 -
is_number(x) = isliteral(x) && (kindof(x) === Tokens.INTEGER || kindof(x) === Tokens.FLOAT)
225 -
is_nothing(x) = isliteral(x) && kindof(x) === Tokens.NOTHING
226 -
227 -
isajuxtaposition(ps::ParseState, ret::EXPR) = ((is_number(ret) && (isidentifier(ps.nt) || kindof(ps.nt) === Tokens.LPAREN || kindof(ps.nt) === Tokens.CMD || kindof(ps.nt) === Tokens.STRING || kindof(ps.nt) === Tokens.TRIPLE_STRING)) ||
228 -
        ((typof(ret) === UnaryOpCall && is_prime(ret.args[2]) && isidentifier(ps.nt)) ||
205 +
isajuxtaposition(ps::ParseState, ret::EXPR) = ((isnumber(ret) && (isidentifier(ps.nt) || kindof(ps.nt) === Tokens.LPAREN || kindof(ps.nt) === Tokens.CMD || kindof(ps.nt) === Tokens.STRING || kindof(ps.nt) === Tokens.TRIPLE_STRING)) ||
206 +
        ((is_prime(ret.head) && isidentifier(ps.nt)) ||
229 207
        ((kindof(ps.t) === Tokens.RPAREN || kindof(ps.t) === Tokens.RSQUARE) && (isidentifier(ps.nt) || kindof(ps.nt) === Tokens.CMD)) ||
230 208
        ((kindof(ps.t) === Tokens.STRING || kindof(ps.t) === Tokens.TRIPLE_STRING) && (kindof(ps.nt) === Tokens.STRING || kindof(ps.nt) === Tokens.TRIPLE_STRING)))) || ((kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT) || kindof(ps.t) in (Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE)) && isidentifier(ps.nt))
231 209
@@ -237,55 +215,10 @@
Loading
237 215
`ParseState`, or exists as a (sub) expression of `x`.
238 216
"""
239 217
function has_error(x::EXPR)
240 -
    return typof(x) == ErrorToken || (x.args !== nothing && any(has_error, x.args))
218 +
    return headof(x) == :errortoken || (x.args !== nothing && any(has_error, x.args)) || (x.trivia !== nothing && any(has_error, x.trivia))
241 219
end
242 220
has_error(ps::ParseState) = ps.errored
243 221
244 -
# When using the FancyDiagnostics package, Meta.parse is the
245 -
# same as CSTParser.parse. Manually call the flisp parser here
246 -
# to make sure we test what we want, even when people load the
247 -
# FancyDiagnostics package.
248 -
function flisp_parse(str::AbstractString, pos::Int; greedy::Bool=true, raise::Bool=true)
249 -
    if VERSION < v"1.6-DEV"
250 -
        bstr = String(str)
251 -
        ex, pos = ccall(:jl_parse_string, Any,
252 -
                        (Ptr{UInt8}, Csize_t, Int32, Int32),
253 -
                        bstr, sizeof(bstr), pos - 1, greedy ? 1 : 0)
254 -
    else
255 -
        filename = "none"
256 -
        rule = greedy ? :statement : :atom
257 -
        ex, pos = Core.Compiler.fl_parse(str, filename, pos - 1, rule)
258 -
    end
259 -
    if raise && isa(ex, Expr) && ex.head === :error
260 -
        throw(Meta.ParseError(ex.args[1]))
261 -
    end
262 -
    if ex === ()
263 -
        raise && throw(Meta.ParseError("end of input"))
264 -
        ex = Expr(:error, "end of input")
265 -
    end
266 -
    # pos is zero-based byte offset
267 -
    return ex, pos + 1
268 -
end
269 -
270 -
function flisp_parse(str::AbstractString; raise::Bool=true)
271 -
    ex, pos = flisp_parse(str, 1, greedy=true, raise=raise)
272 -
    if isa(ex, Expr) && ex.head === :error
273 -
        return ex
274 -
    end
275 -
    if !(pos > ncodeunits(str))
276 -
        raise && throw(Meta.ParseError("extra token after end of expression"))
277 -
        return Expr(:error, "extra token after end of expression")
278 -
    end
279 -
    return ex
280 -
end
281 -
282 -
function flisp_parse(stream::IO; greedy::Bool=true, raise::Bool=true)
283 -
    pos = position(stream)
284 -
    ex, Δ = flisp_parse(read(stream, String), 1, greedy=greedy, raise=raise)
285 -
    seek(stream, pos + Δ - 1)
286 -
    return ex
287 -
end
288 -
289 222
using Base.Meta
290 223
291 224
function norm_ast(a::Any)
@@ -328,13 +261,14 @@
Loading
328 261
    return a
329 262
end
330 263
331 -
function flisp_parsefile(str, display=true)
332 -
    io = IOBuffer(str)
264 +
function flisp_parsefile(str, display = true)
265 +
    pos = 1
333 266
    failed = false
334 267
    x1 = Expr(:file)
335 268
    try
336 -
        while !eof(io)
337 -
            push!(x1.args, flisp_parse(io))
269 +
        while pos <= sizeof(str)
270 +
            x, pos = Meta.parse(str, pos)
271 +
            push!(x1.args, x)
338 272
        end
339 273
    catch er
340 274
        isa(er, InterruptException) && rethrow(er)
@@ -359,9 +293,9 @@
Loading
359 293
    if length(x.args) > 0 && is_nothing(x.args[1])
360 294
        popfirst!(x.args)
361 295
    end
362 -
    if length(x.args) > 0 && is_nothing(x.args[end])
363 -
        pop!(x.args)
364 -
    end
296 +
    # if length(x.args) > 0 && is_nothing(x.args[end])
297 +
    #     pop!(x.args)
298 +
    # end
365 299
    x0 = norm_ast(Expr(x))
366 300
    x0, has_error(ps), sp
367 301
end
@@ -386,8 +320,8 @@
Loading
386 320
        cumfail = 0
387 321
        printstyled(file, color=:green)
388 322
        println()
389 -
        c0, c1 = CSTParser.compare(x0, x1)
390 -
        printstyled(string("    ", c0), bold=true, color=:ligth_red)
323 +
        c0, c1 = compare(x0, x1)
324 +
        printstyled(string("    ", c0), bold = true, color = :light_red)
391 325
        println()
392 326
        printstyled(string("    ", c1), bold=true, color=:light_green)
393 327
        println()
@@ -473,21 +407,138 @@
Loading
473 407
    end
474 408
end
475 409
410 +
# code for updating CST
411 +
"""
412 +
    firstdiff(s0::AbstractString, s1::AbstractString)
413 +
414 +
Returns the last byte index, i, for which s0 and s1 are the same such that:
415 +
    `s0[1:i] == s1[1:i]`
416 +
"""
417 +
function firstdiff(s0::AbstractString, s1::AbstractString)
418 +
    minlength = min(sizeof(s0), sizeof(s1))
419 +
    @inbounds for i in 1:minlength
420 +
        if codeunits(s0)[i] !== codeunits(s1)[i]
421 +
            return i - 1 # This could return a non-commencing byte of a multi-byte unicode sequence.
422 +
        end
423 +
    end
424 +
    return minlength
425 +
end
426 +
427 +
"""
428 +
    revfirstdiff(s0::AbstractString, s1::AbstractString)
429 +
430 +
Reversed version of firstdiff but returns two indices, one for each string.
431 +
"""
432 +
function revfirstdiff(s0::AbstractString, s1::AbstractString)
433 +
    minlength = min(sizeof(s0), sizeof(s1))
434 +
    @inbounds for i in 0:minlength - 1
435 +
        if codeunits(s0)[end - i] !== codeunits(s1)[end - i]
436 +
            return sizeof(s0) - i, sizeof(s1) - i# This could return a non-commencing byte of a multi-byte unicode sequence.
437 +
        end
438 +
    end
439 +
    return 1, 1
440 +
end
441 +
442 +
"""
443 +
    find_arg_at(x, i)
444 +
445 +
Returns the index of the node of `x` within which the byte offset `i` falls.
446 +
"""
447 +
function find_arg_at(x::CSTParser.EXPR, i)
448 +
    @assert i <= x.fullspan
449 +
    offset = 0
450 +
    for (cnt, a) in enumerate(x.args)
451 +
        if i <= offset + a.fullspan
452 +
            return cnt
453 +
        end
454 +
        offset += a.fullspan
455 +
    end
456 +
    error()
457 +
end
458 +
459 +
comp(x, y) = x == y
460 +
function comp(x::CSTParser.EXPR, y::CSTParser.EXPR)
461 +
    comp(x.head, y.head) && 
462 +
    x.span == y.span && 
463 +
    x.fullspan == y.fullspan && 
464 +
    x.val == y.val && 
465 +
    length(x) == length(y) && 
466 +
    all(comp(x[i], y[i]) for i = 1:length(x))
467 +
end
468 +
469 +
function minimal_reparse(s0, s1, x0 = CSTParser.parse(s0, true), x1 = CSTParser.parse(s1, true); inds = false)
470 +
    i0 = firstdiff(s0, s1)
471 +
    i1, i2 = revfirstdiff(s0, s1)
472 +
    # Find unaffected expressions at start
473 +
    # CST should be unaffected (and able to be copied across) up to this point, 
474 +
    # but we need to check.
475 +
    r1 = 1:min(find_arg_at(x0, i0) - 1, length(x0.args), find_arg_at(x1, i0) - 1)
476 +
    for i = 1:min(find_arg_at(x0, i0) - 1, find_arg_at(x1, i0) - 1)
477 +
        if x0.args[i].fullspan !== x1.args[i].fullspan
478 +
            r1 = 1:(i-1)
479 +
            break
480 +
        end
481 +
        r1 = 1:i
482 +
    end
483 +
    # we can re-use x0.args[r1]
484 +
    
485 +
    # assume we'll just use x1.args from here on
486 +
    r2 = (last(r1) + 1):length(x1.args)
487 +
    r3 = 0:-1
488 +
489 +
    # though we now check whether there is a sequence at the end of x0.args and 
490 +
    # x1.args that match
491 +
    for i = 0:min(last(r1), length(x0.args), length(x1.args)) - 1
492 +
        # if x0.args[end - i].fullspan !== x1.args[end - i].fullspan || 
493 +
        #     headof(x0.args[end-i]) == :errortoken ? a : !comp(x0.args[end - i].head, x1.args[end - i].head) 
494 +
        if !comp(x0.args[end - i], x1.args[ end - i])
495 +
            r2 = first(r2):length(x1.args) - i
496 +
            r3 = length(x0.args) .+ ((-i + 1):0)
497 +
            break
498 +
        end
499 +
    end
500 +
    inds && return r1, r2, r3
501 +
    x2 = CSTParser.EXPR(x0.head, CSTParser.EXPR[
502 +
        x0.args[r1]
503 +
        x1.args[r2]
504 +
        x0.args[r3]
505 +
    ], nothing)
506 +
    return x2
507 +
end
508 +
509 +
# Quick and very dirty comparison of two EXPR, makes extra effort for :errortokens
510 +
function quick_comp(a::EXPR, b::EXPR)
511 +
    a.fullspan == b.fullspan && 
512 +
    headof(a) === :errortoken ? headof(b) === :errortoken && length(a.args) > 0 && length(a.args) == length(b.args) && quick_comp(first(a.args), first(b.args)) : 
513 +
        comp(headof(a), headof(b))
514 +
end
515 +
476 516
"""
477 517
check_span(x, neq = [])
478 518
479 519
Recursively checks whether the span of an expression equals the sum of the span
480 520
of its components. Returns a vector of failing expressions.
481 521
"""
482 -
function check_span(x::EXPR, neq=[])
483 -
    (ispunctuation(x) || isidentifier(x) || iskw(x) || isoperator(x) || isliteral(x) || typof(x) == StringH) && return neq
522 +
function check_span(x::EXPR, neq = [])
523 +
    (ispunctuation(x) || isidentifier(x) || iskeyword(x) || isoperator(x) || isliteral(x) || headof(x) == :string) && return neq
484 524
485 525
    s = 0
486 -
    for a in x.args
487 -
        check_span(a, neq)
488 -
        s += a.fullspan
526 +
    if x.args !== nothing
527 +
        for a in x.args
528 +
            check_span(a, neq)
529 +
            s += a.fullspan
530 +
        end
531 +
    end
532 +
    if hastrivia(x)
533 +
        for a in x.trivia
534 +
            check_span(a, neq)
535 +
            s += a.fullspan
536 +
        end
537 +
    end
538 +
    if x.head isa EXPR
539 +
        s += x.head.fullspan
489 540
    end
490 -
    if length(x.args) > 0 && s != x.fullspan
541 +
    if length(x) > 0 && s != x.fullspan
491 542
        push!(neq, x)
492 543
    end
493 544
    neq
@@ -507,11 +558,11 @@
Loading
507 558
Attempt to get a string representation of a nodeless expression.
508 559
"""
509 560
function str_value(x)
510 -
    if typof(x) === IDENTIFIER || typof(x) === LITERAL
561 +
    if headof(x) === :IDENTIFIER || isliteral(x)
511 562
        return valof(x)
512 563
    elseif isidentifier(x)
513 564
        valof(x.args[2])
514 -
    elseif typof(x) === OPERATOR || typof(x) === MacroName
565 +
    elseif isoperator(x)
515 566
        return string(Expr(x))
516 567
    else
517 568
        return ""
@@ -627,12 +678,8 @@
Loading
627 678
    return true
628 679
end
629 680
630 -
"""
631 -
    is_getfield(x::EXPR)
632 681
633 -
Is this an expression of the form `a.b`.
634 -
"""
635 -
is_getfield(x::EXPR) = isbinarycall(x) && length(x) == 3 && kindof(x[2]) === Tokens.DOT
682 +
636 683
637 684
"""
638 685
    disallowednumberjuxt(ret::EXPR)
@@ -640,45 +687,33 @@
Loading
640 687
Does this number literal end in a decimal and so cannot precede a paren for
641 688
implicit multiplication?
642 689
"""
643 -
disallowednumberjuxt(ret::EXPR) = is_number(ret) && last(valof(ret)) == '.'
690 +
disallowednumberjuxt(ret::EXPR) = isnumber(ret) && last(valof(ret)) == '.'
644 691
645 692
646 693
nexttokenstartsdocstring(ps::ParseState) = isidentifier(ps.nt) && val(ps.nt, ps) == "doc" && (kindof(ps.nnt) === Tokens.STRING || kindof(ps.nnt) === Tokens.TRIPLE_STRING)
647 694
648 -
"""
649 -
    is_wrapped_assignment(x::EXPR)
650 -
    
651 -
Is `x` an assignment expression, ignoring any surrounding parentheses.
652 -
"""
653 -
is_wrapped_assignment(x::EXPR) = is_assignment(x) || (isbracketed(x) && is_wrapped_assignment(x.args[2]))
654 -
655 -
"""
656 -
    is_range(x::EXPR)
657 695
658 -
Is `x` a valid iterator for use in `for` loops or generators?
659 -
"""
660 -
is_range(x::EXPR) = isbinarycall(x) && (is_eq(x.args[2]) || is_in(x.args[2]) || is_elof(x.args[2]))
661 696
662 697
"""
663 698
    _do_kw_convert(ps::ParseState, a::EXPR)
664 699
665 700
Should `a` be converted to a keyword-argument expression?
666 701
"""
667 -
_do_kw_convert(ps::ParseState, a::EXPR) = !ps.closer.brace && is_assignment(a)
702 +
_do_kw_convert(ps::ParseState, a::EXPR) = !ps.closer.brace && isassignment(a)
668 703
669 704
"""
670 705
    _kw_convert(ps::ParseState, a::EXPR)
671 706
672 707
Converted an assignment expression to a keyword-argument expression.
673 708
"""
674 -
_kw_convert(a::EXPR) = EXPR(Kw, EXPR[a.args[1], a.args[2], a.args[3]], a.fullspan, a.span)
709 +
_kw_convert(x::EXPR) = EXPR(:kw, EXPR[x.args[1], x.args[2]], EXPR[x.head], x.fullspan, x.span)
675 710
676 711
"""
677 712
    convertsigtotuple(sig::EXPR)
678 713
679 714
When parsing a function or macro signature, should it be converted to a tuple?
680 715
"""
681 -
convertsigtotuple(sig::EXPR) = isbracketed(sig) && !(istuple(sig.args[2]) || (typof(sig.args[2]) === Block) || is_splat(sig.args[2]))
716 +
convertsigtotuple(sig::EXPR) = isbracketed(sig) && !(istuple(sig.args[1]) || (headof(sig.args[1]) === :block) || issplat(sig.args[1]))
682 717
683 718
"""
684 719
    docable(head)
@@ -686,16 +721,11 @@
Loading
686 721
When parsing a block of expressions, can documentation be attached? Prefixed docs at the
687 722
top-level are handled within `parse(ps::ParseState, cont = false)`.
688 723
"""
689 -
docable(head) = head === Begin || head === ModuleH || head === BareModule || head === Quote
724 +
docable(head) = head === :begin || head === :module || head === :baremodule || head === :quote
690 725
691 726
692 727
should_negate_number_literal(ps::ParseState, op::EXPR) = (is_plus(op) || is_minus(op)) && (kindof(ps.nt) === Tokens.INTEGER || kindof(ps.nt) === Tokens.FLOAT) && isemptyws(ps.ws) && kindof(ps.nnt) != Tokens.CIRCUMFLEX_ACCENT
693 728
694 -
isbracketed(x::EXPR) = typof(x) === InvisBrackets # Assumption that x has 3 args, doesn't need checking?
695 -
696 -
unwrapbracket(x::EXPR) = isbracketed(x) ? unwrapbracket(x[2]) : x
697 -
698 -
isbeginorblock(x::EXPR) = typof(x) === Begin || typof(unwrapbracket(x)) == Block
699 729
700 730
"""
701 731
    can_become_comparison(x::EXPR)
@@ -703,7 +733,7 @@
Loading
703 733
Is `x` a binary comparison call (e.g. `a < b`) that can be extended to include more
704 734
arguments?
705 735
"""
706 -
can_become_comparison(x::EXPR) = isbinarycall(x) && (precedence(x.args[2]) == ComparisonOp || is_issubt(x.args[2]) || is_issupt(x.args[2]))
736 +
can_become_comparison(x::EXPR) = (isoperator(x.head) && comp_prec(valof(x.head)) && length(x.args) > 1) || (x.head === :call && isoperator(x.args[1]) && comp_prec(valof(x.args[1])) && length(x.args) > 2 && !hastrivia(x))
707 737
708 738
"""
709 739
    can_become_chain(x::EXPR, op::EXPR)
@@ -711,10 +741,19 @@
Loading
711 741
Is `x` a binary call for `+` or `*` that can be extended to include more
712 742
arguments?
713 743
"""
714 -
can_become_chain(x::EXPR, op::EXPR) = isbinarycall(x) && (is_star(op) || is_plus(op)) && kindof(op) == kindof(x.args[2]) && !x.args[2].dot && x.args[2].span > 0
744 +
can_become_chain(x::EXPR, op::EXPR) = isbinarycall(x) && (is_star(op) || is_plus(op)) && valof(op) == valof(x.args[2]) && !isdotted(x.args[2]) && x.args[2].span > 0
745 +
746 +
747 +
macro cst_str(x)
748 +
    CSTParser.parse(x)
749 +
end
750 +
751 +
function issuffixableliteral(ps::ParseState, x::EXPR) 
752 +
    isidentifier(ps.nt) && isemptyws(ps.ws) && ismacrocall(x) && (valof(x.args[1]) isa String && (endswith(valof(x.args[1]), "_str") || endswith(valof(x.args[1]), "_cmd")))
753 +
end
715 754
716 755
function loop_check(ps, prevpos)
717 -
    if position(ps) <= prevpos
756 +
    if position(ps) <= prevpos && ps.nt.kind !== Tokens.ENDMARKER
718 757
        throw(CSTInfiniteLoop("Infinite loop at $ps"))
719 758
    else
720 759
        position(ps)

@@ -1,33 +1,21 @@
Loading
1 -
function Base.show(io::IO, x::EXPR, offset=0, d=0, er=false)
2 -
    T = typof(x)
3 -
    c =  T === ErrorToken || er ? :red : :normal
1 +
function Base.show(io::IO, x::EXPR, offset = 0, d = 0, er = false)
2 +
    T = headof(x)
3 +
    c =  T === :errortoken || er ? :red : :normal
4 4
    # Print span as 1-based range of the source string. This presentation is
5 5
    # simple to understand when strings are presented to CSTParser.parse().
6 6
    print(io, lpad(offset + 1, 3), ":", rpad(offset + x.fullspan, 3), " ")
7 7
    if isidentifier(x)
8 -
        printstyled(io, " "^d, typof(x) == NONSTDIDENTIFIER ? valof(x.args[2]) : valof(x), color=:yellow)
8 +
        printstyled(io, " "^d, headof(x) == :NONSTDIDENTIFIER ? valof(x.args[2]) : valof(x), color = :yellow)
9 9
        x.meta !== nothing && show(io, x.meta)
10 -
        println(io)
10 +
        print(io)
11 11
    elseif isoperator(x)
12 -
        printstyled(io, " "^d, "OP: ", kindof(x), "\n", color=c)
13 -
    elseif iskw(x)
14 -
        printstyled(io, " "^d, kindof(x), "\n", color=:magenta)
12 +
        printstyled(io, " "^d, "OP: ", valof(x), color = c)
13 +
    elseif iskeyword(x)
14 +
        printstyled(io, " "^d, headof(x), color = :magenta)
15 15
    elseif ispunctuation(x)
16 -
        if kindof(x) === Tokens.LPAREN
17 -
            printstyled(io, " "^d, "(\n", color=c)
18 -
        elseif kindof(x) === Tokens.RPAREN
19 -
            printstyled(io, " "^d, ")\n", color=c)
20 -
        elseif kindof(x) === Tokens.LSQUARE
21 -
            printstyled(io, " "^d, "[\n", color=c)
22 -
        elseif kindof(x) === Tokens.RSQUARE
23 -
            printstyled(io, " "^d, "]\n", color=c)
24 -
        elseif kindof(x) === Tokens.COMMA
25 -
            printstyled(io, " "^d, ",\n", color=c)
26 -
        else
27 -
            printstyled(io, " "^d, "PUNC: ", kindof(x), "\n", color=c)
28 -
        end
16 +
        printstyled(io, " "^d, punctuationprinting[headof(x)], color = c)
29 17
    elseif isliteral(x)
30 -
        printstyled(io, " "^d, "$(kindof(x)): ", valof(x), "\n", color=c)
18 +
        printstyled(io, " "^d, "$(headof(x)): ", valof(x) === nothing ? "nothing" : valof(x), color = c)
31 19
    else
32 20
        printstyled(io, " "^d, T, color=c)
33 21
        if x.meta !== nothing
@@ -35,15 +23,25 @@
Loading
35 23
            show(io, x.meta)
36 24
            print(io, ")")
37 25
        end
38 -
        println(io)
39 26
        x.args === nothing && return
40 27
        for a in x.args
28 +
            println(io)
41 29
            show(io, a, offset, d + 1, er)
42 30
            offset += a.fullspan
43 31
        end
44 32
    end
45 33
end
46 34
35 +
const punctuationprinting = Dict(:COMMA => ",",
36 +
:LPAREN => "(",
37 +
:RPAREN => ")",
38 +
:LSQUARE => "[",
39 +
:RSQUARE => "]",
40 +
:LBRACE => "{",
41 +
:RBRACE => "}",
42 +
:ATSIGN => "@",
43 +
:DOT => ".")
44 +
47 45
struct CSTInfiniteLoop <: Exception
48 46
    msg::AbstractString
49 47
end

@@ -10,14 +10,8 @@
Loading
10 10
        if kindof(ps.nt) ∈ term_c # error handling if an unexpected closer is hit
11 11
            if kindof(ps.nt) === Tokens.ENDMARKER
12 12
                break
13 -
            elseif kindof(ps.nt) === Tokens.RPAREN
14 -
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
15 -
            elseif kindof(ps.nt) === Tokens.RBRACE
16 -
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
17 -
            elseif kindof(ps.nt) === Tokens.RSQUARE
18 -
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
19 13
            else
20 -
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
14 +
                push!(ret, mErrorToken(ps, EXPR(next(ps)), UnexpectedToken))
21 15
            end
22 16
        else
23 17
            if docable
@@ -37,22 +31,40 @@
Loading
37 31
error expression if an invalid expression is parsed (anything other than
38 32
`=`, `in`, `∈`).
39 33
"""
40 -
function parse_iterator(ps::ParseState, outer=parse_outer(ps))
41 -
    arg = @closer ps :range @closer ps :ws parse_expression(ps)
34 +
function parse_iterator(ps::ParseState, outer = parse_outer(ps))
35 +
    arg = @closer ps :range @closer ps :ws @nocloser ps :wsop parse_expression(ps)
42 36
    if !is_range(arg)
43 37
        arg = mErrorToken(ps, arg, InvalidIterator)
38 +
    else 
39 +
        arg = adjust_iter(arg)
44 40
    end
45 41
    if outer !== nothing
46 -
        arg.args[1] = setparent!(EXPR(Outer, EXPR[outer, arg.args[1]]), arg)
42 +
        arg.args[1] = setparent!(EXPR(:outer, EXPR[arg.args[1]], EXPR[outer]), arg)
47 43
        arg.fullspan += outer.fullspan
48 44
        arg.span = outer.fullspan + arg.span
49 45
    end
50 46
    return arg
51 47
end
52 48
49 +
function adjust_iter(x::EXPR)
50 +
    # Assumes x is a valid iterator
51 +
    if x.head === :call # isoperator(x.args[1]) && x.args[1].val in ("in", "∈")
52 +
        EXPR(EXPR(:OPERATOR, 0, 0, "="), EXPR[x.args[2], x.args[3]], EXPR[x.args[1]])
53 +
    else 
54 +
        x
55 +
    end
56 +
end
57 +
58 +
"""
59 +
    is_range(x::EXPR)
60 +
61 +
Is `x` a valid iterator for use in `for` loops or generators?
62 +
"""
63 +
is_range(x::EXPR) = isassignment(x) || (x.head === :call && (is_in(x.args[1]) || is_elof(x.args[1])))
64 +
53 65
function parse_outer(ps)
54 66
    if kindof(ps.nt) === Tokens.OUTER && kindof(ps.nws) !== EmptyWS && !Tokens.isoperator(kindof(ps.nnt))
55 -
        outer = INSTANCE(next(ps))
67 +
        outer = EXPR(next(ps))
56 68
    end
57 69
end
58 70
@@ -62,23 +74,12 @@
Loading
62 74
Parses a group of iterators e.g. used in a `for` loop or generator. Can allow
63 75
for a succeeding `Filter` expression.
64 76
"""
65 -
function parse_iterators(ps::ParseState, allowfilter=false)
66 -
    arg = parse_iterator(ps)
67 -
    if iscomma(ps.nt) # we've hit a comma separated list of iterators.
68 -
        arg = EXPR(Block, EXPR[arg])
69 -
        prevpos = position(ps)
70 -
        while iscomma(ps.nt)
71 -
            accept_comma(ps, arg)
72 -
            nextarg = parse_iterator(ps)
73 -
            push!(arg, nextarg)
74 -
            prevpos = loop_check(ps, prevpos)
75 -
        end
76 -
    end
77 -
78 -
    if allowfilter
79 -
        arg = parse_filter(ps, arg)
77 +
function parse_iterators(ps::ParseState, iters, trivia)
78 +
    push!(iters, parse_iterator(ps))
79 +
    if iscomma(ps.nt)
80 +
        push!(trivia, accept_comma(ps))
81 +
        parse_iterators(ps, iters, trivia)
80 82
    end
81 -
    return arg
82 83
end
83 84
84 85
"""
@@ -88,14 +89,13 @@
Loading
88 89
"""
89 90
function parse_filter(ps::ParseState, arg)
90 91
    if kindof(ps.nt) === Tokens.IF # assumes we're inside a generator
91 -
        if typof(arg) === Block
92 -
            arg = EXPR(Filter, arg.args)
92 +
        trivia = EXPR[EXPR(next(ps))]
93 +
        cond = @closer ps :range parse_expression(ps)
94 +
        if headof(arg) === :block
95 +
            arg = EXPR(:filter, EXPR[cond; arg.args], trivia)
93 96
        else
94 -
            arg = EXPR(Filter, EXPR[arg])
97 +
            arg = EXPR(:filter, EXPR[cond, arg], trivia)
95 98
        end
96 -
        push!(arg, mKEYWORD(next(ps)))
97 -
        cond = @closer ps :range parse_expression(ps)
98 -
        push!(arg, cond)
99 99
    end
100 100
    return arg
101 101
end
@@ -110,27 +110,31 @@
Loading
110 110
        arg = @closer ps :unary @closer ps :inwhere @precedence ps PowerOp parse_expression(ps)
111 111
        if istuple(arg)
112 112
            pushfirst!(arg.args, ret)
113 -
            ret = EXPR(Call, arg.args)
114 -
        elseif iswherecall(arg) && istuple(arg.args[1])
115 -
            ret = mWhereOpCall(EXPR(Call, EXPR[ret; arg.args[1].args]), arg.args[2], arg.args[3:end])
113 +
            ret = EXPR(:call, arg.args, arg.trivia)
116 114
        else
117 -
            ret = mUnaryOpCall(ret, arg)
115 +
            ret = EXPR(:call, EXPR[ret, arg], nothing)
118 116
        end
119 117
    elseif is_and(ret) || is_decl(ret) || is_exor(ret)
120 118
        arg = @precedence ps 20 parse_expression(ps)
121 -
        if is_exor(ret) && istuple(arg) && length(arg) == 3 && is_splat(arg.args[2])
122 -
            arg = EXPR(InvisBrackets, arg.args)
119 +
        if is_exor(ret) && istuple(arg) && length(arg) == 3 && issplat(arg.args[2])
120 +
            arg = EXPR(:brackets, arg.args)
123 121
        end
124 -
        ret = mUnaryOpCall(ret, arg)
125 -
    elseif is_issubt(ret) || is_issupt(ret)
126 -
        arg = @precedence ps PowerOp parse_expression(ps)
127 -
        ret = EXPR(Call, EXPR[ret; arg.args])
122 +
        ret = EXPR(ret, EXPR[arg], nothing)
128 123
    else
129 -
        !ismacro && typof(ret) === MacroName && (ismacro = true)
130 -
        args = EXPR[ret, mPUNCTUATION(next(ps))]
131 -
        @closeparen ps @default ps parse_comma_sep(ps, args, !ismacro)
132 -
        accept_rparen(ps, args)
133 -
        ret = EXPR(ismacro ? MacroCall : Call, args)
124 +
        !ismacro && ismacroname(ret) && (ismacro = true)
125 +
        syntaxcall = is_issubt(ret) || is_issupt(ret)
126 +
        args = if syntaxcall
127 +
            EXPR[]
128 +
        elseif ismacro
129 +
            EXPR[ret, EXPR(:NOTHING, 0, 0)]
130 +
        else
131 +
            EXPR[ret]
132 +
        end
133 +
        # args = ismacro ? EXPR[ret, EXPR(:NOTHING, 0, 0)] : EXPR[ret] 
134 +
        trivia = EXPR[EXPR(next(ps))]
135 +
        @closeparen ps @default ps parse_comma_sep(ps, args, trivia, !ismacro, insert_params_at = ismacro ? 3 : 2)
136 +
        accept_rparen(ps, trivia)
137 +
        ret = EXPR(ismacro ? :macrocall : syntaxcall ? ret : :call, args, trivia)
134 138
    end
135 139
    return ret
136 140
end
@@ -139,7 +143,7 @@
Loading
139 143
Parses a comma separated list, optionally allowing for conversion of 
140 144
assignment (`=`) expressions to `Kw`.
141 145
"""
142 -
function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, kw=true, block=false, istuple=false)
146 +
function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, trivia::Vector{EXPR}, kw = true, block = false, istuple = false; insert_params_at = 2)
143 147
    prevpos = position(ps)
144 148
    @nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while !closer(ps)
145 149
        a = parse_expression(ps)
@@ -148,38 +152,45 @@
Loading
148 152
        end
149 153
        push!(args, a)
150 154
        if iscomma(ps.nt)
151 -
            accept_comma(ps, args)
152 -
        else# if kindof(ps.ws) == SemiColonWS
155 +
            if istuple
156 +
                # As soon as we hit a comma in a list we can no longer convert to a block expr
157 +
                block = false
158 +
            end
159 +
            accept_comma(ps, trivia)
160 +
        elseif kindof(ps.ws) == SemiColonWS
153 161
            break
162 +
        elseif !closer(ps)
163 +
            # We've not hit a closing token, nor separating punctuation so let's insert an error-wrapped comma
164 +
            push!(trivia, EXPR(:errortoken, EXPR[EXPR(:COMMA, 0, 0)], nothing))
154 165
        end
155 166
        prevpos = loop_check(ps, prevpos)
156 167
    end
157 -
    if istuple && length(args) > 2
168 +
    if istuple && length(args) > 1
158 169
        block = false
159 170
    end
160 171
161 172
    if kindof(ps.ws) == SemiColonWS
162 -
        if @nocloser ps :newline @closer ps :comma @nocloser ps :semicolon closer(ps)
163 -
            if block && !(length(args) == 1 && ispunctuation(args[1])) && !(typof(last(args)) === UnaryOpCall && is_dddot(last(args).args[2]))
164 -
                push!(args, EXPR(Block, EXPR[pop!(args)]))
165 -
            elseif kw && kindof(ps.nt) === Tokens.RPAREN
166 -
                push!(args, EXPR(Parameters, EXPR[], 0, 0))
173 +
        if @nocloser ps :inwhere @nocloser ps :newline @closer ps :comma @nocloser ps :semicolon closer(ps)
174 +
            if block && !(length(args) == 0 && ispunctuation(trivia[1])) && !(isunarycall(last(args)) && is_dddot(last(args).args[2]))
175 +
                push!(args, EXPR(:block, EXPR[pop!(args)]))
176 +
            else
177 +
                insert!(args, insert_params_at, EXPR(:parameters, EXPR[], nothing, 0, 0))
167 178
            end
168 179
        else
169 180
            a = @nocloser ps :newline @closer ps :comma @nocloser ps :inwhere parse_expression(ps)
170 -
            if block && !(length(args) == 1 && ispunctuation(args[1])) && !is_splat(last(args)) && !(istuple && iscomma(ps.nt))
181 +
            if block && !(length(args) == 0 && ispunctuation(trivia[1])) && !issplat(last(args)) && !(istuple && iscomma(ps.nt))
171 182
                args1 = EXPR[pop!(args), a]
172 183
                prevpos = position(ps)
173 -
                @nocloser ps :newline @closer ps :comma while @nocloser ps :semicolon !closer(ps)
184 +
                @nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while @nocloser ps :semicolon !closer(ps)
174 185
                    a = parse_expression(ps)
175 186
                    push!(args1, a)
176 187
                    prevpos = loop_check(ps, prevpos)
177 188
                end
178 -
                body = EXPR(Block, args1)
189 +
                body = EXPR(:block, args1)
179 190
                push!(args, body)
180 191
                args = body
181 192
            else
182 -
                parse_parameters(ps, args, EXPR[a])
193 +
                parse_parameters(ps, args, EXPR[a], insert_params_at)
183 194
            end
184 195
        end
185 196
    end
@@ -191,15 +202,12 @@
Loading
191 202
192 203
Parses parameter arguments for a function call (e.g. following a semicolon).
193 204
"""
194 -
function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR}=EXPR[]; usekw=true)
205 +
function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR} = EXPR[], insert_params_at = 2; usekw = true)
206 +
    trivia = EXPR[]
195 207
    isfirst = isempty(args1)
196 208
    prevpos = position(ps)
197 209
    @nocloser ps :inwhere @nocloser ps :newline  @closer ps :comma while !isfirst || (@nocloser ps :semicolon !closer(ps))
198 -
        if isfirst
199 -
            a = parse_expression(ps)
200 -
        else
201 -
            a = first(args1)
202 -
        end
210 +
        a = isfirst ? parse_expression(ps) : first(args1)
203 211
        if usekw && _do_kw_convert(ps, a)
204 212
            a = _kw_convert(a)
205 213
        end
@@ -210,55 +218,69 @@
Loading
210 218
            push!(args1, a)
211 219
        end
212 220
        if iscomma(ps.nt)
213 -
            accept_comma(ps, args1)
221 +
            accept_comma(ps, trivia)
222 +
        elseif kindof(ps.ws) !== SemiColonWS && !closer(ps)
223 +
            push!(trivia, EXPR(:errortoken, EXPR[EXPR(:COMMA, 0, 0)], nothing))
214 224
        end
215 225
        if kindof(ps.ws) == SemiColonWS
216 226
            parse_parameters(ps, args1; usekw=usekw)
217 227
        end
218 -
        if isfirst
219 -
            prevpos = loop_check(ps, prevpos)
220 -
        else
221 -
            prevpos = position(ps)
222 -
        end
228 +
        prevpos = isfirst ? loop_check(ps, prevpos) : position(ps)
223 229
        isfirst = true
224 230
    end
225 231
    if !isempty(args1)
226 -
        paras = EXPR(Parameters, args1)
227 -
        push!(args, paras)
232 +
        insert!(args, insert_params_at, EXPR(:parameters, args1, trivia))
228 233
    end
229 234
    return
230 235
end
231 236
237 +
function parse_macroname(ps)
238 +
    at = EXPR(ps)
239 +
    if !isemptyws(ps.ws)
240 +
        ws = ps.ws.endbyte - ps.ws.startbyte + 1
241 +
        mname = INSTANCE(next(ps))
242 +
        mname.val = valof(mname) isa String ? string("@", " "^ws, valof(mname)) : string("@", " "^ws)
243 +
        mname.span += ws
244 +
        mname.fullspan += ws
245 +
        mname = mErrorToken(ps, mname, UnexpectedWhiteSpace)
246 +
    else
247 +
        next(ps)
248 +
        # set span/fullspan min length at 1 to account for the case of a lonely '@'
249 +
        mname = EXPR(:IDENTIFIER, max(1, ps.nt.startbyte - ps.t.startbyte + 1), max(1, ps.t.endbyte - ps.t.startbyte + 2), string("@", val(ps.t, ps)))
250 +
    end
251 +
end
252 +
232 253
"""
233 254
    parse_macrocall(ps)
234 255
235 256
Parses a macro call. Expects to start on the `@`.
236 257
"""
237 258
function parse_macrocall(ps::ParseState)
238 -
    at = mPUNCTUATION(ps)
239 -
    if !isemptyws(ps.ws)
240 -
        mname = mErrorToken(ps, INSTANCE(next(ps)), UnexpectedWhiteSpace)
241 -
    else
242 -
        mname = EXPR(MacroName, EXPR[at, mIDENTIFIER(next(ps))])
243 -
    end
244 -
259 +
    mname = parse_macroname(ps)
245 260
    # Handle cases with @ at start of dotted expressions
246 261
    if kindof(ps.nt) === Tokens.DOT && isemptyws(ps.ws)
247 262
        prevpos = position(ps)
248 263
        while kindof(ps.nt) === Tokens.DOT
249 -
            op = mOPERATOR(next(ps))
250 -
            nextarg = mIDENTIFIER(next(ps))
251 -
            mname = mBinaryOpCall(mname, op, EXPR(Quotenode, EXPR[nextarg]))
264 +
            op = EXPR(:OPERATOR, next(ps))
265 +
            nextarg = EXPR(:IDENTIFIER, next(ps))
266 +
            mname = EXPR(op, EXPR[mname, EXPR(:quotenode, EXPR[nextarg], nothing)], nothing)
252 267
            prevpos = loop_check(ps, prevpos)
253 268
        end
254 269
    end
255 270
256 271
    if iscomma(ps.nt)
257 -
        return EXPR(MacroCall, EXPR[mname], mname.fullspan, mname.span)
272 +
        return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0)], nothing, mname.fullspan, mname.span)
258 273
    elseif isemptyws(ps.ws) && kindof(ps.nt) === Tokens.LPAREN
259 274
        return parse_call(ps, mname, true)
275 +
    elseif isemptyws(ps.ws) && kindof(ps.nt) === Tokens.LBRACE
276 +
        next(ps)
277 +
        return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0), @default ps @closebrace ps parse_braces(ps)], nothing)
278 +
    elseif isemptyws(ps.ws) && kindof(ps.nt) === Tokens.LSQUARE
279 +
        next(ps)
280 +
        return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0), @default ps parse_array(ps)], nothing)
260 281
    else
261 -
        args = EXPR[mname]
282 +
        #TODO add special hndling for @doc
283 +
        args = EXPR[mname, EXPR(:NOTHING, 0, 0)]
262 284
        insquare = ps.closer.insquare
263 285
        prevpos = position(ps)
264 286
        @default ps while !closer(ps)
@@ -273,7 +295,7 @@
Loading
273 295
            end
274 296
            prevpos = loop_check(ps, prevpos)
275 297
        end
276 -
        return EXPR(MacroCall, args)
298 +
        return EXPR(:macrocall, args, nothing)
277 299
    end
278 300
end
279 301
@@ -283,87 +305,98 @@
Loading
283 305
Having hit `for` not at the beginning of an expression return a generator.
284 306
Comprehensions are parsed as SQUAREs containing a generator.
285 307
"""
286 -
function parse_generator(ps::ParseState, ret::EXPR)
287 -
    kw = mKEYWORD(next(ps))
288 -
    ret = EXPR(Generator, EXPR[ret, kw])
289 -
    ranges = @closesquare ps parse_iterators(ps, true)
290 -
291 -
    if typof(ranges) === Block
292 -
        append!(ret, ranges)
308 +
function parse_generator(ps::ParseState, first::EXPR)
309 +
    kw = EXPR(next(ps))
310 +
    iters, trivia = EXPR[], EXPR[]
311 +
    @closesquare ps parse_iterators(ps, iters, trivia)
312 +
    if kindof(ps.nt) === Tokens.IF # filter
313 +
        push!(trivia, EXPR(next(ps)))
314 +
        cond = @closer ps :range parse_expression(ps)
315 +
        pushfirst!(iters, cond)
316 +
        iters = EXPR(:filter, iters, trivia)
317 +
        trivia = EXPR[]
318 +
    end
319 +
    if kindof(ps.nt) === Tokens.FOR
320 +
        return EXPR(:flatten, EXPR[EXPR(:generator, EXPR[parse_generator(ps, first); iters], [kw; trivia])], nothing)
293 321
    else
294 -
        push!(ret, ranges)
322 +
        return EXPR(:generator, EXPR[first; iters], EXPR[kw; trivia])
295 323
    end
324 +
end
296 325
297 -
    if typof(ret.args[1]) === Generator || typof(ret.args[1]) === Flatten
298 -
        ret = EXPR(Flatten, EXPR[ret])
299 -
    end
300 326
301 -
    return ret
327 +
function get_appropriate_child_to_expand(x)
328 +
    if headof(x) === :generator && !(headof(x.args[1]) in (:generator, :flatten))
329 +
        return x, x.args[1]
330 +
    elseif headof(x) === :flatten &&  headof(x.args[1]) === :generator && headof(x.args[1].args[1]) === :generator
331 +
        x.args[1], x.args[1].args[1]
332 +
    else
333 +
        get_appropriate_child_to_expand(x.args[1])
334 +
    end
302 335
end
303 336
337 +
function parse_importexport_item(ps, is_colon = false)
338 +
    if kindof(ps.nt) === Tokens.AT_SIGN
339 +
        mname = parse_macroname(next(ps))
340 +
    elseif kindof(ps.nt) === Tokens.LPAREN
341 +
        a = EXPR(:brackets, EXPR[], EXPR[EXPR(next(ps))])
342 +
        push!(a, @closeparen ps parse_expression(ps))
343 +
        pushtotrivia!(a, accept_rparen(ps))
344 +
        a
345 +
    elseif kindof(ps.nt) === Tokens.EX_OR
346 +
        @closer ps :comma parse_expression(ps)
347 +
    elseif !is_colon && isoperator(ps.nt)
348 +
        next(ps)
349 +
        EXPR(:OPERATOR, ps.nt.startbyte - ps.t.startbyte,  1 + ps.t.endbyte - ps.t.startbyte, val(ps.t, ps))
350 +
    elseif VERSION > v"1.3.0-" && isidentifier(ps.nt) && isemptyws(ps.nws) && (kindof(ps.nnt) === Tokens.STRING || kindof(ps.nnt) === Tokens.TRIPLE_STRING)
351 +
        EXPR(:NONSTDIDENTIFIER, EXPR[INSTANCE(next(ps)), INSTANCE(next(ps))])
352 +
        #TODO fix nonstdid handling
353 +
    else
354 +
        INSTANCE(next(ps))
355 +
    end
356 +
end
304 357
"""
305 358
Helper function for parsing import/using statements.
306 359
"""
307 -
function parse_dot_mod(ps::ParseState, is_colon=false)
308 -
    args = EXPR[]
360 +
function parse_dot_mod(ps::ParseState, is_colon = false, allow_as = false)
361 +
    ret = EXPR(EXPR(:OPERATOR, 0, 0, "."), EXPR[], EXPR[])
309 362
310 363
    prevpos = position(ps)
311 364
    while kindof(ps.nt) === Tokens.DOT || kindof(ps.nt) === Tokens.DDOT || kindof(ps.nt) === Tokens.DDDOT
312 -
        d = mOPERATOR(next(ps))
365 +
        d = EXPR(:OPERATOR, next(ps))
313 366
        trailing_ws = d.fullspan - d.span
314 367
        if is_dot(d)
315 -
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
368 +
            push!(ret, EXPR(:OPERATOR, 1 + trailing_ws, 1, "."))
316 369
        elseif is_ddot(d)
317 -
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
318 -
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
370 +
            push!(ret, EXPR(:OPERATOR, 1, 1, "."))
371 +
            push!(ret, EXPR(:OPERATOR, 1 + trailing_ws, 1, "."))
319 372
        elseif is_dddot(d)
320 -
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
321 -
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
322 -
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
373 +
            push!(ret, EXPR(:OPERATOR, 1, 1, "."))
374 +
            push!(ret, EXPR(:OPERATOR, 1, 1, "."))
375 +
            push!(ret, EXPR(:OPERATOR, 1 + trailing_ws, 1, "."))
323 376
        end
324 377
        prevpos = loop_check(ps, prevpos)
325 378
    end
326 379
327 -
    # import/export ..
328 -
    # TODO: Not clear what this is for?
329 -
    # if iscomma(ps.nt) || kindof(ps.ws) == NewLineWS || kindof(ps.nt) === Tokens.ENDMARKER
330 -
    #     if length(args) == 2
331 -
    #         return EXPR[INSTANCE(ps)]
332 -
    #     end
333 -
    # end
334 -
335 380
    prevpos = position(ps)
336 381
    while true
337 -
        if kindof(ps.nt) === Tokens.AT_SIGN
338 -
            at = mPUNCTUATION(next(ps))
339 -
            a = INSTANCE(next(ps))
340 -
            push!(args, EXPR(MacroName, EXPR[at, a]))
341 -
        elseif kindof(ps.nt) === Tokens.LPAREN
342 -
            a = EXPR(InvisBrackets, EXPR[mPUNCTUATION(next(ps))])
343 -
            push!(a, @closeparen ps parse_expression(ps))
344 -
            accept_rparen(ps, a)
345 -
            push!(args, a)
346 -
        elseif kindof(ps.nt) === Tokens.EX_OR
347 -
            a = @closer ps :comma parse_expression(ps)
348 -
            push!(args, a)
349 -
        elseif !is_colon && isoperator(ps.nt)
350 -
            next(ps)
351 -
            push!(args, mOPERATOR(ps.nt.startbyte - ps.t.startbyte,  1 + ps.t.endbyte - ps.t.startbyte, kindof(ps.t), false))
352 -
        elseif VERSION > v"1.3.0-" && isidentifier(ps.nt) && isemptyws(ps.nws) && (kindof(ps.nnt) === Tokens.STRING || kindof(ps.nnt) === Tokens.TRIPLE_STRING)
353 -
            push!(args, EXPR(NONSTDIDENTIFIER, EXPR[INSTANCE(next(ps)), INSTANCE(next(ps))]))
354 -
        else
355 -
            push!(args, INSTANCE(next(ps)))
356 -
        end
382 +
        push!(ret, parse_importexport_item(ps, is_colon))
357 383
358 384
        if kindof(ps.nt) === Tokens.DOT
359 -
            push!(args, mPUNCTUATION(next(ps)))
385 +
            pushtotrivia!(ret, EXPR(next(ps)))
360 386
        elseif isoperator(ps.nt) && (ps.nt.dotop || kindof(ps.nt) === Tokens.DOT)
361 -
            push!(args, mPUNCTUATION(Tokens.DOT, 1, 1))
387 +
            pushtotrivia!(ret, EXPR(:DOT, 1, 1))
362 388
            ps.nt = RawToken(kindof(ps.nt), ps.nt.startpos, ps.nt.endpos, ps.nt.startbyte + 1, ps.nt.endbyte, ps.nt.token_error, false, ps.nt.suffix)
363 389
        else
390 +
            @static if VERSION > v"1.6-"
391 +
                if allow_as && !isnewlinews(ps.ws) && isidentifier(ps.nt) && val(ps.nt, ps) == "as"
392 +
                    as = EXPR(:AS, next(ps))
393 +
                    as_val = parse_importexport_item(ps, is_colon)
394 +
                    ret = EXPR(:as, EXPR[ret, as_val], EXPR[as])
395 +
                end
396 +
            end
364 397
            break
365 398
        end
366 399
        prevpos = loop_check(ps, prevpos)
367 400
    end
368 -
    args
401 +
    ret
369 402
end

@@ -0,0 +1,793 @@
Loading
1 +
module Iterating
2 +
using ..CSTParser: EXPR, headof, hastrivia, isoperator, valof, isstringliteral, is_exor, is_lparen, is_rparen
3 +
4 +
function Base.getindex(x::EXPR, i)
5 +
    try
6 +
        a = _getindex(x, i)
7 +
        if a === nothing
8 +
            error("indexing error for $(x.head) expression at $i")
9 +
        end
10 +
        return a
11 +
    catch err
12 +
        @info typeof(x)
13 +
        @info x
14 +
        error("indexing error for $(x.head) expression at $i")
15 +
    end
16 +
end
17 +
18 +
function _getindex(x::EXPR, i)
19 +
    if headof(x) === :abstract
20 +
        _abstract(x, i)
21 +
    elseif headof(x) === :as
22 +
        odda_event(x, i)
23 +
    elseif headof(x) === :block
24 +
        _block(x, i)
25 +
    elseif headof(x) === :braces
26 +
        _braces(x, i)
27 +
    elseif headof(x) === :brackets
28 +
        oddt_evena(x, i)
29 +
    elseif headof(x) === :call
30 +
        _call(x, i)
31 +
    elseif headof(x) === :comparison || headof(x) === :file
32 +
        x.args[i]
33 +
    elseif headof(x) === :comprehension
34 +
        tat(x, i)
35 +
    elseif headof(x) === :const
36 +
        _const(x, i)
37 +
    elseif headof(x) === :curly
38 +
        _curly(x, i)
39 +
    elseif headof(x) === :do
40 +
        odda_event(x, i)
41 +
    elseif headof(x) === :elseif
42 +
        _elseif(x, i)
43 +
    elseif headof(x) === :errortoken
44 +
        x.args[i]
45 +
    elseif headof(x) === :export
46 +
        oddt_evena(x, i)
47 +
    elseif headof(x) === :filter
48 +
        _filter(x, i)
49 +
    elseif headof(x) === :flatten
50 +
        x.args[1]
51 +
    elseif headof(x) === :for
52 +
        taat(x, i)
53 +
    elseif headof(x) === :function || headof(x) === :macro
54 +
        _function(x, i)
55 +
    elseif headof(x) === :generator
56 +
        odda_event(x, i)
57 +
    elseif headof(x) === :global || headof(x) === :local
58 +
        _global(x, i)
59 +
    elseif headof(x) === :if
60 +
        if isoperator(first(x.trivia)) # ternary op
61 +
            odda_event(x, i)
62 +
        else
63 +
            _if(x, i)
64 +
        end
65 +
    elseif headof(x) === :kw
66 +
        _kw(x, i)
67 +
    elseif headof(x) === :let
68 +
        taat(x, i)
69 +
    elseif headof(x) === :return
70 +
        oddt_evena(x, i)
71 +
    elseif headof(x) === :macrocall
72 +
        _macrocall(x, i)
73 +
    elseif headof(x) === :module
74 +
        _module(x, i)
75 +
    elseif headof(x) === :outer
76 +
        ta(x, i)
77 +
    elseif headof(x) === :parameters
78 +
        if length(x.args) > 1 && headof(x.args[2]) === :parameters
79 +
            if i == length(x)
80 +
                x.args[2]
81 +
            elseif i == 1
82 +
                x.args[1]
83 +
            elseif isodd(i)
84 +
                x.args[div(i + 1, 2) + 1]
85 +
            else
86 +
                x.trivia[div(i, 2)]
87 +
            end
88 +
        else
89 +
            odda_event(x, i)
90 +
        end
91 +
        # if hastrivia(x)
92 +
        #     odda_event(x, i)
93 +
        # else
94 +
        #     x.args[i]
95 +
        # end
96 +
    elseif headof(x) === :primitive
97 +
        _primitive(x, i)
98 +
    elseif headof(x) === :quote
99 +
        _quote(x, i)
100 +
    elseif headof(x) === :quotenode
101 +
        _quotenode(x, i)
102 +
    elseif headof(x) === :ref
103 +
        _call(x, i)
104 +
    elseif headof(x) === :row
105 +
        x.args[i]
106 +
    elseif headof(x) === :string
107 +
        _string(x, i)
108 +
    elseif headof(x) === :struct
109 +
        _struct(x, i)
110 +
    elseif headof(x) === :toplevel
111 +
        x.args[i]
112 +
    elseif headof(x) === :try
113 +
        _try(x, i)
114 +
    elseif headof(x) === :tuple
115 +
        _tuple(x, i)
116 +
    elseif headof(x) === :typed_comprehension
117 +
        odda_event(x, i)
118 +
    elseif headof(x) === :typed_vcat || headof(x) === :typed_hcat
119 +
        _typed_vcat(x, i)
120 +
    elseif headof(x) === :using || headof(x) === :import
121 +
        _using(x, i)
122 +
    elseif headof(x) === :vcat || headof(x) === :hcat || headof(x) === :bracescat
123 +
        _vcat(x, i)
124 +
    elseif headof(x) === :vect
125 +
        _vect(x, i)
126 +
    elseif headof(x) === :where
127 +
        _where(x, i)
128 +
    elseif headof(x) === :while
129 +
        taat(x, i)
130 +
    elseif isoperator(headof(x))
131 +
        if valof(headof(x)) == ":"
132 +
            _colon_in_using(x, i)
133 +
        elseif valof(headof(x)) == "."
134 +
            _dot(x, i)
135 +
        elseif valof(headof(x)) == "=" && hastrivia(x) 
136 +
            # a loop that has been lowered from in/∈ to = at parse time
137 +
            if i == 1
138 +
                x.args[1]
139 +
            elseif i == 2
140 +
                x.trivia[1]
141 +
            elseif i == 3
142 +
                x.args[2]
143 +
            end
144 +
        elseif valof(headof(x)) == "->" && headof(x).fullspan == 0
145 +
            # anon function as used in a 'do' block
146 +
            if i == 1
147 +
                x.args[1]
148 +
            else 
149 +
                x.args[2]
150 +
            end
151 +
        elseif !hastrivia(x)
152 +
            if i == 1
153 +
                x.args[1]
154 +
            elseif i == 2
155 +
                x.head
156 +
            elseif i == 3
157 +
                x.args[2]
158 +
            end
159 +
        else
160 +
            if i == 1
161 +
                x.head
162 +
            elseif i == 2
163 +
                x.trivia[1]
164 +
            elseif i == length(x)
165 +
                last(x.trivia)
166 +
            elseif isodd(i)
167 +
                x.args[div(i+1, 2) - 1]
168 +
            else
169 +
                x.trivia[div(i, 2)]
170 +
            end
171 +
        end
172 +
    else 
173 +
        error("Indexing $(Expr(x)) at $i")
174 +
    end
175 +
end
176 +
Base.iterate(x::EXPR) = length(x) == 0 ? nothing : (x[1], 1)
177 +
Base.iterate(x::EXPR, s) = s < length(x) ? (x[s + 1], s + 1) : nothing
178 +
Base.firstindex(x::EXPR) = 1
179 +
Base.lastindex(x::EXPR) = x.args === nothing ? 0 : length(x)
180 +
181 +
# Base.setindex!(x::EXPR, val, i) = Base.setindex!(x.args, val, i)
182 +
# Base.first(x::EXPR) = x.args === nothing ? nothing : first(x.args)
183 +
# Base.last(x::EXPR) = x.args === nothing ? nothing : last(x.args)
184 +
185 +
186 +
function ta(x, i)
187 +
    if i == 1
188 +
        x.trivia[1]
189 +
    elseif i == 2
190 +
        x.args[1]
191 +
    end
192 +
end
193 +
function tat(x, i)
194 +
    if i == 1
195 +
        x.trivia[1]
196 +
    elseif i == 2
197 +
        x.args[1]
198 +
    elseif i == 3
199 +
        x.trivia[2]
200 +
    end
201 +
end
202 +
203 +
function taat(x, i)
204 +
    if i == 1
205 +
        x.trivia[1]
206 +
    elseif i == 2
207 +
        x.args[1]
208 +
    elseif i == 3
209 +
        x.args[2]
210 +
    elseif i == 4
211 +
        x.trivia[2]
212 +
    end
213 +
end
214 +
215 +
216 +
function oddt_evena(x, i)
217 +
    if isodd(i)
218 +
        x.trivia[div(i + 1, 2)]
219 +
    else
220 +
        x.args[div(i, 2)]
221 +
    end
222 +
end
223 +
224 +
function odda_event(x, i)
225 +
    if isodd(i)
226 +
        x.args[div(i + 1, 2)]
227 +
    else
228 +
        x.trivia[div(i, 2)]
229 +
    end
230 +
end
231 +
232 +
233 +
function _const(x, i) 
234 +
    if length(x.trivia) === 1
235 +
        ta(x, i)
236 +
    elseif length(x.trivia) === 2
237 +
        #global const
238 +
        if i < 3
239 +
            x.trivia[i]
240 +
        elseif i == 3
241 +
            x.args[1]
242 +
        end
243 +
    end
244 +
end