1
"""
2
closer(ps::ParseState)
3

4
A magical function determining whether the parsing of an expression should continue or
5
stop.
6
"""
7
function closer(ps::ParseState)
8 23
    kindof(ps.nt) === Tokens.ENDMARKER ||
9
    (ps.closer.newline && kindof(ps.ws) == NewLineWS && !iscomma(ps.t)) ||
10
    (ps.closer.semicolon && kindof(ps.ws) == SemiColonWS) ||
11
    (isoperator(ps.nt) && precedence(ps.nt) <= ps.closer.precedence) ||
12
    (kindof(ps.nt) === Tokens.WHERE && ps.closer.precedence == LazyAndOp) ||
13
    (ps.closer.inwhere && kindof(ps.nt) === Tokens.WHERE) ||
14
    (ps.closer.inwhere && ps.closer.ws && kindof(ps.t) === Tokens.RPAREN && isoperator(ps.nt) && precedence(ps.nt) < DeclarationOp) ||
15
    (ps.closer.precedence > WhereOp && (
16
        (kindof(ps.nt) === Tokens.LPAREN && !(ps.t.kind === Tokens.EX_OR)) ||
17
        kindof(ps.nt) === Tokens.LBRACE ||
18
        kindof(ps.nt) === Tokens.LSQUARE ||
19
        (kindof(ps.nt) === Tokens.STRING && isemptyws(ps.ws)) ||
20
        ((kindof(ps.nt) === Tokens.RPAREN || kindof(ps.nt) === Tokens.RSQUARE) && isidentifier(ps.nt))
21
    )) ||
22
    (iscomma(ps.nt) && ps.closer.precedence > 0) ||
23
    (ps.closer.comma && iscomma(ps.nt)) ||
24
    (ps.closer.tuple && (iscomma(ps.nt) || isassignment(ps.nt))) ||
25
    (kindof(ps.nt) === Tokens.FOR && ps.closer.precedence > -1) ||
26
    (ps.closer.block && kindof(ps.nt) === Tokens.END) ||
27
    (ps.closer.paren && kindof(ps.nt) === Tokens.RPAREN) ||
28
    (ps.closer.brace && kindof(ps.nt) === Tokens.RBRACE) ||
29
    (ps.closer.square && kindof(ps.nt) === Tokens.RSQUARE) ||
30
    (@static VERSION < v"1.4" ? false : ((ps.closer.insquare || ps.closer.inmacro) && kindof(ps.nt) === Tokens.APPROX && kindof(ps.nws) == EmptyWS)) ||
31
    kindof(ps.nt) === Tokens.ELSEIF ||
32
    kindof(ps.nt) === Tokens.ELSE ||
33
    kindof(ps.nt) === Tokens.CATCH ||
34
    kindof(ps.nt) === Tokens.FINALLY ||
35
    (ps.closer.ifop && isoperator(ps.nt) && (precedence(ps.nt) <= 0 || kindof(ps.nt) === Tokens.COLON)) ||
36
    (ps.closer.range && (kindof(ps.nt) === Tokens.FOR || iscomma(ps.nt) || kindof(ps.nt) === Tokens.IF)) ||
37
    (ps.closer.ws && !isemptyws(ps.ws) &&
38
        !iscomma(ps.nt) &&
39
        !iscomma(ps.t) &&
40
        !(!ps.closer.inmacro && kindof(ps.nt) === Tokens.FOR) &&
41
        !(kindof(ps.nt) === Tokens.DO) &&
42
        !(
43
            (isbinaryop(ps.nt) && !(ps.closer.wsop && isemptyws(ps.nws) && isunaryop(ps.nt) && precedence(ps.nt) > 7)) ||
44
            (isunaryop(ps.t) && kindof(ps.ws) == WS && ps.lt.kind !== CSTParser.Tokens.COLON)
45
        )) ||
46
    (ps.closer.unary && (kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT, Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE) && isidentifier(ps.nt)))
47
end
48

49
"""
50
    @closer ps rule body
51

52
Continues parsing closing on `rule`.
53
"""
54
macro closer(ps, opt, body)
55 21
    quote
56 23
        local tmp1 = getfield($(esc(ps)).closer, $opt)
57 23
        setfield!($(esc(ps)).closer, $opt, true)
58 23
        out = $(esc(body))
59 23
        setfield!($(esc(ps)).closer, $opt, tmp1)
60 0
        out
61
    end
62
end
63

64
"""
65
    @nocloser ps rule body
66

67
Continues parsing not closing on `rule`.
68
"""
69
macro nocloser(ps, opt, body)
70 21
    quote
71 23
        local tmp1 = getfield($(esc(ps)).closer, $opt)
72 23
        setfield!($(esc(ps)).closer, $opt, false)
73 23
        out = $(esc(body))
74 23
        setfield!($(esc(ps)).closer, $opt, tmp1)
75 23
        out
76
    end
77
end
78

79
macro closeparen(ps, body)
80 21
    quote
81 21
        local tmp1 = $(esc(ps)).closer.paren
82 21
        $(esc(ps)).closer.paren = true
83 23
        out = $(esc(body))
84 21
        $(esc(ps)).closer.paren = tmp1
85 0
        out
86
    end
87
end
88

89
macro closesquare(ps, body)
90 21
    quote
91 21
        local tmp1 = $(esc(ps)).closer.square
92 21
        $(esc(ps)).closer.square = true
93 23
        out = $(esc(body))
94 21
        $(esc(ps)).closer.square = tmp1
95 0
        out
96
    end
97
end
98
macro closebrace(ps, body)
99 21
    quote
100 21
        local tmp1 = $(esc(ps)).closer.brace
101 21
        $(esc(ps)).closer.brace = true
102 23
        out = $(esc(body))
103 21
        $(esc(ps)).closer.brace = tmp1
104 0
        out
105
    end
106
end
107

108
"""
109
    @precedence ps prec body
110

111
Continues parsing binary operators until it hits a more loosely binding
112
operator (with precdence lower than `prec`).
113
"""
114
macro precedence(ps, prec, body)
115 21
    quote
116 23
        local tmp1 = $(esc(ps)).closer.precedence
117 23
        $(esc(ps)).closer.precedence = $(esc(prec))
118 23
        out = $(esc(body))
119 21
        $(esc(ps)).closer.precedence = tmp1
120 0
        out
121
    end
122
end
123

124

125
# Closer_TMP and ancillary functions help reduce code generation
126
struct Closer_TMP
127 23
    newline::Bool
128
    semicolon::Bool
129
    inmacro::Bool
130
    tuple::Bool
131
    comma::Bool
132
    insquare::Bool
133
    range::Bool
134
    ifop::Bool
135
    ws::Bool
136
    wsop::Bool
137
    unary::Bool
138
    precedence::Int
139
end
140

141
@noinline function create_tmp(c::Closer)
142 23
    Closer_TMP(c.newline,
143
        c.semicolon,
144
        c.inmacro,
145
        c.tuple,
146
        c.comma,
147
        c.insquare,
148
        c.range,
149
        c.ifop,
150
        c.ws,
151
        c.wsop,
152
        c.unary,
153
        c.precedence)
154
end
155

156
@noinline function update_from_tmp!(c::Closer, tmp::Closer_TMP)
157 23
    c.newline = tmp.newline
158 21
    c.semicolon = tmp.semicolon
159 21
    c.inmacro = tmp.inmacro
160 21
    c.tuple = tmp.tuple
161 21
    c.comma = tmp.comma
162 21
    c.insquare = tmp.insquare
163 21
    c.range = tmp.range
164 21
    c.ifop = tmp.ifop
165 21
    c.ws = tmp.ws
166 21
    c.wsop = tmp.wsop
167 21
    c.unary = tmp.unary
168 23
    c.precedence = tmp.precedence
169
end
170

171

172
@noinline function update_to_default!(c::Closer)
173 23
    c.newline = true
174 21
    c.semicolon = true
175 21
    c.inmacro = false
176 21
    c.tuple = false
177 21
    c.comma = false
178 21
    c.insquare = false
179 21
    c.range = false
180 21
    c.ifop = false
181 21
    c.ws = false
182 21
    c.wsop = false
183 21
    c.unary = false
184 23
    c.precedence = -1
185
end
186

187

188
"""
189
    @default ps body
190

191
Parses the next expression using default closure rules.
192
"""
193
macro default(ps, body)
194 21
    quote
195 23
        TMP = create_tmp($(esc(ps)).closer)
196 23
        update_to_default!($(esc(ps)).closer)
197 23
        out = $(esc(body))
198 23
        update_from_tmp!($(esc(ps)).closer, TMP)
199 0
        out
200
    end
201
end
202

203

204

205 23
isidentifier(x::EXPR) = typof(x) === IDENTIFIER || typof(x) === NONSTDIDENTIFIER
206

207 23
isunarycall(x::EXPR) = typof(x) === UnaryOpCall
208 23
isbinarycall(x::EXPR) = typof(x) === BinaryOpCall
209 23
iswherecall(x::EXPR) = typof(x) === WhereOpCall
210 23
isdeclaration(x::EXPR) = isbinarycall(x) && is_decl(x[2])
211 23
isinterpolant(x::EXPR) = isunarycall(x) && is_exor(x[1])
212 23
istuple(x::EXPR) = typof(x) === TupleH
213 23
is_either_id_op_interp(x::EXPR) = isidentifier(x) || isoperator(x) || isinterpolant(x)
214 23
is_splat(x::EXPR) = isunarycall(x) && is_dddot(x[2])
215

216

217 23
isliteral(x::EXPR) = typof(x) === LITERAL
218 23
iskw(x::EXPR) = typof(x) === KEYWORD # TODO: should change to `iskeyword`
219 23
ispunctuation(x::EXPR) = typof(x) === PUNCTUATION
220

221 0
isstring(x) = typof(x) === StringH || (isliteral(x) && (kindof(x) === Tokens.STRING || kindof(x) === Tokens.TRIPLE_STRING))
222 0
is_integer(x) = isliteral(x) && kindof(x) === Tokens.INTEGER
223 0
is_float(x) = isliteral(x) && kindof(x) === Tokens.FLOAT
224 23
is_number(x) = isliteral(x) && (kindof(x) === Tokens.INTEGER || kindof(x) === Tokens.FLOAT)
225 23
is_nothing(x) = isliteral(x) && kindof(x) === Tokens.NOTHING
226

227 23
isajuxtaposition(ps::ParseState, ret::EXPR) = ((is_number(ret) && (isidentifier(ps.nt) || kindof(ps.nt) === Tokens.LPAREN || kindof(ps.nt) === Tokens.CMD || kindof(ps.nt) === Tokens.STRING || kindof(ps.nt) === Tokens.TRIPLE_STRING)) ||
228
        ((typof(ret) === UnaryOpCall && is_prime(ret.args[2]) && isidentifier(ps.nt)) ||
229
        ((kindof(ps.t) === Tokens.RPAREN || kindof(ps.t) === Tokens.RSQUARE) && (isidentifier(ps.nt) || kindof(ps.nt) === Tokens.CMD)) ||
230
        ((kindof(ps.t) === Tokens.STRING || kindof(ps.t) === Tokens.TRIPLE_STRING) && (kindof(ps.nt) === Tokens.STRING || kindof(ps.nt) === Tokens.TRIPLE_STRING)))) || ((kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT) || kindof(ps.t) in (Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE)) && isidentifier(ps.nt))
231

232
"""
233
    has_error(ps::ParseState)
234
    has_error(x::EXPR)
235

236
Determine whether a parsing error occured while processing text with the given
237
`ParseState`, or exists as a (sub) expression of `x`.
238
"""
239
function has_error(x::EXPR)
240 23
    return typof(x) == ErrorToken || (x.args !== nothing && any(has_error, x.args))
241
end
242 21
has_error(ps::ParseState) = ps.errored
243

244
# When using the FancyDiagnostics package, Meta.parse is the
245
# same as CSTParser.parse. Manually call the flisp parser here
246
# to make sure we test what we want, even when people load the
247
# FancyDiagnostics package.
248
function flisp_parse(str::AbstractString, pos::Int; greedy::Bool=true, raise::Bool=true)
249 23
    if VERSION < v"1.6-DEV"
250 0
        bstr = String(str)
251 23
        ex, pos = ccall(:jl_parse_string, Any,
252
                        (Ptr{UInt8}, Csize_t, Int32, Int32),
253
                        bstr, sizeof(bstr), pos - 1, greedy ? 1 : 0)
254
    else
255 0
        filename = "none"
256 0
        rule = greedy ? :statement : :atom
257 0
        ex, pos = Core.Compiler.fl_parse(str, filename, pos - 1, rule)
258
    end
259 23
    if raise && isa(ex, Expr) && ex.head === :error
260 0
        throw(Meta.ParseError(ex.args[1]))
261
    end
262 23
    if ex === ()
263 0
        raise && throw(Meta.ParseError("end of input"))
264 0
        ex = Expr(:error, "end of input")
265
    end
266
    # pos is zero-based byte offset
267 23
    return ex, pos + 1
268
end
269

270
function flisp_parse(str::AbstractString; raise::Bool=true)
271 23
    ex, pos = flisp_parse(str, 1, greedy=true, raise=raise)
272 23
    if isa(ex, Expr) && ex.head === :error
273 0
        return ex
274
    end
275 23
    if !(pos > ncodeunits(str))
276 0
        raise && throw(Meta.ParseError("extra token after end of expression"))
277 0
        return Expr(:error, "extra token after end of expression")
278
    end
279 23
    return ex
280
end
281

282
function flisp_parse(stream::IO; greedy::Bool=true, raise::Bool=true)
283 23
    pos = position(stream)
284 23
    ex, Δ = flisp_parse(read(stream, String), 1, greedy=greedy, raise=raise)
285 23
    seek(stream, pos + Δ - 1)
286 23
    return ex
287
end
288

289
using Base.Meta
290

291
function norm_ast(a::Any)
292 23
    if isa(a, Expr)
293 23
        for (i, arg) in enumerate(a.args)
294 23
            a.args[i] = norm_ast(arg)
295
        end
296 23
        if a.head === :line
297 0
            return Expr(:line, a.args[1], :none)
298
        end
299 23
        if a.head === :macrocall
300 21
            fa = a.args[1]
301 23
            if fa === Symbol("@int128_str")
302 0
                return Base.parse(Int128, a.args[3])
303 23
            elseif fa === Symbol("@uint128_str")
304 2
                return Base.parse(UInt128, a.args[3])
305 23
            elseif fa === Symbol("@bigint_str")
306 0
                return  Base.parse(BigInt, a.args[3])
307 23
            elseif fa == Symbol("@big_str")
308 0
                s = a.args[3]
309 0
                n = tryparse(BigInt, s)
310 0
                if !(n === nothing)
311 0
                    return (n)
312
                end
313 0
                n = tryparse(BigFloat, s)
314 0
                if !(n === nothing)
315 0
                    return isnan((n)) ? :NaN : (n)
316
                end
317 0
                return s
318
            end
319 23
        elseif length(a.args) >= 2 && isexpr(a, :call) && a.args[1] == :- && isa(a.args[2], Number)
320 23
            return -a.args[2]
321
        end
322 23
        return a
323 4
    elseif isa(a, QuoteNode)
324 23
        return Expr(:quote, norm_ast(a.value))
325 23
    elseif isa(a, AbstractFloat) && isnan(a)
326 0
        return :NaN
327
    end
328 23
    return a
329
end
330

331
function flisp_parsefile(str, display=true)
332 23
    io = IOBuffer(str)
333 4
    failed = false
334 21
    x1 = Expr(:file)
335 23
    try
336 23
        while !eof(io)
337 23
            push!(x1.args, flisp_parse(io))
338
        end
339
    catch er
340 0
        isa(er, InterruptException) && rethrow(er)
341 0
        if display
342 0
            Base.showerror(stdout, er, catch_backtrace())
343 0
            println()
344
        end
345 0
        return x1, true
346
    end
347 23
    if length(x1.args) > 0  && x1.args[end] === nothing
348 23
        pop!(x1.args)
349
    end
350 23
    x1 = norm_ast(x1)
351 23
    remlineinfo!(x1)
352 23
    return x1, false
353
end
354

355
function cst_parsefile(str)
356 23
    x, ps = CSTParser.parse(ParseState(str), true)
357 23
    sp = check_span(x)
358
    # remove leading/trailing nothings
359 23
    if length(x.args) > 0 && is_nothing(x.args[1])
360 23
        popfirst!(x.args)
361
    end
362 23
    if length(x.args) > 0 && is_nothing(x.args[end])
363 0
        pop!(x.args)
364
    end
365 23
    x0 = norm_ast(Expr(x))
366 23
    x0, has_error(ps), sp
367
end
368

369
function check_file(file, ret, neq)
370 23
    str = read(file, String)
371 23
    x0, cstfailed, sp = cst_parsefile(str)
372 23
    x1, flispfailed = flisp_parsefile(str)
373

374 23
    print("\r                             ")
375 23
    if !isempty(sp)
376 0
        printstyled(file, color=:blue)
377 0
        @show sp
378 0
        println()
379 0
        push!(ret, (file, :span))
380
    end
381 23
    if cstfailed
382 0
        printstyled(file, color=:yellow)
383 0
        println()
384 0
        push!(ret, (file, :errored))
385 23
    elseif !(x0 == x1)
386 0
        cumfail = 0
387 2
        printstyled(file, color=:green)
388 2
        println()
389 2
        c0, c1 = CSTParser.compare(x0, x1)
390 2
        printstyled(string("    ", c0), bold=true, color=:ligth_red)
391 2
        println()
392 2
        printstyled(string("    ", c1), bold=true, color=:light_green)
393 2
        println()
394 23
        push!(ret, (file, :noteq))
395
    end
396
end
397

398
function check_base(dir=dirname(Base.find_source_file("essentials.jl")), display=false)
399 23
    N = 0
400 4
    neq = 0
401 4
    err = 0
402 4
    aerr = 0
403 4
    fail = 0
404 4
    bfail = 0
405 21
    ret = []
406 23
    oldstderr = stderr
407 21
    redirect_stderr()
408 23
    for (rp, d, files) in walkdir(dir)
409 23
        for f in files
410 23
            file = joinpath(rp, f)
411 23
            if endswith(file, ".jl")
412 21
                N += 1
413 23
                try
414 23
                    print("\r", rpad(string(N), 5), rpad(string(round(fail / N * 100, sigdigits=3)), 8), rpad(string(round(err / N * 100, sigdigits=3)), 8), rpad(string(round(neq / N * 100, sigdigits=3)), 8))
415

416 23
                    check_file(file, ret, neq)
417
                catch er
418 0
                    isa(er, InterruptException) && rethrow(er)
419 0
                    if display
420 0
                        Base.showerror(stdout, er, catch_backtrace())
421 0
                        println()
422
                    end
423 0
                    fail += 1
424 0
                    printstyled(file, color=:red)
425 0
                    println()
426 23
                    push!(ret, (file, :failed))
427
                end
428
            end
429
        end
430
    end
431 23
    redirect_stderr(oldstderr)
432 23
    if bfail + fail + err + neq > 0
433 0
        println("\r$N files")
434 0
        printstyled("failed", color=:red)
435 0
        println(" : $fail    $(100 * fail / N)%")
436 0
        printstyled("errored", color=:yellow)
437 0
        println(" : $err     $(100 * err / N)%")
438 0
        printstyled("not eq.", color=:green)
439 0
        println(" : $neq    $(100 * neq / N)%", "  -  $aerr     $(100 * aerr / N)%")
440 0
        printstyled("base failed", color=:magenta)
441 0
        println(" : $bfail    $(100 * bfail / N)%")
442 0
        println()
443
    else
444 23
        println("\r")
445
    end
446 23
    ret
447
end
448

449
"""
450
    compare(x,y)
451

452
Recursively checks whether two Base.Expr are the same. Returns unequal sub-
453
expressions.
454
"""
455 2
compare(x, y) = x == y ? true : (x, y)
456

457
function compare(x::Expr, y::Expr)
458 2
    if x == y
459 2
        return true
460
    else
461 2
        if x.head != y.head
462 0
            return (x, y)
463
        end
464 2
        if length(x.args) != length(y.args)
465 0
            return (x.args, y.args)
466
        end
467 2
        for i = 1:length(x.args)
468 2
            t = compare(x.args[i], y.args[i])
469 2
            if t != true
470 2
                return t
471
            end
472
        end
473
    end
474
end
475

476
"""
477
check_span(x, neq = [])
478

479
Recursively checks whether the span of an expression equals the sum of the span
480
of its components. Returns a vector of failing expressions.
481
"""
482
function check_span(x::EXPR, neq=[])
483 23
    (ispunctuation(x) || isidentifier(x) || iskw(x) || isoperator(x) || isliteral(x) || typof(x) == StringH) && return neq
484

485 4
    s = 0
486 23
    for a in x.args
487 23
        check_span(a, neq)
488 23
        s += a.fullspan
489
    end
490 23
    if length(x.args) > 0 && s != x.fullspan
491 0
        push!(neq, x)
492
    end
493 23
    neq
494
end
495

496 0
function speed_test()
497 0
    dir = dirname(Base.find_source_file("essentials.jl"))
498 0
    println("speed test : ", @timed(for i = 1:5
499 0
        parse(read(joinpath(dir, "essentials.jl"), String), true);
500 0
        parse(read(joinpath(dir, "abstractarray.jl"), String), true);
501
    end)[2])
502
end
503

504
"""
505
    str_value(x)
506

507
Attempt to get a string representation of a nodeless expression.
508
"""
509
function str_value(x)
510 23
    if typof(x) === IDENTIFIER || typof(x) === LITERAL
511 0
        return valof(x)
512 23
    elseif isidentifier(x)
513 0
        valof(x.args[2])
514 23
    elseif typof(x) === OPERATOR || typof(x) === MacroName
515 23
        return string(Expr(x))
516
    else
517 0
        return ""
518
    end
519
end
520

521 23
_unescape_string(s::AbstractString) = sprint(_unescape_string, s, sizehint=lastindex(s))
522
function _unescape_string(io, s::AbstractString)
523 23
    a = Iterators.Stateful(s)
524 23
    for c in a
525 23
        if !isempty(a) && c == '\\'
526 21
            c = popfirst!(a)
527 23
            if c == 'x' || c == 'u' || c == 'U'
528 0
                n = k = 0
529 23
                m = c == 'x' ? 2 :
530
                    c == 'u' ? 4 : 8
531 23
                while (k += 1) <= m && !isempty(a)
532 21
                    nc = Base.peek(a)
533 23
                    n = '0' <= nc <= '9' ? n << 4 + nc - '0' :
534
                        'a' <= nc <= 'f' ? n << 4 + nc - 'a' + 10 :
535
                        'A' <= nc <= 'F' ? n << 4 + nc - 'A' + 10 : break
536 23
                    popfirst!(a)
537
                end
538 23
                if k == 1
539
                    # throw(ArgumentError("invalid $(m == 2 ? "hex (\\x)" :
540
                    #                         "unicode (\\u)") escape sequence used in $(repr(s))"))
541
                    # push error to ParseState?
542 0
                    n = 0
543
                end
544 23
                if m == 2 # \x escape sequence
545 23
                    write(io, UInt8(n))
546
                else
547 23
                    print(io, Char(n))
548
                end
549 23
            elseif '0' <= c <= '7'
550 0
                k = 1
551 23
                n = c - '0'
552 23
                while (k += 1) <= 3 && !isempty(a)
553 21
                    c  = Base.peek(a)
554 23
                    n = ('0' <= c <= '7') ? n << 3 + c - '0' : break
555 23
                    popfirst!(a)
556
                end
557 23
                if n > 255
558
                    # throw(ArgumentError("octal escape sequence out of range"))
559
                    # push error to ParseState?
560 0
                    n = 255
561
                end
562 23
                write(io, UInt8(n))
563
            else
564 23
                print(io, c == 'a' ? '\a' :
565
                          c == 'b' ? '\b' :
566
                          c == 't' ? '\t' :
567
                          c == 'n' ? '\n' :
568
                          c == 'v' ? '\v' :
569
                          c == 'f' ? '\f' :
570
                          c == 'r' ? '\r' :
571
                          c == 'e' ? '\e' : c)
572
            end
573
        else
574 23
            print(io, c)
575
        end
576
    end
577
end
578

579

580
function valid_escaped_seq(s::AbstractString)
581 23
    a = Iterators.Stateful(s)
582 23
    for c in a
583 23
        if !isempty(a) && c == '\\'
584 21
            c = popfirst!(a)
585 23
            if c == 'x' || c == 'u' || c == 'U'
586 0
                n = k = 0
587 23
                m = c == 'x' ? 2 :
588
                    c == 'u' ? 4 : 8
589 23
                while (k += 1) <= m && !isempty(a)
590 21
                    nc = Base.peek(a)
591 23
                    n = '0' <= nc <= '9' ? n << 4 + (nc - '0') :
592
                        'a' <= nc <= 'f' ? n << 4 + (nc - 'a' + 10) :
593
                        'A' <= nc <= 'F' ? n << 4 + (nc - 'A' + 10) : break
594 23
                    popfirst!(a)
595
                end
596 23
                if k == 1 || n > 0x10ffff
597 0
                    u = m == 4 ? 'u' : 'U'
598 0
                    return false
599
                end
600 23
            elseif '0' <= c <= '7'
601 0
                k = 1
602 23
                n = c - '0'
603 23
                while (k += 1) <= 3 && !isempty(a)
604 0
                    c = Base.peek(a)
605 0
                    n = ('0' <= c <= '7') ? n << 3 + c - '0' : break
606 0
                    popfirst!(a)
607
                end
608 23
                if n > 255
609 0
                    return false
610
                end
611
            else
612 23
                c == 'a' ||
613
                c == 'b' ||
614
                c == 't' ||
615
                c == 'n' ||
616
                c == 'v' ||
617
                c == 'f' ||
618
                c == 'r' ||
619
                c == 'e' ||
620
                c == '\\' ||
621
                c == '"' ||
622
                c == '\'' ||
623
                return false
624
            end
625
        end
626
    end
627 23
    return true
628
end
629

630
"""
631
    is_getfield(x::EXPR)
632

633
Is this an expression of the form `a.b`.
634
"""
635 23
is_getfield(x::EXPR) = isbinarycall(x) && length(x) == 3 && kindof(x[2]) === Tokens.DOT
636

637
"""
638
    disallowednumberjuxt(ret::EXPR)
639

640
Does this number literal end in a decimal and so cannot precede a paren for
641
implicit multiplication?
642
"""
643 23
disallowednumberjuxt(ret::EXPR) = is_number(ret) && last(valof(ret)) == '.'
644

645

646 23
nexttokenstartsdocstring(ps::ParseState) = isidentifier(ps.nt) && val(ps.nt, ps) == "doc" && (kindof(ps.nnt) === Tokens.STRING || kindof(ps.nnt) === Tokens.TRIPLE_STRING)
647

648
"""
649
    is_wrapped_assignment(x::EXPR)
650
    
651
Is `x` an assignment expression, ignoring any surrounding parentheses.
652
"""
653 23
is_wrapped_assignment(x::EXPR) = is_assignment(x) || (isbracketed(x) && is_wrapped_assignment(x.args[2]))
654

655
"""
656
    is_range(x::EXPR)
657

658
Is `x` a valid iterator for use in `for` loops or generators?
659
"""
660 23
is_range(x::EXPR) = isbinarycall(x) && (is_eq(x.args[2]) || is_in(x.args[2]) || is_elof(x.args[2]))
661

662
"""
663
    _do_kw_convert(ps::ParseState, a::EXPR)
664

665
Should `a` be converted to a keyword-argument expression?
666
"""
667 23
_do_kw_convert(ps::ParseState, a::EXPR) = !ps.closer.brace && is_assignment(a)
668

669
"""
670
    _kw_convert(ps::ParseState, a::EXPR)
671

672
Converted an assignment expression to a keyword-argument expression.
673
"""
674 23
_kw_convert(a::EXPR) = EXPR(Kw, EXPR[a.args[1], a.args[2], a.args[3]], a.fullspan, a.span)
675

676
"""
677
    convertsigtotuple(sig::EXPR)
678

679
When parsing a function or macro signature, should it be converted to a tuple?
680
"""
681 23
convertsigtotuple(sig::EXPR) = isbracketed(sig) && !(istuple(sig.args[2]) || (typof(sig.args[2]) === Block) || is_splat(sig.args[2]))
682

683
"""
684
    docable(head)
685

686
When parsing a block of expressions, can documentation be attached? Prefixed docs at the
687
top-level are handled within `parse(ps::ParseState, cont = false)`.
688
"""
689 23
docable(head) = head === Begin || head === ModuleH || head === BareModule || head === Quote
690

691

692 23
should_negate_number_literal(ps::ParseState, op::EXPR) = (is_plus(op) || is_minus(op)) && (kindof(ps.nt) === Tokens.INTEGER || kindof(ps.nt) === Tokens.FLOAT) && isemptyws(ps.ws) && kindof(ps.nnt) != Tokens.CIRCUMFLEX_ACCENT
693

694 23
isbracketed(x::EXPR) = typof(x) === InvisBrackets # Assumption that x has 3 args, doesn't need checking?
695

696 23
unwrapbracket(x::EXPR) = isbracketed(x) ? unwrapbracket(x[2]) : x
697

698 23
isbeginorblock(x::EXPR) = typof(x) === Begin || typof(unwrapbracket(x)) == Block
699

700
"""
701
    can_become_comparison(x::EXPR)
702

703
Is `x` a binary comparison call (e.g. `a < b`) that can be extended to include more
704
arguments?
705
"""
706 23
can_become_comparison(x::EXPR) = isbinarycall(x) && (precedence(x.args[2]) == ComparisonOp || is_issubt(x.args[2]) || is_issupt(x.args[2]))
707

708
"""
709
    can_become_chain(x::EXPR, op::EXPR)
710

711
Is `x` a binary call for `+` or `*` that can be extended to include more
712
arguments?
713
"""
714 23
can_become_chain(x::EXPR, op::EXPR) = isbinarycall(x) && (is_star(op) || is_plus(op)) && kindof(op) == kindof(x.args[2]) && !x.args[2].dot && x.args[2].span > 0
715

716
function loop_check(ps, prevpos)
717 23
    if position(ps) <= prevpos
718 0
        throw(CSTInfiniteLoop("Infinite loop at $ps"))
719
    else
720 23
        position(ps)
721
    end
722
end

Read our documentation on viewing source code .

Loading