Commit ⋅ julia-vscode/CSTParser.jl

    # (len < 45 || (len == 45 && s <= "0o3777777777777777777777777777777777777777777")) && return Base.parse(UInt128, s)
    # return Base.parse(BigInt, s)
    (len < 45 || (len == 45 && s <= "0o3777777777777777777777777777777777777777777")) && return Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), nothing, s)
    return flisp_parse(s)
    return Meta.parse(s)
end

function _literal_expr(x)
    if kindof(x) === Tokens.TRUE
    if headof(x) === :TRUE
        return true
    elseif kindof(x) === Tokens.FALSE
    elseif headof(x) === :FALSE
        return false
    elseif is_nothing(x)
        return nothing
    elseif kindof(x) === Tokens.INTEGER || kindof(x) === Tokens.BIN_INT || kindof(x) === Tokens.HEX_INT || kindof(x) === Tokens.OCT_INT
    elseif headof(x) === :INTEGER || headof(x) === :BININT || headof(x) === :HEXINT || headof(x) === :OCTINT
        return Expr_int(x)
    elseif kindof(x) === Tokens.FLOAT
    elseif isfloat(x)
        return Expr_float(x)
    elseif kindof(x) === Tokens.CHAR
    elseif ischar(x)
        return Expr_char(x)
    elseif kindof(x) === Tokens.MACRO
    elseif headof(x) === :MACRO
        return Symbol(valof(x))
    elseif kindof(x) === Tokens.STRING
    elseif headof(x) === :STRING || headof(x) === :TRIPLESTRING
        return valof(x)
    elseif kindof(x) === Tokens.TRIPLE_STRING
        return valof(x)
    elseif kindof(x) === Tokens.CMD
    elseif headof(x) === :CMD
        return Expr_cmd(x)
    elseif kindof(x) === Tokens.TRIPLE_CMD
    elseif headof(x) === :TRIPLECMD
        return Expr_tcmd(x)
    end
end

@@ -95,7 +93,7 @@

    is_oct && return sized_uint_oct_literal(val)
    is_bin && return sized_uint_literal(val, 1)
    # sizeof(val) <= sizeof(TYPEMAX_INT64_STR) && return Base.parse(Int64, val)
    return flisp_parse(val)
    return Meta.parse(val)
    # # val < TYPEMAX_INT64_STR && return Base.parse(Int64, val)
    # sizeof(val) <= sizeof(TYPEMAX_INTval < TYPEMAX_INT128_STR128_STR) && return Base.parse(Int128, val)
    # # val < TYPEMAX_INT128_STR && return Base.parse(Int128, val)

@@ -120,525 +118,71 @@


# Expressions

# Fallback
function Expr(x::EXPR)
    if isidentifier(x)
        if typof(x) === NONSTDIDENTIFIER
        if headof(x) === :NONSTDIDENTIFIER
            Symbol(normalize_julia_identifier(valof(x.args[2])))
        else
            return Symbol(normalize_julia_identifier(valof(x)))
        end
    elseif iskw(x)
        if kindof(x) === Tokens.BREAK
    elseif iskeyword(x)
        if headof(x) === :BREAK
            return Expr(:break)
        elseif kindof(x) === Tokens.CONTINUE
        elseif headof(x) === :CONTINUE
            return Expr(:continue)
        else
            return Symbol(lowercase(string(kindof(x))))
            return Symbol(lowercase(string(headof(x))))
        end
    elseif isoperator(x)
        ret = x.val isa String ? Symbol(valof(x)) : UNICODE_OPS_REVERSE[kindof(x)]
        return x.dot ? Symbol(:., ret) : ret
        return Symbol(valof(x))
    elseif ispunctuation(x)
        return string(kindof(x))
        if headof(x) === :DOT
            return :(.)
        else 
            error()
        end
    elseif isliteral(x)
        return _literal_expr(x)
    elseif isunarycall(x)
        return _unary_expr(x)
    elseif isbinarycall(x)
        return _binary_expr(x)
    elseif iswherecall(x)
        return _where_expr(x)
    elseif typof(x) === ConditionalOpCall
        return Expr(:if, Expr(x.args[1]), Expr(x.args[3]), Expr(x.args[5]))
    elseif typof(x) === ErrorToken
        ret = Expr(:error)
        if x.args !== nothing
            for a in x.args
                if !(ispunctuation(a))
                    push!(ret.args, Expr(a))
                end
            end
        end
        return ret
    elseif typof(x) === ChainOpCall
        ret = Expr(:call, Expr(x.args[2]))
        for i = 1:length(x.args)
            if isodd(i)
                push!(ret.args, Expr(x.args[i]))
            end
        end
        return ret
    elseif typof(x) === Comparison
        ret = Expr(:comparison)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === ColonOpCall
        return Expr(:call, :(:), Expr(x.args[1]), Expr(x.args[3]), Expr(x.args[5]))
    elseif typof(x) === TopLevel
        ret = Expr(:toplevel)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === MacroName
        if isidentifier(x.args[2])
            if valof(x.args[2]) == "."
                return Symbol("@", "__dot__")
            else
                return Symbol("@", valof(x.args[2]))
            end
        elseif isoperator(x.args[2])
            return Symbol("@", Expr(x.args[2]))

        else
            return Symbol("@")
        end
    elseif typof(x) === MacroCall
        ret = Expr(:macrocall)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        insert!(ret.args, 2, nothing)
        if ret.args[1] isa Expr && ret.args[1].head == :. && string(ret.args[1].args[2].value)[1] != '@'
            clear_at!(ret.args[1])
            ret.args[1].args[2] = QuoteNode(Symbol(string('@', ret.args[1].args[2].value)))
        end
        ret
    elseif typof(x) === x_Str
        if isbinarycall(x.args[1]) && issyntaxcall(x.args[1].args[2])
            mname = Expr(x.args[1])
            mname.args[2] = QuoteNode(Symbol("@", mname.args[2].value, "_str"))
            ret = Expr(:macrocall, mname, nothing)
        else
            ret = Expr(:macrocall, Symbol("@", valof(x.args[1]), "_str"), nothing)
        end
        for i = 2:length(x.args)
            push!(ret.args, valof(x.args[i]))
        end
        return ret
    elseif typof(x) === x_Cmd
        ret = Expr(:macrocall, Symbol("@", valof(x.args[1]), "_cmd"), nothing)
        for i = 2:length(x.args)
            push!(ret.args, valof(x.args[i]))
        end
        return ret
    elseif typof(x) === Quotenode
        return QuoteNode(Expr(x.args[end]))
    elseif typof(x) === Call
        if kindof(x.args[1]) === Tokens.ISSUBTYPE || kindof(x.args[1]) === Tokens.ISSUPERTYPE
            ret = Expr(Expr(x.args[1]))
            for i in 2:length(x.args)
                a = x.args[i]
                if typof(a) === Parameters
                    insert!(ret.args, 2, Expr(a))
                elseif !(ispunctuation(a))
                    push!(ret.args, Expr(a))
                end
            end
            return ret
        end
        ret = Expr(:call)
        for a in x.args
            if typof(a) === Parameters
                insert!(ret.args, 2, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Braces
        ret = Expr(:braces)
        for a in x.args
            if typof(a) === Parameters
                insert!(ret.args, 1, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === BracesCat
        ret = Expr(:bracescat)
        for a in x.args
            if typof(a) === Parameters
                insert!(ret.args, 1, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Struct
        return Expr(:struct, false, Expr(x.args[2]), Expr(x.args[3]))
    elseif typof(x) === Mutable
        return length(x.args) == 4 ? Expr(:struct, true, Expr(x.args[2]), Expr(x.args[3])) : Expr(:struct, true, Expr(x.args[3]), Expr(x.args[4]))
    elseif typof(x) === Abstract
        return length(x.args) == 2 ? Expr(:abstract, Expr(x.args[2])) : Expr(:abstract, Expr(x.args[3]))
    elseif typof(x) === Primitive
        return Expr(:primitive, Expr(x.args[3]), Expr(x.args[4]))
    elseif typof(x) === FunctionDef
        ret = Expr(:function)
        for a in x.args
            if !(ispunctuation(a) || iskw(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Macro
        if length(x.args) == 3
            Expr(:macro, Expr(x.args[2]))
        else
            Expr(:macro, Expr(x.args[2]), Expr(x.args[3]))
        end
    elseif typof(x) === ModuleH
        return Expr(:module, true, Expr(x.args[2]), Expr(x.args[3]))
    elseif typof(x) === BareModule
        return Expr(:module, false, Expr(x.args[2]), Expr(x.args[3]))
    elseif typof(x) === If
        return _if_expr(x)
    elseif typof(x) === Try
        ret = Expr(:try)
        for a in x.args
            if !(ispunctuation(a) || iskw(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Let
        return _let_expr(x)
    elseif typof(x) === Do
        return Expr(:do, Expr(x.args[1]), Expr(:->, Expr(x.args[3]), Expr(x.args[4])))
    elseif typof(x) === Outer
        return Expr(:outer, Expr(x.args[2]))
    elseif typof(x) === For
        ret = Expr(:for)
        if typof(x.args[2]) === Block
            arg = Expr(:block)
            for a in x.args[2].args
                if !(ispunctuation(a))
                    push!(arg.args, fix_range(a))
                end
            end
            push!(ret.args, arg)
        else
            push!(ret.args, fix_range(x.args[2]))
        end
        push!(ret.args, Expr(x.args[3]))
        return ret
    elseif typof(x) === While
        ret = Expr(:while)
        for a in x.args
            if !(ispunctuation(a) || iskw(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif istuple(x)
        ret = Expr(:tuple)
        for a in x.args
            if typof(a) == Parameters
                insert!(ret.args, 1, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Curly
        ret = Expr(:curly)
        for a in x.args
            if typof(a) === Parameters
                insert!(ret.args, 2, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Vect
        ret = Expr(:vect)
        for a in x.args
            if typof(a) === Parameters
                pushfirst!(ret.args, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Row
        ret = Expr(:row)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Hcat
        ret = Expr(:hcat)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Vcat
        ret = Expr(:vcat)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Block
        ret = Expr(:block)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Kw
        return Expr(:kw, Expr(x.args[1]), Expr(x.args[3]))
    elseif typof(x) === Parameters
        ret = Expr(:parameters)
        for a in x.args
            if typof(a) === Parameters
                insert!(ret.args, 2, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Return
        ret = Expr(:return)
        for i = 2:length(x.args)
            a = x.args[i]
            push!(ret.args, Expr(a))
        end
        return ret
    elseif isbracketed(x)
        return Expr(x.args[2])
    elseif typof(x) === Begin
        return Expr(x.args[2])
    elseif typof(x) === Quote
        if length(x.args) == 1
            return Expr(:quote, Expr(x.args[1]))
        elseif isbracketed(x.args[2]) && (isoperator(x.args[2].args[2]) || isliteral(x.args[2].args[2]) || isidentifier(x.args[2].args[2]))
            return QuoteNode(Expr(x.args[2]))
        return Expr(x.args[1])
    elseif x.head isa EXPR
        Expr(Expr(x.head), Expr.(x.args)...)
    elseif x.head === :quotenode
        QuoteNode(Expr(x.args[1]))
    elseif x.head === :globalrefdoc
        GlobalRef(Core, Symbol("@doc"))
    elseif x.head === :globalrefcmd
        if VERSION > v"1.1"
            GlobalRef(Core, Symbol("@cmd"))
        else
            return Expr(:quote, Expr(x.args[2]))
        end
    elseif typof(x) === Global
        ret = Expr(:global)
        if typof(x.args[2]) === Const
            ret = Expr(:const, Expr(:global, Expr(x.args[2].args[2])))
        elseif length(x.args) == 2 && istuple(x.args[2])
            for a in x.args[2].args
                if !(ispunctuation(a))
                    push!(ret.args, Expr(a))
                end
            end
        else
            for i = 2:length(x.args)
                a = x.args[i]
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Local
        ret = Expr(:local)
        if typof(x.args[2]) === Const
            ret = Expr(:const, Expr(:global, Expr(x.args[2].args[2])))
        elseif length(x.args) == 2 && istuple(x.args[2])
            for a in x.args[2].args
                if !(ispunctuation(a))
                    push!(ret.args, Expr(a))
                end
            end
        else
            for i = 2:length(x.args)
                a = x.args[i]
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Const
        ret = Expr(:const)
        for i = 2:length(x.args)
            a = x.args[i]
            push!(ret.args, Expr(a))
        end
        return ret
    elseif typof(x) === GlobalRefDoc
        return GlobalRef(Core, Symbol("@doc"))
    elseif typof(x) === Ref
        ret = Expr(:ref)
        for a in x.args
            if typof(a) === Parameters
                insert!(ret.args, 2, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === TypedHcat
        ret = Expr(:typed_hcat)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === TypedVcat
        ret = Expr(:typed_vcat)
        for a in x.args
            if typof(a) === Parameters
                insert!(ret.args, 2, Expr(a))
            elseif !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Comprehension || typof(x) === DictComprehension
        ret = Expr(:comprehension)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Flatten
        iters, args = get_inner_gen(x)
        i = popfirst!(iters)
        ex = Expr(:generator, Expr(args[1]), convert_iter_assign(i[1]))
        for i in iters
            if length(i) == 1
                ex = Expr(:generator, ex, convert_iter_assign(i[1]))
                ex = Expr(:flatten, ex)
            else
                ex = Expr(:generator, ex)
                for j in i
                    push!(ex.args, convert_iter_assign(j))
                end
                ex = Expr(:flatten, ex)
            end
        end
        return ex
    elseif typof(x) === Generator
        ret = Expr(:generator, Expr(x.args[1]))
        for i = 3:length(x.args)
            a = x.args[i]
            if !(ispunctuation(a))
                push!(ret.args, convert_iter_assign(a))
            end
        end
        return ret
    elseif typof(x) === Filter
        ret = Expr(:filter)
        push!(ret.args, Expr(last(x.args)))
        for i in 1:length(x.args) - 1
            a = x.args[i]
            if !(is_if(a) || ispunctuation(a))
                push!(ret.args, convert_iter_assign(a))
            end
        end
        return ret
    elseif typof(x) === TypedComprehension
        ret = Expr(:typed_comprehension)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === Import
        return expr_import(x, :import)
    elseif typof(x) === Using
        return expr_import(x, :using)
    elseif typof(x) === Export
        ret = Expr(:export)
        for i = 2:length(x.args)
            a = x.args[i]
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
    elseif typof(x) === FileH
        ret = Expr(:file)
        for a in x.args
            push!(ret.args, Expr(a))
        end
        return ret
    elseif typof(x) === StringH
        ret = Expr(:string)
        for (i, a) in enumerate(x.args)
            if isunarycall(a)
                a = a.args[2]
            elseif isliteral(a) && kindof(a) === Tokens.STRING && span(a) == 0 || ((i == 1 || i == length(x.args)) && span(a) == 1) || (valof(a) === nothing || isempty(valof(a)))
                continue
            else isliteral(a) && kindof(a) === Tokens.TRIPLE_STRING && span(a) == 0 || ((i == 1 || i == length(x.args)) && span(a) == 3) || (valof(a) === nothing || isempty(valof(a)))
            end
            push!(ret.args, Expr(a))
        end
        return ret
            Symbol("@cmd")
        end
    elseif x.head === :macrocall && is_getfield_w_quotenode(x.args[1]) && !ismacroname(x.args[1].args[2].args[1])
        # Shift '@' to the right
        new_name = Expr(:., remove_at(x.args[1].args[1]), QuoteNode(Symbol("@", valof(x.args[1].args[2].args[1]))))
        Expr(:macrocall, new_name, Expr.(x.args[2:end])...)
    elseif x.head === :macrocall && isidentifier(x.args[1]) && valof(x.args[1]) == "@."
        Expr(:macrocall, Symbol("@__dot__"), Expr.(x.args[2:end])...)
    elseif x.head === :string && length(x.args) > 0 && (x.args[1].head === :STRING || x.args[1].head === :TRIPLESTRING) && isempty(valof(x.args[1]))
        # Special conversion needed - the initial text section is treated as empty for the represented string following lowest-common-prefix adjustments, but exists in the source.
        Expr(:string, Expr.(x.args[2:end])...)
    elseif x.args === nothing
        Expr(Symbol(lowercase(String(x.head))))
    else
        ret = Expr(:call)
        for a in x.args
            if !(ispunctuation(a))
                push!(ret.args, Expr(a))
            end
        end
        return ret
        Expr(Symbol(lowercase(String(x.head))), Expr.(x.args)...)
    end
end

# Op. expressions

function _unary_expr(x)
    if isoperator(x.args[1]) && issyntaxunarycall(x.args[1])
        Expr(Expr(x.args[1]), Expr(x.args[2]))
    elseif isoperator(x.args[2]) && issyntaxunarycall(x.args[2])
        Expr(Expr(x.args[2]), Expr(x.args[1]))
function remove_at(x)
    if isidentifier(x) && valof(x) !== nothing && first(valof(x)) == '@'
        return Symbol(valof(x)[2:end])
    elseif is_getfield_w_quotenode(x)
        Expr(:., remove_at(x.args[1]), QuoteNode(remove_at(x.args[2].args[1])))
    else
        Expr(:call, Expr(x.args[1]), Expr(x.args[2]))
    end
end
function _binary_expr(x)
    if issyntaxcall(x.args[2]) && !(kindof(x.args[2]) in (Tokens.COLON,))
        if kindof(x.args[2]) === Tokens.DOT
            arg1, arg2 = Expr(x.args[1]), Expr(x.args[3])
            if arg2 isa Expr && arg2.head === :macrocall && endswith(string(arg2.args[1]), "_cmd")
                return Expr(:macrocall, Expr(:., arg1, QuoteNode(arg2.args[1])), nothing, arg2.args[3])
            elseif arg2 isa Expr && arg2.head === :braces
                return Expr(:., arg1, Expr(:quote, arg2))
            end
        end
        Expr(Expr(x.args[2]), Expr(x.args[1]), Expr(x.args[3]))
    else
        Expr(:call, Expr(x.args[2]), Expr(x.args[1]), Expr(x.args[3]))
        Expr(x)
    end
end

function _where_expr(x)
    ret = Expr(:where, Expr(x.args[1]))
    for i = 3:length(x.args)
        a = x.args[i]
        if typof(a) === Parameters
            insert!(ret.args, 2, Expr(a))
        elseif !(ispunctuation(a) || iskw(a))
            push!(ret.args, Expr(a))
        end
    end
    return ret
end


# cross compatability for line number insertion in macrocalls
if VERSION > v"1.1-"
    Expr_cmd(x) = Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), nothing, valof(x))

@@ -690,210 +234,4 @@

    x
end

function _if_expr(x)
    ret = Expr(:if)
    iselseif = false
    n = length(x.args)
    i = 0
    while i < n
        i += 1
        a = x.args[i]
        if iskw(a) && kindof(a) === Tokens.ELSEIF
            i += 1
            r1 = Expr(x.args[i].args[1])
            push!(ret.args, Expr(:elseif, r1.args...))
        elseif !(ispunctuation(a) || iskw(a))
            push!(ret.args, Expr(a))
        end
    end
    ret
end

function _let_expr(x)
    ret = Expr(:let)
    if length(x.args) == 3
        push!(ret.args, Expr(:block))
        push!(ret.args, Expr(x.args[2]))
        return ret
    elseif typof(x.args[2]) === Block
        arg = Expr(:block)
        for a in x.args[2].args
            if !(ispunctuation(a))
                push!(arg.args, fix_range(a))
            end
        end
        push!(ret.args, arg)
    else
        push!(ret.args, fix_range(x.args[2]))
    end
    push!(ret.args, Expr(x.args[3]))
    ret
end

function fix_range(a)
    if isbinarycall(a) && (is_in(a.args[2]) || is_elof(a.args[2]))
        Expr(:(=), Expr(a.args[1]), Expr(a.args[3]))
    else
        Expr(a)
    end
end

function get_inner_gen(x, iters=[], arg=[])
    if typof(x) == Flatten
        get_inner_gen(x.args[1], iters, arg)
    elseif typof(x) === Generator
        # push!(iters, get_iter(x))
        get_iters(x, iters)
        if typof(x.args[1]) === Generator || typof(x.args[1]) === Flatten
            get_inner_gen(x.args[1], iters, arg)
        else
            push!(arg, x.args[1])
        end
    end
    return iters, arg
end

function get_iter(x)
    if typof(x) === Generator
        return x.args[3]
    end
end

function get_iters(x, iters)
    iters1 = []
    if typof(x) === Generator
        # return x.args[3]
        for i = 3:length(x.args)
            if typof(x.args[i]) !== PUNCTUATION
                push!(iters1, x.args[i])
            end
        end
    end
    push!(iters, iters1)
end

function convert_iter_assign(a)
    if isbinarycall(a) && (is_in(a.args[2]) || is_elof(a.args[2]))
        return Expr(:(=), Expr(a.args[1]), Expr(a.args[3]))
    else
        return Expr(a)
    end
end

function _get_import_block(x, i, ret)
    while is_dot(x.args[i + 1])
        i += 1
        push!(ret.args, :.)
    end
    while i < length(x.args) && !(is_comma(x.args[i + 1]))
        i += 1
        a = x.args[i]
        if !(ispunctuation(a)) && !(is_dot(a) || is_colon(a))
            push!(ret.args, Expr(a))
        end
    end

    return i
end

function expr_import(x, kw)
    col = findall(a -> isoperator(a) && precedence(a) == ColonOp, x.args)
    comma = findall(is_comma, x.args)

    header = []
    args = [Expr(:.)]
    i = 1 # skip keyword
    while i < length(x.args)
        i += 1
        a = x.args[i]
        if is_colon(a)
            push!(header, popfirst!(args))
            push!(args, Expr(:.))
        elseif is_comma(a)
            push!(args, Expr(:.))
        elseif !(ispunctuation(a))
            push!(last(args).args, Expr(a))
        end
    end
    if isempty(header)
        return Expr(kw, args...)
    else
        return Expr(kw, Expr(:(:), header..., args...))
    end
end



const UNICODE_OPS_REVERSE = Dict{Tokenize.Tokens.Kind,Symbol}()
for (k, v) in Tokenize.Tokens.UNICODE_OPS
    UNICODE_OPS_REVERSE[v] = Symbol(k)
end

UNICODE_OPS_REVERSE[Tokens.EQ] = :(=)
UNICODE_OPS_REVERSE[Tokens.PLUS_EQ] = :(+=)
UNICODE_OPS_REVERSE[Tokens.MINUS_EQ] = :(-=)
UNICODE_OPS_REVERSE[Tokens.STAR_EQ] = :(*=)
UNICODE_OPS_REVERSE[Tokens.FWD_SLASH_EQ] = :(/=)
UNICODE_OPS_REVERSE[Tokens.FWDFWD_SLASH_EQ] = :(//=)
UNICODE_OPS_REVERSE[Tokens.OR_EQ] = :(|=)
UNICODE_OPS_REVERSE[Tokens.CIRCUMFLEX_EQ] = :(^=)
UNICODE_OPS_REVERSE[Tokens.DIVISION_EQ] = :(÷=)
UNICODE_OPS_REVERSE[Tokens.REM_EQ] = :(%=)
UNICODE_OPS_REVERSE[Tokens.LBITSHIFT_EQ] = :(<<=)
UNICODE_OPS_REVERSE[Tokens.RBITSHIFT_EQ] = :(>>=)
UNICODE_OPS_REVERSE[Tokens.LBITSHIFT] = :(<<)
UNICODE_OPS_REVERSE[Tokens.RBITSHIFT] = :(>>)
UNICODE_OPS_REVERSE[Tokens.UNSIGNED_BITSHIFT] = :(>>>)
UNICODE_OPS_REVERSE[Tokens.UNSIGNED_BITSHIFT_EQ] = :(>>>=)
UNICODE_OPS_REVERSE[Tokens.BACKSLASH_EQ] = :(\=)
UNICODE_OPS_REVERSE[Tokens.AND_EQ] = :(&=)
UNICODE_OPS_REVERSE[Tokens.COLON_EQ] = :(:=)
UNICODE_OPS_REVERSE[Tokens.PAIR_ARROW] = :(=>)
UNICODE_OPS_REVERSE[Tokens.APPROX] = :(~)
UNICODE_OPS_REVERSE[Tokens.EX_OR_EQ] = :($=)
UNICODE_OPS_REVERSE[Tokens.XOR_EQ] = :(⊻=)
UNICODE_OPS_REVERSE[Tokens.RIGHT_ARROW] = :(-->)
UNICODE_OPS_REVERSE[Tokens.LAZY_OR] = :(||)
UNICODE_OPS_REVERSE[Tokens.LAZY_AND] = :(&&)
UNICODE_OPS_REVERSE[Tokens.ISSUBTYPE] = :(<:)
UNICODE_OPS_REVERSE[Tokens.ISSUPERTYPE] = :(>:)
UNICODE_OPS_REVERSE[Tokens.GREATER] = :(>)
UNICODE_OPS_REVERSE[Tokens.LESS] = :(<)
UNICODE_OPS_REVERSE[Tokens.GREATER_EQ] = :(>=)
UNICODE_OPS_REVERSE[Tokens.GREATER_THAN_OR_EQUAL_TO] = :(≥)
UNICODE_OPS_REVERSE[Tokens.LESS_EQ] = :(<=)
UNICODE_OPS_REVERSE[Tokens.LESS_THAN_OR_EQUAL_TO] = :(≤)
UNICODE_OPS_REVERSE[Tokens.EQEQ] = :(==)
UNICODE_OPS_REVERSE[Tokens.EQEQEQ] = :(===)
UNICODE_OPS_REVERSE[Tokens.IDENTICAL_TO] = :(≡)
UNICODE_OPS_REVERSE[Tokens.NOT_EQ] = :(!=)
UNICODE_OPS_REVERSE[Tokens.NOT_EQUAL_TO] = :(≠)
UNICODE_OPS_REVERSE[Tokens.NOT_IS] = :(!==)
UNICODE_OPS_REVERSE[Tokens.NOT_IDENTICAL_TO] = :(≢)
UNICODE_OPS_REVERSE[Tokens.IN] = :(in)
UNICODE_OPS_REVERSE[Tokens.ISA] = :(isa)
UNICODE_OPS_REVERSE[Tokens.LPIPE] = :(<|)
UNICODE_OPS_REVERSE[Tokens.RPIPE] = :(|>)
UNICODE_OPS_REVERSE[Tokens.COLON] = :(:)
UNICODE_OPS_REVERSE[Tokens.DDOT] = :(..)
UNICODE_OPS_REVERSE[Tokens.EX_OR] = :($)
UNICODE_OPS_REVERSE[Tokens.PLUS] = :(+)
UNICODE_OPS_REVERSE[Tokens.MINUS] = :(-)
UNICODE_OPS_REVERSE[Tokens.PLUSPLUS] = :(++)
UNICODE_OPS_REVERSE[Tokens.OR] = :(|)
UNICODE_OPS_REVERSE[Tokens.STAR] = :(*)
UNICODE_OPS_REVERSE[Tokens.FWD_SLASH] = :(/)
UNICODE_OPS_REVERSE[Tokens.REM] = :(%)
UNICODE_OPS_REVERSE[Tokens.BACKSLASH] = :(\)
UNICODE_OPS_REVERSE[Tokens.AND] = :(&)
UNICODE_OPS_REVERSE[Tokens.FWDFWD_SLASH] = :(//)
UNICODE_OPS_REVERSE[Tokens.CIRCUMFLEX_ACCENT] = :(^)
UNICODE_OPS_REVERSE[Tokens.DECLARATION] = :(::)
UNICODE_OPS_REVERSE[Tokens.CONDITIONAL] = :?
UNICODE_OPS_REVERSE[Tokens.DOT] = :(.)
UNICODE_OPS_REVERSE[Tokens.NOT] = :(!)
UNICODE_OPS_REVERSE[Tokens.PRIME] = Symbol(''')
UNICODE_OPS_REVERSE[Tokens.DDDOT] = :(...)
UNICODE_OPS_REVERSE[Tokens.TRANSPOSE] = Symbol(".'")
UNICODE_OPS_REVERSE[Tokens.ANON_FUNC] = :(->)
UNICODE_OPS_REVERSE[Tokens.WHERE] = :where

@@ -1,44 +1,44 @@

function accept_rparen(ps)
    if kindof(ps.nt) === Tokens.RPAREN
        return mPUNCTUATION(next(ps))
        return EXPR(next(ps))
    else
        return mErrorToken(ps, mPUNCTUATION(Tokens.RPAREN, 0, 0), UnexpectedToken)
        return mErrorToken(ps, EXPR(:RPAREN, 0, 0), UnexpectedToken)
    end
end
accept_rparen(ps::ParseState, args) = push!(args, accept_rparen(ps))

function accept_rsquare(ps)
    if kindof(ps.nt) === Tokens.RSQUARE
        return mPUNCTUATION(next(ps))
        return EXPR(next(ps))
    else
        return mErrorToken(ps, mPUNCTUATION(Tokens.RSQUARE, 0, 0), UnexpectedToken)
        return mErrorToken(ps, EXPR(:RSQUARE, 0, 0), UnexpectedToken)
    end
end
accept_rsquare(ps::ParseState, args) = push!(args, accept_rsquare(ps))

function accept_rbrace(ps)
    if kindof(ps.nt) === Tokens.RBRACE
        return mPUNCTUATION(next(ps))
        return EXPR(next(ps))
    else
        return mErrorToken(ps, mPUNCTUATION(Tokens.RBRACE, 0, 0), UnexpectedToken)
        return mErrorToken(ps, EXPR(:RBRACE, 0, 0), UnexpectedToken)
    end
end
accept_rbrace(ps::ParseState, args) = push!(args, accept_rbrace(ps))

function accept_end(ps::ParseState)
    if kindof(ps.nt) === Tokens.END
        return mKEYWORD(next(ps))
        return EXPR(next(ps))
    else
        return mErrorToken(ps, mKEYWORD(Tokens.END, 0, 0), UnexpectedToken)
        return mErrorToken(ps, EXPR(:END, 0, 0), UnexpectedToken)
    end
end
accept_end(ps::ParseState, args) = push!(args, accept_end(ps))

function accept_comma(ps)
    if iscomma(ps.nt)
        return mPUNCTUATION(next(ps))
        return EXPR(next(ps))
    else
        return mErrorToken(mPUNCTUATION(Tokens.COMMA, 0, 0), UnexpectedToken)
        return EXPR(:RPAREN, 0, 0)
    end
end
accept_comma(ps::ParseState, args) = push!(args, accept_comma(ps))

@@ -52,9 +52,9 @@

end

function requires_no_ws(x, ps)
    if !(ps.nt.kind === Tokens.RPAREN ||
        ps.nt.kind === Tokens.RBRACE ||
        ps.nt.kind === Tokens.RSQUARE) && x.span != x.fullspan
    if !(kindof(ps.nt) === Tokens.RPAREN ||
        kindof(ps.nt) === Tokens.RBRACE ||
        kindof(ps.nt) === Tokens.RSQUARE) && x.span != x.fullspan
        return mErrorToken(ps, x, UnexpectedWhiteSpace)
    else
        return x

@@ -1,95 +1,165 @@

	1	+	# terminals
	2	+	isidentifier(x::EXPR) = headof(x) === :IDENTIFIER \|\| headof(x) === :NONSTDIDENTIFIER
	3	+	isoperator(x::EXPR) = headof(x) === :OPERATOR
	4	+	isnonstdid(x::EXPR) = headof(x) === :NONSTDIDENTIFIER
	5	+	iskeyword(x::EXPR) = headof(x) in (:ABSTRACT, :BAREMODULE, :BEGIN, :BREAK, :CATCH, :CONST, :CONTINUE, :DO, :ELSE, :ELSEIF, :END, :EXPORT, :FINALLY, :FOR, :FUNCTION, :GLOBAL, :IF, :IMPORT, :importall, :LET, :LOCAL, :MACRO, :MODULE, :MUTABLE, :NEW, :OUTER, :PRIMITIVE, :QUOTE, :RETURN, :STRUCT, :TRY, :TYPE, :USING, :WHILE)
	6	+	isliteral(x::EXPR) = isstringliteral(x) \|\| iscmd(x) \|\| ischar(x) \|\| headof(x) in (:INTEGER, :BININT, :HEXINT, :OCTINT, :FLOAT, :NOTHING, :TRUE, :FALSE)
	7	+	ispunctuation(x::EXPR) = is_comma(x) \|\| is_lparen(x) \|\| is_rparen(x) \|\| is_lsquare(x) \|\| is_rsquare(x) \|\| is_lbrace(x) \|\| is_rbrace(x) \|\| headof(x) === :ATSIGN \|\| headof(x) === :DOT
	8	+	isstringliteral(x) = headof(x) === :STRING \|\| headof(x) === :TRIPLESTRING
	9	+	isstring(x) = headof(x) === :string \|\| isstringliteral(x)
	10	+	iscmd(x) = headof(x) === :CMD \|\| headof(x) === :TRIPLECMD
	11	+	ischar(x) = headof(x) === :CHAR
	12	+	isinteger(x) = headof(x) === :INTEGER
	13	+	isfloat(x) = headof(x) === :FLOAT
	14	+	isnumber(x) = isinteger(x) \|\| isfloat(x)
	15	+	is_nothing(x) = headof(x) === :NOTHING
	16	+	is_either_id_op_interp(x::EXPR) = isidentifier(x) \|\| isoperator(x) \|\| isinterpolant(x)
	17	+	is_id_or_macroname(x::EXPR) = isidentifier(x) \|\| ismacroname(x)
	18	+
	19	+	# expressions
	20	+	iscall(x::EXPR) = headof(x) === :call
	21	+	isunarycall(x::EXPR) = (headof(x) === :call && length(x) == 2 && (isoperator(x.args[1]) \|\| isoperator(x.args[2])))
	22	+	isunarysyntax(x::EXPR) = (isoperator(x.head) && length(x.args) == 1)
	23	+	isbinarycall(x::EXPR) = headof(x) === :call && length(x) == 3 && isoperator(x.args[1])
	24	+	isbinarycall(x::EXPR, op) = headof(x) === :call && length(x) == 3 && isoperator(x.args[1]) && valof(x.args[1]) == op
	25	+	isbinarysyntax(x::EXPR) = (isoperator(x.head) && length(x.args) == 2)
	26	+	ischainedcall(x::EXPR) = headof(x) === :call && isoperator(x.args[1]) && (valof(x.args[1]) == "+" \|\| valof(x.args[1]) == "*") && hastrivia(x) && isoperator(first(x.trivia))
	27	+	iswhere(x::EXPR) = headof(x) === :where
	28	+	istuple(x::EXPR) = headof(x) === :tuple
	29	+	ismacrocall(x::EXPR) = headof(x) === :macrocall
	30	+	ismacroname(x::EXPR) = (isidentifier(x) && valof(x) !== nothing && !isempty(valof(x)) && first(valof(x)) == '@') \|\| (is_getfield_w_quotenode(x) && (ismacroname(unquotenode(rhs_getfield(x))) \|\| ismacroname(x.args[1])))
	31	+
	32	+	iskwarg(x::EXPR) = headof(x) === :kw
	33	+	isparameters(x::EXPR) = headof(x) === :parameters
	34	+	iscurly(x::EXPR) = headof(x) === :curly
	35	+	isbracketed(x::EXPR) = headof(x) === :brackets
	36	+
	37	+
	38	+	isassignment(x::EXPR) = isbinarysyntax(x) && valof(x.head) == "="
	39	+	isdeclaration(x::EXPR) = isbinarysyntax(x) && valof(x.head) == "::"
	40	+	isinterpolant(x::EXPR) = isunarysyntax(x) && valof(x.head) == "\$"
	41	+	issplat(x::EXPR) = isunarysyntax(x) && valof(x.head) == "..."
	42	+
	43	+
	44	+	isbeginorblock(x::EXPR) = headof(x) === :begin \|\| headof(unwrapbracket(x)) == :block
	45	+
	46	+
	47	+	"""
	48	+	is_getfield(x::EXPR)
	49	+
	50	+	Is this an expression of the form `a.b`.
	51	+	"""
	52	+	is_getfield(x::EXPR) = x.head isa EXPR && isoperator(x.head) && valof(x.head) == "." && length(x.args) == 2
	53	+	is_getfield_w_quotenode(x) = is_getfield(x) && headof(x.args[2]) === :quotenode && length(x.args[2].args) > 0
	54	+	rhs_getfield(x) = x.args[2]
	55	+	unquotenode(x) = x.args[1]
	56	+
	57	+	function get_rhs_of_getfield(ret::EXPR)
	58	+	if headof(ret.args[2]) === :quotenode \|\| headof(ret.args[2]) === :quote
	59	+	ret.args[2].args[1]
	60	+	else
	61	+	ret.args[2]
	62	+	end
	63	+	end
	64	+
	65	+
	66	+	"""
	67	+	is_wrapped_assignment(x::EXPR)
	68	+
	69	+	Is `x` an assignment expression, ignoring any surrounding parentheses.
	70	+	"""
	71	+	is_wrapped_assignment(x::EXPR) = isassignment(x) \|\| (isbracketed(x) && is_wrapped_assignment(x.args[1]))
	72	+
	73	+
1	74		function is_func_call(x::EXPR)
2		-	if typof(x) === Call
	75	+	if isoperator(x.head)
	76	+	if length(x.args) == 2
	77	+	return is_decl(x.head) && is_func_call(x.args[1])
	78	+	elseif length(x.args) == 1
	79	+	return !(is_exor(x.head) \|\| is_decl(x.head))
	80	+	end
	81	+	elseif x.head === :call
3	82		return true
4		-	elseif iswherecall(x)
	83	+	elseif x.head === :where \|\| isbracketed(x)
5	84		return is_func_call(x.args[1])
6		-	elseif isbracketed(x)
7		-	return is_func_call(x.args[2])
8		-	elseif isunarycall(x)
9		-	return !(isoperator(x.args[1]) && (kindof(x.args[1]) === Tokens.EX_OR \|\| kindof(x.args[1]) === Tokens.DECLARATION))
10		-	elseif isbinarycall(x)
11		-	if issyntaxcall(x.args[2])
12		-	if is_decl(x.args[2])
13		-	return is_func_call(x.args[1])
14		-	else
15		-	return false
16		-	end
17		-	else
18		-	true
19		-	end
20	85		else
21	86		return false
22	87		end
23	88		end
24	89
25		-	is_assignment(x) = isbinarycall(x) && kindof(x.args[2]) === Tokens.EQ
	90	+
	91	+	fcall_name(x::EXPR) = iscall(x) && length(x.args) > 0 && valof(x.args[1])
	92	+	hasparent(x::EXPR) = parentof(x) isa EXPR
26	93
27	94		# OPERATOR
28		-	is_exor(x) = isoperator(x) && kindof(x) === Tokens.EX_OR && x.dot == false
29		-	is_decl(x) = isoperator(x) && kindof(x) === Tokens.DECLARATION
30		-	is_issubt(x) = isoperator(x) && kindof(x) === Tokens.ISSUBTYPE
31		-	is_issupt(x) = isoperator(x) && kindof(x) === Tokens.ISSUPERTYPE
32		-	is_and(x) = isoperator(x) && kindof(x) === Tokens.AND && x.dot == false
33		-	is_not(x) = isoperator(x) && kindof(x) === Tokens.NOT && x.dot == false
34		-	is_plus(x) = isoperator(x) && kindof(x) === Tokens.PLUS && x.dot == false
35		-	is_minus(x) = isoperator(x) && kindof(x) === Tokens.MINUS && x.dot == false
36		-	is_star(x) = isoperator(x) && kindof(x) === Tokens.STAR && x.dot == false
37		-	is_eq(x) = isoperator(x) && kindof(x) === Tokens.EQ && x.dot == false
38		-	is_dot(x) = isoperator(x) && kindof(x) === Tokens.DOT
39		-	is_ddot(x) = isoperator(x) && kindof(x) === Tokens.DDOT
40		-	is_dddot(x) = isoperator(x) && kindof(x) === Tokens.DDDOT
41		-	is_pairarrow(x) = isoperator(x) && kindof(x) === Tokens.PAIR_ARROW && x.dot == false
42		-	is_in(x) = isoperator(x) && kindof(x) === Tokens.IN && x.dot == false
43		-	is_elof(x) = isoperator(x) && kindof(x) === Tokens.ELEMENT_OF && x.dot == false
44		-	is_colon(x) = isoperator(x) && kindof(x) === Tokens.COLON
45		-	is_prime(x) = isoperator(x) && kindof(x) === Tokens.PRIME
46		-	is_cond(x) = isoperator(x) && kindof(x) === Tokens.CONDITIONAL
47		-	is_where(x) = isoperator(x) && kindof(x) === Tokens.WHERE
48		-	is_anon_func(x) = isoperator(x) && kindof(x) === Tokens.ANON_FUNC
49		-
50		-	is_comma(x) = ispunctuation(x) && kindof(x) === Tokens.COMMA
51		-	is_lparen(x) = ispunctuation(x) && kindof(x) === Tokens.LPAREN
52		-	is_rparen(x) = ispunctuation(x) && kindof(x) === Tokens.RPAREN
53		-	is_lbrace(x) = ispunctuation(x) && kindof(x) === Tokens.LBRACE
54		-	is_rbrace(x) = ispunctuation(x) && kindof(x) === Tokens.RBRACE
55		-	is_lsquare(x) = ispunctuation(x) && kindof(x) === Tokens.LSQUARE
56		-	is_rsquare(x) = ispunctuation(x) && kindof(x) === Tokens.RSQUARE
	95	+	is_approx(x::EXPR) = isoperator(x) && valof(x) == "~"
	96	+	is_exor(x) = isoperator(x) && valof(x) == "\$"
	97	+	is_decl(x) = isoperator(x) && valof(x) == "::"
	98	+	is_issubt(x) = isoperator(x) && valof(x) == "<:"
	99	+	is_issupt(x) = isoperator(x) && valof(x) == ">:"
	100	+	is_and(x) = isoperator(x) && valof(x) == "&"
	101	+	is_not(x) = isoperator(x) && valof(x) == "!"
	102	+	is_plus(x) = isoperator(x) && valof(x) == "+"
	103	+	is_minus(x) = isoperator(x) && valof(x) == "-"
	104	+	is_star(x) = isoperator(x) && valof(x) == "*"
	105	+	is_eq(x) = isoperator(x) && valof(x) == "="
	106	+	is_dot(x) = isoperator(x) && valof(x) == "."
	107	+	is_ddot(x) = isoperator(x) && valof(x) == ".."
	108	+	is_dddot(x) = isoperator(x) && valof(x) == "..."
	109	+	is_pairarrow(x) = isoperator(x) && valof(x) == "=>"
	110	+	is_in(x) = isoperator(x) && valof(x) == "in"
	111	+	is_elof(x) = isoperator(x) && valof(x) == "∈"
	112	+	is_colon(x) = isoperator(x) && valof(x) == ":"
	113	+	is_prime(x) = isoperator(x) && valof(x) == "'"
	114	+	is_cond(x) = isoperator(x) && valof(x) == "?"
	115	+	is_where(x) = isoperator(x) && valof(x) == "where"
	116	+	is_anon_func(x) = isoperator(x) && valof(x) == "->"
	117	+
	118	+	is_comma(x) = headof(x) === :COMMA
	119	+	is_lparen(x) = headof(x) === :LPAREN
	120	+	is_rparen(x) = headof(x) === :RPAREN
	121	+	is_lbrace(x) = headof(x) === :LBRACE
	122	+	is_rbrace(x) = headof(x) === :RBRACE
	123	+	is_lsquare(x) = headof(x) === :LSQUARE
	124	+	is_rsquare(x) = headof(x) === :RSQUARE
57	125
58	126		# KEYWORD
59		-	is_if(x) = iskw(x) && kindof(x) === Tokens.IF
60		-	is_import(x) = iskw(x) && kindof(x) === Tokens.IMPORT
	127	+	is_if(x) = iskeyword(x) && headof(x) === :IF
	128	+	is_import(x) = iskeyword(x) && headof(x) === :IMPORT
61	129
62	130
63	131		# Literals
64		-	is_lit_string(x) = kindof(x) === Tokens.STRING \|\| kindof(x) === Tokens.TRIPLE_STRING
65		-
66		-	issubtypedecl(x::EXPR) = isbinarycall(x) && is_issubt(x.args[2])
67		-
68		-	rem_subtype(x::EXPR) = issubtypedecl(x) ? x[1] : x
69		-	rem_decl(x::EXPR) = isdeclaration(x) ? x[1] : x
70		-	rem_curly(x::EXPR) = typof(x) === Curly ? x.args[1] : x
71		-	rem_call(x::EXPR) = typof(x) === Call ? x[1] : x
72		-	rem_where(x::EXPR) = iswherecall(x) ? x[1] : x
73		-	rem_wheres(x::EXPR) = iswherecall(x) ? rem_wheres(x[1]) : x
74		-	rem_where_subtype(x::EXPR) = (iswherecall(x) \|\| issubtypedecl(x)) ? x[1] : x
75		-	rem_where_decl(x::EXPR) = (iswherecall(x) \|\| isdeclaration(x)) ? x[1] : x
76		-	rem_invis(x::EXPR) = isbracketed(x) ? rem_invis(x[2]) : x
77		-	rem_dddot(x::EXPR) = is_splat(x) ? x[1] : x
	132	+
	133	+	issubtypedecl(x::EXPR) = isoperator(x.head) && valof(x.head) == "<:"
	134	+
	135	+	rem_subtype(x::EXPR) = issubtypedecl(x) ? x.args[1] : x
	136	+	rem_decl(x::EXPR) = isdeclaration(x) ? x.args[1] : x
	137	+	rem_curly(x::EXPR) = headof(x) === :curly ? x.args[1] : x
	138	+	rem_call(x::EXPR) = headof(x) === :call ? x.args[1] : x
	139	+	rem_where(x::EXPR) = iswhere(x) ? x.args[1] : x
	140	+	rem_wheres(x::EXPR) = iswhere(x) ? rem_wheres(x.args[1]) : x
	141	+	rem_where_subtype(x::EXPR) = (iswhere(x) \|\| issubtypedecl(x)) ? x.args[1] : x
	142	+	rem_wheres_subtypes(x::EXPR) = (iswhere(x) \|\| issubtypedecl(x)) ? rem_wheres_subtypes(x.args[1]) : x
	143	+	rem_where_decl(x::EXPR) = (iswhere(x) \|\| isdeclaration(x)) ? x.args[1] : x
	144	+	rem_wheres_decls(x::EXPR) = (iswhere(x) \|\| isdeclaration(x)) ? rem_wheres_decls(x.args[1]) : x
	145	+	rem_invis(x::EXPR) = isbracketed(x) ? rem_invis(x.args[1]) : x
	146	+	rem_dddot(x::EXPR) = issplat(x) ? x.args[1] : x
78	147		const rem_splat = rem_dddot
79		-	rem_kw(x::EXPR) = typof(x) === Kw ? x[1] : x
	148	+	rem_kw(x::EXPR) = headof(x) === :kw ? x.args[1] : x
	149	+	unwrapbracket(x::EXPR) = isbracketed(x) ? unwrapbracket(x.args[1]) : x
80	150
81		-	is_some_call(x) = typof(x) === Call \|\| isunarycall(x) \|\| (isbinarycall(x) && !(kindof(x.args[2]) === Tokens.DOT \|\| issyntaxcall(x.args[2])))
82		-	is_eventually_some_call(x) = is_some_call(x) \|\| ((isdeclaration(x) \|\| iswherecall(x)) && is_eventually_some_call(x[1]))
	151	+	is_some_call(x) = headof(x) === :call \|\| isunarycall(x)
	152	+	is_eventually_some_call(x) = is_some_call(x) \|\| ((isdeclaration(x) \|\| iswhere(x)) && is_eventually_some_call(x.args[1]))
83	153
84		-	defines_function(x::EXPR) = typof(x) === FunctionDef \|\| (is_assignment(x) && is_eventually_some_call(x[1]))
85		-	defines_macro(x) = typof(x) == Macro
	154	+	defines_function(x::EXPR) = headof(x) === :function \|\| (isassignment(x) && is_eventually_some_call(x.args[1]))
	155	+	defines_macro(x) = headof(x) == :macro
86	156		defines_datatype(x) = defines_struct(x) \|\| defines_abstract(x) \|\| defines_primitive(x)
87		-	defines_struct(x) = typof(x) === Struct \|\| defines_mutable(x)
88		-	defines_mutable(x) = typof(x) === Mutable
89		-	defines_abstract(x) = typof(x) === Abstract
90		-	defines_primitive(x) = typof(x) === Primitive
91		-	defines_module(x) = typof(x) === ModuleH \|\| typof(x) === BareModule
92		-	defines_anon_function(x) = isbinarycall(x) && is_anon_func(x.args[2])
	157	+	defines_struct(x) = headof(x) === :struct
	158	+	defines_mutable(x) = defines_struct(x) && x.args[1].head == :TRUE
	159	+	defines_abstract(x) = headof(x) === :abstract
	160	+	defines_primitive(x) = headof(x) === :primitive
	161	+	defines_module(x) = headof(x) === :module
	162	+	defines_anon_function(x) = isoperator(x.head) && valof(x.head) == "->"
93	163
94	164		has_sig(x::EXPR) = defines_datatype(x) \|\| defines_function(x) \|\| defines_macro(x) \|\| defines_anon_function(x)
95	165

@@ -100,46 +170,39 @@

Should only be called when has_sig(x) == true.
"""
function get_sig(x::EXPR)
    if isbinarycall(x)
    if headof(x) isa EXPR # headof(headof(x)) === :OPERATOR valof(headof(x)) == "="
        return x.args[1]
    elseif typof(x) === Struct || typof(x) === FunctionDef || typof(x) === Macro
    elseif headof(x) === :struct || headof(x) === :mutable 
        return x.args[2]
    elseif typof(x) === Mutable || typof(x) === Abstract || typof(x) === Primitive
        return x.args[3]
    elseif  headof(x) === :abstract || headof(x) === :primitive || headof(x) === :function || headof(x) === :macro
        return x.args[1]
    end
end

function get_name(x::EXPR)
    if typof(x) === Struct || typof(x) === Mutable || typof(x) === Abstract || typof(x) === Primitive
    if defines_datatype(x)
        sig = get_sig(x)
        sig = rem_subtype(sig)
        sig = rem_wheres(sig)
        sig = rem_subtype(sig)
        sig = rem_curly(sig)
    elseif typof(x) === ModuleH || typof(x) === BareModule
    elseif defines_module(x)
        sig = x.args[2]
    elseif typof(x) === FunctionDef || typof(x) === Macro
    elseif defines_function(x) || defines_macro(x)
        sig = get_sig(x)
        sig = rem_wheres(sig)
        sig = rem_decl(sig)
        sig = rem_call(sig)
        sig = rem_curly(sig)
        sig = rem_invis(sig)
        if isbinarycall(sig) && kindof(sig.args[2]) === Tokens.DOT
            if length(sig.args) > 2 && sig.args[3].args isa Vector{EXPR} && length(sig.args[3].args) > 0
                sig = sig.args[3].args[1]
            end
        # if isbinarysyntax(sig) && is_dot(sig.head)
        if is_getfield_w_quotenode(sig)
            sig = sig.args[2].args[1]
        end
        return sig
    elseif is_getfield_w_quotenode(x)
        return x.args[2].args[1]
    elseif isbinarycall(x)
        length(x.args) < 2 && return x
        if kindof(x.args[2]) === Tokens.DOT
            if length(x.args) > 2 && typof(x.args[3]) === Quotenode && x.args[3].args isa Vector{EXPR} && length(x.args[3].args) > 0
                return get_name(x.args[3].args[1])
            else
                return x
            end
        end
        sig = x.args[1]
        if isunarycall(sig)
            return get_name(sig.args[1])

@@ -150,6 +213,10 @@

        sig = rem_curly(sig)
        sig = rem_invis(sig)
        return get_name(sig)
    elseif isbinarysyntax(x) && valof(x.head) == "<:"
        return get_name(x.args[1])
    elseif isunarysyntax(x) && valof(x.head) == "..."
        return get_name(x.args[1])
    else
        sig = x
        if isunarycall(sig)

@@ -96,6 +96,7 @@


Base.position(ps::ParseState) = ps.nt.startbyte

current_line(ps::ParseState) = ps.nt.startpos[1]
"""
    lex_ws_comment(l::Lexer, c)


@@ -178,14 +179,14 @@

@inline val(token::AbstractToken, ps::ParseState) = String(ps.l.io.data[token.startbyte + 1:token.endbyte + 1])
both_symbol_and_op(t::AbstractToken) = kindof(t) === Tokens.WHERE || kindof(t) === Tokens.IN || kindof(t) === Tokens.ISA
isprefixableliteral(t::AbstractToken) = (kindof(t) === Tokens.STRING || kindof(t) === Tokens.TRIPLE_STRING || kindof(t) === Tokens.CMD || kindof(t) === Tokens.TRIPLE_CMD)
isassignment(t::AbstractToken) = Tokens.begin_assignments < kindof(t) < Tokens.end_assignments
isassignmentop(t::AbstractToken) = Tokens.begin_assignments < kindof(t) < Tokens.end_assignments

isidentifier(t::AbstractToken) = kindof(t) === Tokens.IDENTIFIER
isliteral(t::AbstractToken) = Tokens.begin_literal < kindof(t) < Tokens.end_literal
isbool(t::AbstractToken) =  Tokens.TRUE ≤ kindof(t) ≤ Tokens.FALSE
iscomma(t::AbstractToken) =  kindof(t) === Tokens.COMMA
iscolon(t::AbstractToken) =  kindof(t) === Tokens.COLON
iskw(t::AbstractToken) = Tokens.iskeyword(kindof(t))
isinstance(t::AbstractToken) = isidentifier(t) || isliteral(t) || isbool(t) || iskw(t)
iskeyword(t::AbstractToken) = Tokens.iskeyword(kindof(t))
isinstance(t::AbstractToken) = isidentifier(t) || isliteral(t) || isbool(t) || iskeyword(t)
ispunctuation(t::AbstractToken) = iscomma(t) || kindof(t) === Tokens.END || Tokens.LSQUARE ≤ kindof(t) ≤ Tokens.RPAREN || kindof(t) === Tokens.AT_SIGN


@@ -7,13 +7,12 @@

const ComparisonOp  = 6
const PipeOp        = 7
const ColonOp       = 8
const PlusOp        = 9
@static if Base.operator_precedence(:<<) == 12
    const PlusOp        = 9
    const BitShiftOp    = 10
    const TimesOp       = 11
    const RationalOp    = 12
else
    const PlusOp        = 9
    const TimesOp       = 10
    const RationalOp    = 11
    const BitShiftOp    = 12

@@ -26,77 +25,6 @@

const DddotOp       = 7
const AnonFuncOp    = 14

@enum(Head,IDENTIFIER,
NONSTDIDENTIFIER,
PUNCTUATION,
OPERATOR,
KEYWORD,
LITERAL,
NoHead,
Call,
UnaryOpCall,
BinaryOpCall,
WhereOpCall,
ConditionalOpCall,
ChainOpCall,
ColonOpCall,
Abstract,
Begin,
Block,
Braces,
BracesCat,
Const,
Comparison,
Curly,
Do,
Filter,
Flatten,
For,
FunctionDef,
Generator,
Global,
GlobalRefDoc,
If,
Kw,
Let,
Local,
Macro,
MacroCall,
MacroName,
Mutable,
Outer,
Parameters,
Primitive,
Quote,
Quotenode,
InvisBrackets,
StringH,
Struct,
Try,
TupleH,
FileH,
Return,
While,
x_Cmd,
x_Str,
ModuleH,
BareModule,
TopLevel,
Export,
Import,
Using,
Comprehension,
DictComprehension,
TypedComprehension,
Hcat,
TypedHcat,
Ref,
Row,
Vcat,
TypedVcat,
Vect,
ErrorToken)

@enum(ErrorKind,
    UnexpectedToken,
    CannotJuxtapose,

@@ -110,62 +38,57 @@

    InvalidIterator,
    StringInterpolationWithTrailingWhitespace,
    TooLongChar,
    Unknown)

const NoKind = Tokenize.Tokens.begin_keywords
    Unknown,
    SignatureOfFunctionDefIsNotACall)

mutable struct EXPR
    typ::Head
    head::Union{Symbol,EXPR}
    args::Union{Nothing,Vector{EXPR}}
    trivia::Union{Nothing,Vector{EXPR}}
    fullspan::Int
    span::Int
    val::Union{Nothing,String}
    kind::Tokenize.Tokens.Kind
    dot::Bool
    parent::Union{Nothing,EXPR}
    meta
end

function EXPR(T::Head, args::Vector{EXPR}, fullspan::Int, span::Int)
    ex = EXPR(T, args, fullspan, span, nothing, NoKind, false, nothing, nothing)
function EXPR(head::Union{Symbol,EXPR}, args::Vector{EXPR}, trivia::Union{Vector{EXPR},Nothing}, fullspan::Int, span::Int)
    ex = EXPR(head, args, trivia, fullspan, span, nothing, nothing, nothing)
    if head isa EXPR
        setparent!(head, ex)
    end
    for c in args
        setparent!(c, ex)
    end
    if trivia isa Vector{EXPR}
        for c in trivia
            setparent!(c, ex)
        end
    end
    ex
end

function EXPR(T::Head, args::Vector{EXPR})
    ret = EXPR(T, args, 0, 0)
function EXPR(head::Union{Symbol,EXPR}, args::Vector{EXPR}, trivia::Union{Vector{EXPR},Nothing} = EXPR[])
    ret = EXPR(head, args, trivia, 0, 0)
    update_span!(ret)
    ret
end

# These methods are for terminal/childless expressions.
@noinline EXPR(head::Union{Symbol,EXPR}, fullspan::Int, span::Int, val = nothing) = EXPR(head, nothing, nothing, fullspan, span, val, nothing, nothing)
@noinline EXPR(head::Union{Symbol,EXPR}, ps::ParseState) = EXPR(head, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps))
@noinline EXPR(ps::ParseState) = EXPR(tokenkindtoheadmap(kindof(ps.t)), ps)



@noinline mIDENTIFIER(ps::ParseState) = EXPR(IDENTIFIER, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, nothing)

mPUNCTUATION(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(PUNCTUATION, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
@noinline mPUNCTUATION(ps::ParseState) = EXPR(PUNCTUATION, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)

mOPERATOR(fullspan::Int, span::Int, kind::Tokens.Kind, dotop::Bool) = EXPR(OPERATOR, nothing, fullspan, span, nothing, kind, dotop, nothing, nothing)
@noinline mOPERATOR(ps::ParseState) = EXPR(OPERATOR, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, ps.t.suffix ? val(ps.t, ps) : nothing, kindof(ps.t), ps.t.dotop, nothing, nothing)

mKEYWORD(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(KEYWORD, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
@noinline mKEYWORD(ps::ParseState) = EXPR(KEYWORD, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)

mLITERAL(fullspan::Int, span::Int, val::String, kind::Tokens.Kind) = EXPR(LITERAL, nothing, fullspan, span, val, kind, false, nothing, nothing)
@noinline function mLITERAL(ps::ParseState)

    if kindof(ps.t) === Tokens.STRING || kindof(ps.t) === Tokens.TRIPLE_STRING ||
        kindof(ps.t) === Tokens.CMD || kindof(ps.t) === Tokens.TRIPLE_CMD
        return parse_string_or_cmd(ps)
    else
        v = val(ps.t, ps)
        if kindof(ps.t) === Tokens.CHAR && length(v) > 3 && !(v[2] == '\\' && valid_escaped_seq(v[2:prevind(v, length(v))]))
            return mErrorToken(ps, mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\''), kindof(ps.t)), TooLongChar)
            return mErrorToken(ps, EXPR(:CHAR, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\'')), TooLongChar)
        end
        return mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v, kindof(ps.t))
        return EXPR(literalmap(kindof(ps.t)), ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v)
    end
end


@@ -174,12 +97,28 @@

span(x::EXPR) = x.span

function update_span!(x::EXPR)
    (x.args isa Nothing || isempty(x.args)) && return
    (x.args isa Nothing || isempty(x.args)) && !hastrivia(x) && return
    x.fullspan = 0
    for i = 1:length(x.args)
        x.fullspan += x.args[i].fullspan
    end
    x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
    if hastrivia(x)
        for i = 1:length(x.trivia)
            x.fullspan += x.trivia[i].fullspan
        end
    end
    if x.head isa EXPR
        x.fullspan += x.head.fullspan
        # TODO: special case for trailing unary ops?
    end
    if x.head isa EXPR && isoperator(x.head) && (is_dddot(x.head) || is_prime(x.head))
        # trailing unary operator
        x.span  = x.fullspan - x.head.fullspan + x.head.span
    elseif lastchildistrivia(x)
        x.span = x.fullspan - last(x.trivia).fullspan + last(x.trivia).span
    elseif !isempty(x.args)
        x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
    end
    return
end


@@ -190,6 +129,13 @@

    push!(e.args, arg)
end

function pushtotrivia!(e::EXPR, arg::EXPR)
    e.span = e.fullspan + arg.span
    e.fullspan += arg.fullspan
    setparent!(arg, e)
    push!(e.trivia, arg)
end

function Base.pushfirst!(e::EXPR, arg::EXPR)
    e.fullspan += arg.fullspan
    setparent!(arg, e)

@@ -227,69 +173,37 @@


function INSTANCE(ps::ParseState)
    if isidentifier(ps.t)
        return mIDENTIFIER(ps)
        return EXPR(:IDENTIFIER, ps)
    elseif isliteral(ps.t)
        return mLITERAL(ps)
    elseif iskw(ps.t)
        return mKEYWORD(ps)
    elseif iskeyword(ps.t)
        return EXPR(ps)
    elseif isoperator(ps.t)
        return mOPERATOR(ps)
        return EXPR(:OPERATOR, ps)
    elseif ispunctuation(ps.t)
        return mPUNCTUATION(ps)
        return EXPR(ps)
    elseif kindof(ps.t) === Tokens.ERROR
        ps.errored = true
        return EXPR(ErrorToken, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, Unknown)
        return EXPR(:errortoken, nothing, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), nothing, Unknown)
    else
        return mErrorToken(ps, Unknown)
    end
end

function mUnaryOpCall(op::EXPR, arg::EXPR)
    fullspan = op.fullspan + arg.fullspan
    ex = EXPR(UnaryOpCall, EXPR[op, arg], fullspan, fullspan - arg.fullspan + arg.span)
    setparent!(op, ex)
    setparent!(op, ex)
    return ex
end
function mBinaryOpCall(arg1::EXPR, op::EXPR, arg2::EXPR)
    fullspan = arg1.fullspan + op.fullspan + arg2.fullspan
    ex = EXPR(BinaryOpCall, EXPR[arg1, op, arg2], fullspan, fullspan - arg2.fullspan + arg2.span)
    setparent!(arg1, ex)
    setparent!(op, ex)
    setparent!(arg2, ex)
    return ex
end
function mWhereOpCall(arg1::EXPR, op::EXPR, args::Vector{EXPR})
    ex = EXPR(WhereOpCall, EXPR[arg1; op; args], arg1.fullspan + op.fullspan, 0)
    setparent!(arg1, ex)
    setparent!(op, ex)
    for a in args
        ex.fullspan += a.fullspan
        setparent!(a, ex)
    end
    ex.span = ex.fullspan - last(args).fullspan + last(args).span
    return ex
end

function mErrorToken(ps::ParseState, k::ErrorKind)
    ps.errored = true
    return EXPR(ErrorToken, EXPR[], 0, 0, nothing, NoKind, false, nothing, k)
    return EXPR(:errortoken, EXPR[], nothing, 0, 0, nothing, nothing, k)
end
function mErrorToken(ps::ParseState, x::EXPR, k)
    ps.errored = true
    ret = EXPR(ErrorToken, EXPR[x], x.fullspan, x.span, nothing, NoKind, false, nothing, k)
    setparent!(ret[1], ret)
    ret = EXPR(:errortoken, EXPR[x], nothing, x.fullspan, x.span, nothing, nothing, k)
    setparent!(ret.args[1], ret)
    return ret
end

TRUE() = mLITERAL(0, 0, "", Tokens.TRUE)
FALSE() = mLITERAL(0, 0, "", Tokens.FALSE)
NOTHING() = mLITERAL(0, 0, "", Tokens.NOTHING)
GlobalRefDOC() = EXPR(GlobalRefDoc, EXPR[])

typof(x::EXPR) = x.typ
headof(x::EXPR) = x.head
valof(x::EXPR) = x.val
kindof(x::EXPR) = x.kind
kindof(t::Tokens.AbstractToken) = t.kind
parentof(x::EXPR) = x.parent
errorof(x::EXPR) = errorof(x.meta)

@@ -299,14 +213,164 @@

299	213		c.parent = p
300	214		return c
301	215		end
	216	+	hastrivia(x::EXPR) = x.trivia !== nothing && length(x.trivia) > 0
	217	+
	218	+	function lastchildistrivia(x::EXPR)
	219	+	return hastrivia(x) && (last(x.trivia).head in (:END, :RPAREN, :RSQUARE, :RBRACE) \|\| (x.head in (:parameters, :tuple) && length(x.args) <= length(x.trivia)))
	220	+	end
	221	+
	222	+	function Base.length(x::EXPR)
	223	+	headof(x) === :NONSTDIDENTIFIER && return 0
	224	+	n = x.args isa Nothing ? 0 : length(x.args)
	225	+	n += hastrivia(x) ? length(x.trivia) : 0
	226	+	x.head isa EXPR && !(x.head.span === 0) && (n += 1)
	227	+	return n
	228	+	end
	229	+
	230	+	function literalmap(k::Tokens.Kind)
	231	+	if k === Tokens.INTEGER
	232	+	return :INTEGER
	233	+	elseif k === Tokens.BIN_INT
	234	+	return :BININT
	235	+	elseif k === Tokens.HEX_INT
	236	+	return :HEXINT
	237	+	elseif k === Tokens.OCT_INT
	238	+	return :OCTINT
	239	+	elseif k === Tokens.FLOAT
	240	+	return :FLOAT
	241	+	elseif k === Tokens.STRING
	242	+	return :STRING
	243	+	elseif k === Tokens.TRIPLE_STRING
	244	+	return :TRIPLESTRING
	245	+	elseif k === Tokens.CHAR
	246	+	return :CHAR
	247	+	elseif k === Tokens.CMD
	248	+	return :CMD
	249	+	elseif k === Tokens.TRIPLE_CMD
	250	+	return :TRIPLECMD
	251	+	elseif k === Tokens.TRUE
	252	+	return :TRUE
	253	+	elseif k === Tokens.FALSE
	254	+	return :FALSE
	255	+	end
	256	+	end
302	257
303	258
304		-	Base.iterate(x::EXPR) = length(x) == 0 ? nothing : (x.args[1], 1)
305		-	Base.iterate(x::EXPR, s) = s < length(x) ? (x.args[s + 1], s + 1) : nothing
306		-	Base.length(x::EXPR) = x.args isa Nothing ? 0 : length(x.args)
307		-	Base.firstindex(x::EXPR) = 1
308		-	Base.lastindex(x::EXPR) = x.args === nothing ? 0 : lastindex(x.args)
309		-	Base.getindex(x::EXPR, i) = x.args[i]
310		-	Base.setindex!(x::EXPR, val, i) = Base.setindex!(x.args, val, i)
311		-	Base.first(x::EXPR) = x.args === nothing ? nothing : first(x.args)
312		-	Base.last(x::EXPR) = x.args === nothing ? nothing : last(x.args)
	259	+
	260	+	function tokenkindtoheadmap(k::Tokens.Kind)
	261	+	if k == Tokens.COMMA
	262	+	:COMMA
	263	+	elseif k == Tokens.LPAREN
	264	+	:LPAREN
	265	+	elseif k == Tokens.RPAREN
	266	+	:RPAREN
	267	+	elseif k == Tokens.LSQUARE
	268	+	:LSQUARE
	269	+	elseif k == Tokens.RSQUARE
	270	+	:RSQUARE
	271	+	elseif k == Tokens.LBRACE
	272	+	:LBRACE
	273	+	elseif k == Tokens.RBRACE
	274	+	:RBRACE
	275	+	elseif k == Tokens.AT_SIGN
	276	+	:ATSIGN
	277	+	elseif k == Tokens.DOT
	278	+	:DOT
	279	+	elseif k === Tokens.ABSTRACT
	280	+	return :ABSTRACT
	281	+	elseif k === Tokens.BAREMODULE
	282	+	return :BAREMODULE
	283	+	elseif k === Tokens.BEGIN
	284	+	return :BEGIN
	285	+	elseif k === Tokens.BREAK
	286	+	return :BREAK
	287	+	elseif k === Tokens.CATCH
	288	+	return :CATCH
	289	+	elseif k === Tokens.CONST
	290	+	return :CONST
	291	+	elseif k === Tokens.CONTINUE
	292	+	return :CONTINUE
	293	+	elseif k === Tokens.DO
	294	+	return :DO
	295	+	elseif k === Tokens.ELSE
	296	+	return :ELSE
	297	+	elseif k === Tokens.ELSEIF
	298	+	return :ELSEIF
	299	+	elseif k === Tokens.END
	300	+	return :END
	301	+	elseif k === Tokens.EXPORT
	302	+	return :EXPORT
	303	+	elseif k === Tokens.FINALLY
	304	+	return :FINALLY
	305	+	elseif k === Tokens.FOR
	306	+	return :FOR
	307	+	elseif k === Tokens.FUNCTION
	308	+	return :FUNCTION
	309	+	elseif k === Tokens.GLOBAL
	310	+	return :GLOBAL
	311	+	elseif k === Tokens.IF
	312	+	return :IF
	313	+	elseif k === Tokens.IMPORT
	314	+	return :IMPORT
	315	+	elseif k === Tokens.IMPORTALL
	316	+	return :importall
	317	+	elseif k === Tokens.LET
	318	+	return :LET
	319	+	elseif k === Tokens.LOCAL
	320	+	return :LOCAL
	321	+	elseif k === Tokens.MACRO
	322	+	return :MACRO
	323	+	elseif k === Tokens.MODULE
	324	+	return :MODULE
	325	+	elseif k === Tokens.MUTABLE
	326	+	return :MUTABLE
	327	+	elseif k === Tokens.NEW
	328	+	return :NEW
	329	+	elseif k === Tokens.OUTER
	330	+	return :OUTER
	331	+	elseif k === Tokens.PRIMITIVE
	332	+	return :PRIMITIVE
	333	+	elseif k === Tokens.QUOTE
	334	+	return :QUOTE
	335	+	elseif k === Tokens.RETURN
	336	+	return :RETURN
	337	+	elseif k === Tokens.STRUCT
	338	+	return :STRUCT
	339	+	elseif k === Tokens.TRY
	340	+	return :TRY
	341	+	elseif k === Tokens.TYPE
	342	+	return :TYPE
	343	+	elseif k === Tokens.USING
	344	+	return :USING
	345	+	elseif k === Tokens.WHILE
	346	+	return :WHILE
	347	+	elseif k === Tokens.INTEGER
	348	+	return :INTEGER
	349	+	elseif k === Tokens.BIN_INT
	350	+	return :BININT
	351	+	elseif k === Tokens.HEX_INT
	352	+	return :HEXINT
	353	+	elseif k === Tokens.OCT_INT
	354	+	return :OCTINT
	355	+	elseif k === Tokens.FLOAT
	356	+	return :FLOAT
	357	+	elseif k === Tokens.STRING
	358	+	return :STRING
	359	+	elseif k === Tokens.TRIPLE_STRING
	360	+	return :TRIPLESTRING
	361	+	elseif k === Tokens.CHAR
	362	+	return :CHAR
	363	+	elseif k === Tokens.CMD
	364	+	return :CMD
	365	+	elseif k === Tokens.TRIPLE_CMD
	366	+	return :TRIPLECMD
	367	+	elseif k === Tokens.TRUE
	368	+	return :TRUE
	369	+	elseif k === Tokens.FALSE
	370	+	return :FALSE
	371	+	elseif k === Tokens.ENDMARKER
	372	+	return :errortoken
	373	+	else
	374	+	error("")
	375	+	end
	376	+	end

@@ -74,30 +74,39 @@

        end
    end

    # there are interpolations in the string
    if prefixed != false || iscmd
        t_str = val(ps.t, ps)
    isinterpolated = false

    t_str = val(ps.t, ps)
    if istrip && length(t_str) == 6
        if iscmd
            return wrapwithcmdmacro(EXPR(:TRIPLESTRING , sfullspan, sspan, ""))
        else
            return EXPR(:TRIPLESTRING, sfullspan, sspan, "")
        end
    elseif length(t_str) == 2
        if iscmd
            return wrapwithcmdmacro(EXPR(:STRING , sfullspan, sspan, ""))
        else
            return EXPR(:STRING, sfullspan, sspan, "")
        end
    elseif prefixed != false || iscmd
        _val = istrip ? t_str[4:prevind(t_str, sizeof(t_str), 3)] : t_str[2:prevind(t_str, sizeof(t_str))]
        if iscmd
            _val = replace(_val, "\\\\" => "\\")
            _val = replace(_val, "\\`" => "`")
        else
            if endswith(_val, "\\\\")
                _val = _val[1:end - 1]
            end
            _val = replace(_val, "\\\"" => "\"")
            _val = unescape_prefixed(_val)
        end
        expr = mLITERAL(sfullspan, sspan, _val, kindof(ps.t))
        expr = EXPR(istrip ? :TRIPLESTRING : :STRING, sfullspan, sspan, _val)
        if istrip
            adjust_lcp(expr)
            ret = EXPR(StringH, EXPR[expr], sfullspan, sspan)
            ret = EXPR(:string, EXPR[expr], nothing, sfullspan, sspan)
        else
            return expr
            return iscmd ? wrapwithcmdmacro(expr) : expr
        end
    else
        ret = EXPR(StringH, EXPR[], sfullspan, sspan)
        str2 = val(ps.t, ps)
        input = IOBuffer(str2)
        ret = EXPR(:string, EXPR[], EXPR[], sfullspan, sspan)
        input = IOBuffer(t_str)
        startbytes = istrip ? 3 : 1
        seek(input, startbytes)
        b = IOBuffer()

@@ -109,38 +118,48 @@

                write(b, c)
                write(b, read(input, Char))
            elseif c == '$'
                isinterpolated = true
                lspan = position(b)
                str = tostr(b)
                ex = mLITERAL(lspan + startbytes, lspan + startbytes, str, Tokens.STRING)
                push!(ret, ex)
                ex = EXPR(:STRING, lspan + startbytes, lspan + startbytes, str)
                if position(input) == (istrip ? 3 : 1) + 1
                    # Need to add empty :STRING at start to account for \"
                    pushtotrivia!(ret, ex)
                elseif !isempty(str)
                    push!(ret, ex)
                end
                istrip && adjust_lcp(ex)
                startbytes = 0
                op = mOPERATOR(1, 1, Tokens.EX_OR, false)
                op = EXPR(:OPERATOR, 1, 1, "\$")
                if peekchar(input) == '('
                    skip(input, 1) # skip past '('
                    lpfullspan = -position(input)
                    if iswhitespace(peekchar(input)) || peekchar(input) === '#'
                        read_ws_comment(input, readchar(input))
                    end
                    lparen = mPUNCTUATION(Tokens.LPAREN, lpfullspan + position(input) + 1, 1)
                    rparen = mPUNCTUATION(Tokens.RPAREN, 1, 1)
                    lparen = EXPR(:LPAREN, lpfullspan + position(input) + 1, 1)
                    rparen = EXPR(:RPAREN, 1, 1)

                    ps1 = ParseState(input)

                    if kindof(ps1.nt) === Tokens.RPAREN
                        call = mUnaryOpCall(op, EXPR(InvisBrackets, EXPR[lparen, rparen]))
                        push!(ret, call)
                        push!(ret, EXPR(:ERRORTOKEN, EXPR[], nothing))
                        pushtotrivia!(ret, op)
                        pushtotrivia!(ret, lparen)
                        pushtotrivia!(ret, rparen)
                        skip(input, 1)
                    else
                        interp = @closer ps1 :paren parse_expression(ps1)
                        call = mUnaryOpCall(op, EXPR(InvisBrackets, EXPR[lparen, interp, rparen]))
                        push!(ret, call)
                        interp_val = @closer ps1 :paren parse_expression(ps1)
                        push!(ret, interp_val)
                        pushtotrivia!(ret, op)
                        pushtotrivia!(ret, lparen)
                        pushtotrivia!(ret, rparen)
                        seek(input, ps1.nt.startbyte + 1)
                    end
                    # Compared to flisp/JuliaParser, we have an extra lookahead token,
                    # so we need to back up one here
                elseif Tokenize.Lexers.iswhitespace(peekchar(input)) || peekchar(input) === '#'
                    push!(ret, mErrorToken(ps, op, StringInterpolationWithTrailingWhitespace))
                    pushtotrivia!(ret, mErrorToken(ps, op, StringInterpolationWithTrailingWhitespace))
                else
                    pos = position(input)
                    ps1 = ParseState(input)

@@ -152,8 +171,8 @@

                    end
                    # Attribute trailing whitespace to the string
                    t = adjustspan(t)
                    call = mUnaryOpCall(op, t)
                    push!(ret, call)
                    push!(ret, t)
                    pushtotrivia!(ret, op)
                    seek(input, pos + t.fullspan)
                end
            else

@@ -166,23 +185,27 @@

        lspan = position(b)
        if b.size == 0
            ex = mErrorToken(ps, Unknown)
            push!(ret, ex)
        else
            str = tostr(b)
            if istrip
                str = str[1:prevind(str, lastindex(str), 3)]
                # only mark non-interpolated triple strings
                ex = mLITERAL(lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str, length(ret) == 0 ? Tokens.TRIPLE_STRING : Tokens.STRING)
                ex = EXPR(length(ret) == 0 ? :TRIPLESTRING : :STRING, lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str)
                adjust_lcp(ex, true)
            else
                str = str[1:prevind(str, lastindex(str))]
                ex = mLITERAL(lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str, Tokens.STRING)
                ex = EXPR(:STRING, lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str)
            end
            if isempty(str)
                pushtotrivia!(ret, ex)
            else
                push!(ret, ex)
            end
        end
        push!(ret, ex)

    end

    single_string_T = (Tokens.STRING, kindof(ps.t))
    single_string_T = (:STRING, :TRIPLESTRING, literalmap(kindof(ps.t)))
    if istrip
        if lcp !== nothing && !isempty(lcp)
            for expr in exprs_to_adjust

@@ -195,18 +218,18 @@

            end
        end
        # Drop leading newline
        if isliteral(ret.args[1]) && kindof(ret.args[1]) in single_string_T &&
        if !isempty(ret.args) && isliteral(ret.args[1]) && headof(ret.args[1]) in single_string_T &&
                !isempty(valof(ret.args[1])) && valof(ret.args[1])[1] == '\n'
            ret.args[1] = dropleadlingnewline(ret.args[1])
        end
    end

    if (length(ret.args) == 1 && isliteral(ret.args[1]) && kindof(ret.args[1]) in single_string_T)
    if (length(ret.args) == 1 && isliteral(ret.args[1]) && headof(ret.args[1]) in single_string_T) && !isinterpolated
        ret = ret.args[1]
    end
    update_span!(ret)

    return ret
    return iscmd ? wrapwithcmdmacro(ret) : ret
end

function adjustspan(x::EXPR)

@@ -214,4 +237,70 @@

214	237		return x
215	238		end
216	239
217		-	dropleadlingnewline(x::EXPR) = mLITERAL(x.fullspan, x.span, valof(x)[2:end], kindof(x))
	240	+	dropleadlingnewline(x::EXPR) = EXPR(headof(x), x.fullspan, x.span, valof(x)[2:end])
	241	+
	242	+	wrapwithcmdmacro(x) =EXPR(:macrocall, EXPR[EXPR(:globalrefcmd, 0, 0), EXPR(:NOTHING, 0, 0), x])
	243	+
	244	+	"""
	245	+	parse_prefixed_string_cmd(ps::ParseState, ret::EXPR)
	246	+
	247	+	Parse prefixed strings and commands such as `pre"text"`.
	248	+	"""
	249	+	function parse_prefixed_string_cmd(ps::ParseState, ret::EXPR)
	250	+	arg = parse_string_or_cmd(next(ps), ret)
	251	+
	252	+	if ret.head === :IDENTIFIER && valof(ret) == "var" && isstringliteral(arg) && VERSION > v"1.3.0-"
	253	+	return EXPR(:NONSTDIDENTIFIER, EXPR[ret, arg], nothing)
	254	+	elseif headof(arg) === :macrocall && headof(arg.args[1]) === :globalrefcmd
	255	+	mname = EXPR(:IDENTIFIER, ret.fullspan, ret.span, string("@", valof(ret), "_cmd")) # NOTE: sizeof(valof(mname)) != mname.span
	256	+	return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0), arg.args[3]], nothing)
	257	+	elseif is_getfield(ret)
	258	+	if headof(ret.args[2]) === :quote \|\| headof(ret.args[2]) === :quotenode
	259	+	str_type = valof(ret.args[2].args[1]) isa String ? valof(ret.args[2].args[1]) : "" # to handle some malformed case
	260	+	ret.args[2].args[1] = setparent!(EXPR(:IDENTIFIER, ret.args[2].args[1].fullspan, ret.args[2].args[1].span, string("@", str_type, "_str")), ret.args[2])
	261	+	else
	262	+	str_type = valof(ret.args[2]) isa String ? valof(ret.args[2]) : "" # to handle some malformed case
	263	+	ret.args[2] = EXPR(:IDENTIFIER, ret.args[2].fullspan, ret.args[2].span, string("@", str_type, "_str"))
	264	+	end
	265	+
	266	+	return EXPR(:macrocall, EXPR[ret, EXPR(:NOTHING, 0, 0), arg], nothing)
	267	+	else
	268	+	return EXPR(:macrocall, EXPR[EXPR(:IDENTIFIER, ret.fullspan, ret.span, string("@", valof(ret), "_str")), EXPR(:NOTHING, 0, 0), arg], nothing)
	269	+	end
	270	+	end
	271	+
	272	+	function unescape_prefixed(str)
	273	+	edits = UnitRange{Int}[]
	274	+	start = -1
	275	+	for (i, c) in enumerate(str)
	276	+	if start == -1 && c === '\\'
	277	+	start = i
	278	+	end
	279	+	if start > -1
	280	+	if c === '\\'
	281	+	elseif c === '\"'
	282	+	push!(edits, start:i)
	283	+	start = -1
	284	+	else
	285	+	start = -1
	286	+	end
	287	+	end
	288	+	end
	289	+
	290	+	if !isempty(edits) \|\| start > -1
	291	+	str1 = deepcopy(str)
	292	+	if start > -1
	293	+	# slashes preceding closing '"'
	294	+	n = div(length(start:length(str)), 2) - 1
	295	+	str1 = string(str1[1:prevind(str1, start)], repeat("\\", n + 1))
	296	+	end
	297	+
	298	+	for e in reverse(edits)
	299	+	n = div(length(e), 2) - 1
	300	+	str1 = string(str1[1:prevind(str1, first(e))], string(repeat("\\", n), "\""), str1[nextind(str1, last(e)):lastindex(str1)])
	301	+
	302	+	end
	303	+	return str1
	304	+	end
	305	+	return str
	306	+	end

@@ -13,15 +13,16 @@

    (ps.closer.inwhere && kindof(ps.nt) === Tokens.WHERE) ||
    (ps.closer.inwhere && ps.closer.ws && kindof(ps.t) === Tokens.RPAREN && isoperator(ps.nt) && precedence(ps.nt) < DeclarationOp) ||
    (ps.closer.precedence > WhereOp && (
        (kindof(ps.nt) === Tokens.LPAREN && !(ps.t.kind === Tokens.EX_OR)) ||
        (kindof(ps.nt) === Tokens.LPAREN && !(kindof(ps.t) === Tokens.EX_OR)) ||
        kindof(ps.nt) === Tokens.LBRACE ||
        kindof(ps.nt) === Tokens.LSQUARE ||
        (kindof(ps.nt) === Tokens.STRING && isemptyws(ps.ws)) ||
        ((kindof(ps.nt) === Tokens.RPAREN || kindof(ps.nt) === Tokens.RSQUARE) && isidentifier(ps.nt))
    )) ||
    (iscomma(ps.nt) && ps.closer.precedence > 0) ||
    (iscomma(ps.nt) && ps.closer.precedence > AssignmentOp) ||
    kindof(ps.nt) === Tokens.ENDMARKER ||
    (ps.closer.comma && iscomma(ps.nt)) ||
    (ps.closer.tuple && (iscomma(ps.nt) || isassignment(ps.nt))) ||
    (ps.closer.tuple && (iscomma(ps.nt) || isassignmentop(ps.nt))) ||
    (kindof(ps.nt) === Tokens.FOR && ps.closer.precedence > -1) ||
    (ps.closer.block && kindof(ps.nt) === Tokens.END) ||
    (ps.closer.paren && kindof(ps.nt) === Tokens.RPAREN) ||

@@ -41,7 +42,7 @@

        !(kindof(ps.nt) === Tokens.DO) &&
        !(
            (isbinaryop(ps.nt) && !(ps.closer.wsop && isemptyws(ps.nws) && isunaryop(ps.nt) && precedence(ps.nt) > 7)) ||
            (isunaryop(ps.t) && kindof(ps.ws) == WS && ps.lt.kind !== CSTParser.Tokens.COLON)
            (isunaryop(ps.t) && kindof(ps.ws) == WS && kindof(ps.lt) !== CSTParser.Tokens.COLON)
        )) ||
    (ps.closer.unary && (kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT, Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE) && isidentifier(ps.nt)))
end

@@ -201,31 +202,8 @@

end



isidentifier(x::EXPR) = typof(x) === IDENTIFIER || typof(x) === NONSTDIDENTIFIER

isunarycall(x::EXPR) = typof(x) === UnaryOpCall
isbinarycall(x::EXPR) = typof(x) === BinaryOpCall
iswherecall(x::EXPR) = typof(x) === WhereOpCall
isdeclaration(x::EXPR) = isbinarycall(x) && is_decl(x[2])
isinterpolant(x::EXPR) = isunarycall(x) && is_exor(x[1])
istuple(x::EXPR) = typof(x) === TupleH
is_either_id_op_interp(x::EXPR) = isidentifier(x) || isoperator(x) || isinterpolant(x)
is_splat(x::EXPR) = isunarycall(x) && is_dddot(x[2])


isliteral(x::EXPR) = typof(x) === LITERAL
iskw(x::EXPR) = typof(x) === KEYWORD # TODO: should change to `iskeyword`
ispunctuation(x::EXPR) = typof(x) === PUNCTUATION

isstring(x) = typof(x) === StringH || (isliteral(x) && (kindof(x) === Tokens.STRING || kindof(x) === Tokens.TRIPLE_STRING))
is_integer(x) = isliteral(x) && kindof(x) === Tokens.INTEGER
is_float(x) = isliteral(x) && kindof(x) === Tokens.FLOAT
is_number(x) = isliteral(x) && (kindof(x) === Tokens.INTEGER || kindof(x) === Tokens.FLOAT)
is_nothing(x) = isliteral(x) && kindof(x) === Tokens.NOTHING

isajuxtaposition(ps::ParseState, ret::EXPR) = ((is_number(ret) && (isidentifier(ps.nt) || kindof(ps.nt) === Tokens.LPAREN || kindof(ps.nt) === Tokens.CMD || kindof(ps.nt) === Tokens.STRING || kindof(ps.nt) === Tokens.TRIPLE_STRING)) ||
        ((typof(ret) === UnaryOpCall && is_prime(ret.args[2]) && isidentifier(ps.nt)) ||
isajuxtaposition(ps::ParseState, ret::EXPR) = ((isnumber(ret) && (isidentifier(ps.nt) || kindof(ps.nt) === Tokens.LPAREN || kindof(ps.nt) === Tokens.CMD || kindof(ps.nt) === Tokens.STRING || kindof(ps.nt) === Tokens.TRIPLE_STRING)) ||
        ((is_prime(ret.head) && isidentifier(ps.nt)) ||
        ((kindof(ps.t) === Tokens.RPAREN || kindof(ps.t) === Tokens.RSQUARE) && (isidentifier(ps.nt) || kindof(ps.nt) === Tokens.CMD)) ||
        ((kindof(ps.t) === Tokens.STRING || kindof(ps.t) === Tokens.TRIPLE_STRING) && (kindof(ps.nt) === Tokens.STRING || kindof(ps.nt) === Tokens.TRIPLE_STRING)))) || ((kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT) || kindof(ps.t) in (Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE)) && isidentifier(ps.nt))


@@ -237,55 +215,10 @@

`ParseState`, or exists as a (sub) expression of `x`.
"""
function has_error(x::EXPR)
    return typof(x) == ErrorToken || (x.args !== nothing && any(has_error, x.args))
    return headof(x) == :errortoken || (x.args !== nothing && any(has_error, x.args)) || (x.trivia !== nothing && any(has_error, x.trivia))
end
has_error(ps::ParseState) = ps.errored

# When using the FancyDiagnostics package, Meta.parse is the
# same as CSTParser.parse. Manually call the flisp parser here
# to make sure we test what we want, even when people load the
# FancyDiagnostics package.
function flisp_parse(str::AbstractString, pos::Int; greedy::Bool=true, raise::Bool=true)
    if VERSION < v"1.6-DEV"
        bstr = String(str)
        ex, pos = ccall(:jl_parse_string, Any,
                        (Ptr{UInt8}, Csize_t, Int32, Int32),
                        bstr, sizeof(bstr), pos - 1, greedy ? 1 : 0)
    else
        filename = "none"
        rule = greedy ? :statement : :atom
        ex, pos = Core.Compiler.fl_parse(str, filename, pos - 1, rule)
    end
    if raise && isa(ex, Expr) && ex.head === :error
        throw(Meta.ParseError(ex.args[1]))
    end
    if ex === ()
        raise && throw(Meta.ParseError("end of input"))
        ex = Expr(:error, "end of input")
    end
    # pos is zero-based byte offset
    return ex, pos + 1
end

function flisp_parse(str::AbstractString; raise::Bool=true)
    ex, pos = flisp_parse(str, 1, greedy=true, raise=raise)
    if isa(ex, Expr) && ex.head === :error
        return ex
    end
    if !(pos > ncodeunits(str))
        raise && throw(Meta.ParseError("extra token after end of expression"))
        return Expr(:error, "extra token after end of expression")
    end
    return ex
end

function flisp_parse(stream::IO; greedy::Bool=true, raise::Bool=true)
    pos = position(stream)
    ex, Δ = flisp_parse(read(stream, String), 1, greedy=greedy, raise=raise)
    seek(stream, pos + Δ - 1)
    return ex
end

using Base.Meta

function norm_ast(a::Any)

@@ -328,13 +261,14 @@

    return a
end

function flisp_parsefile(str, display=true)
    io = IOBuffer(str)
function flisp_parsefile(str, display = true)
    pos = 1
    failed = false
    x1 = Expr(:file)
    try
        while !eof(io)
            push!(x1.args, flisp_parse(io))
        while pos <= sizeof(str)
            x, pos = Meta.parse(str, pos)
            push!(x1.args, x)
        end
    catch er
        isa(er, InterruptException) && rethrow(er)

@@ -359,9 +293,9 @@

    if length(x.args) > 0 && is_nothing(x.args[1])
        popfirst!(x.args)
    end
    if length(x.args) > 0 && is_nothing(x.args[end])
        pop!(x.args)
    end
    # if length(x.args) > 0 && is_nothing(x.args[end])
    #     pop!(x.args)
    # end
    x0 = norm_ast(Expr(x))
    x0, has_error(ps), sp
end

@@ -386,8 +320,8 @@

        cumfail = 0
        printstyled(file, color=:green)
        println()
        c0, c1 = CSTParser.compare(x0, x1)
        printstyled(string("    ", c0), bold=true, color=:ligth_red)
        c0, c1 = compare(x0, x1)
        printstyled(string("    ", c0), bold = true, color = :light_red)
        println()
        printstyled(string("    ", c1), bold=true, color=:light_green)
        println()

@@ -473,21 +407,138 @@

473	407		end
474	408		end
475	409
	410	+	# code for updating CST
	411	+	"""
	412	+	firstdiff(s0::AbstractString, s1::AbstractString)
	413	+
	414	+	Returns the last byte index, i, for which s0 and s1 are the same such that:
	415	+	`s0[1:i] == s1[1:i]`
	416	+	"""
	417	+	function firstdiff(s0::AbstractString, s1::AbstractString)
	418	+	minlength = min(sizeof(s0), sizeof(s1))
	419	+	@inbounds for i in 1:minlength
	420	+	if codeunits(s0)[i] !== codeunits(s1)[i]
	421	+	return i - 1 # This could return a non-commencing byte of a multi-byte unicode sequence.
	422	+	end
	423	+	end
	424	+	return minlength
	425	+	end
	426	+
	427	+	"""
	428	+	revfirstdiff(s0::AbstractString, s1::AbstractString)
	429	+
	430	+	Reversed version of firstdiff but returns two indices, one for each string.
	431	+	"""
	432	+	function revfirstdiff(s0::AbstractString, s1::AbstractString)
	433	+	minlength = min(sizeof(s0), sizeof(s1))
	434	+	@inbounds for i in 0:minlength - 1
	435	+	if codeunits(s0)[end - i] !== codeunits(s1)[end - i]
	436	+	return sizeof(s0) - i, sizeof(s1) - i# This could return a non-commencing byte of a multi-byte unicode sequence.
	437	+	end
	438	+	end
	439	+	return 1, 1
	440	+	end
	441	+
	442	+	"""
	443	+	find_arg_at(x, i)
	444	+
	445	+	Returns the index of the node of `x` within which the byte offset `i` falls.
	446	+	"""
	447	+	function find_arg_at(x::CSTParser.EXPR, i)
	448	+	@assert i <= x.fullspan
	449	+	offset = 0
	450	+	for (cnt, a) in enumerate(x.args)
	451	+	if i <= offset + a.fullspan
	452	+	return cnt
	453	+	end
	454	+	offset += a.fullspan
	455	+	end
	456	+	error()
	457	+	end
	458	+
	459	+	comp(x, y) = x == y
	460	+	function comp(x::CSTParser.EXPR, y::CSTParser.EXPR)
	461	+	comp(x.head, y.head) &&
	462	+	x.span == y.span &&
	463	+	x.fullspan == y.fullspan &&
	464	+	x.val == y.val &&
	465	+	length(x) == length(y) &&
	466	+	all(comp(x[i], y[i]) for i = 1:length(x))
	467	+	end
	468	+
	469	+	function minimal_reparse(s0, s1, x0 = CSTParser.parse(s0, true), x1 = CSTParser.parse(s1, true); inds = false)
	470	+	i0 = firstdiff(s0, s1)
	471	+	i1, i2 = revfirstdiff(s0, s1)
	472	+	# Find unaffected expressions at start
	473	+	# CST should be unaffected (and able to be copied across) up to this point,
	474	+	# but we need to check.
	475	+	r1 = 1:min(find_arg_at(x0, i0) - 1, length(x0.args), find_arg_at(x1, i0) - 1)
	476	+	for i = 1:min(find_arg_at(x0, i0) - 1, find_arg_at(x1, i0) - 1)
	477	+	if x0.args[i].fullspan !== x1.args[i].fullspan
	478	+	r1 = 1:(i-1)
	479	+	break
	480	+	end
	481	+	r1 = 1:i
	482	+	end
	483	+	# we can re-use x0.args[r1]
	484	+
	485	+	# assume we'll just use x1.args from here on
	486	+	r2 = (last(r1) + 1):length(x1.args)
	487	+	r3 = 0:-1
	488	+
	489	+	# though we now check whether there is a sequence at the end of x0.args and
	490	+	# x1.args that match
	491	+	for i = 0:min(last(r1), length(x0.args), length(x1.args)) - 1
	492	+	# if x0.args[end - i].fullspan !== x1.args[end - i].fullspan \|\|
	493	+	# headof(x0.args[end-i]) == :errortoken ? a : !comp(x0.args[end - i].head, x1.args[end - i].head)
	494	+	if !comp(x0.args[end - i], x1.args[ end - i])
	495	+	r2 = first(r2):length(x1.args) - i
	496	+	r3 = length(x0.args) .+ ((-i + 1):0)
	497	+	break
	498	+	end
	499	+	end
	500	+	inds && return r1, r2, r3
	501	+	x2 = CSTParser.EXPR(x0.head, CSTParser.EXPR[
	502	+	x0.args[r1]
	503	+	x1.args[r2]
	504	+	x0.args[r3]
	505	+	], nothing)
	506	+	return x2
	507	+	end
	508	+
	509	+	# Quick and very dirty comparison of two EXPR, makes extra effort for :errortokens
	510	+	function quick_comp(a::EXPR, b::EXPR)
	511	+	a.fullspan == b.fullspan &&
	512	+	headof(a) === :errortoken ? headof(b) === :errortoken && length(a.args) > 0 && length(a.args) == length(b.args) && quick_comp(first(a.args), first(b.args)) :
	513	+	comp(headof(a), headof(b))
	514	+	end
	515	+
476	516		"""
477	517		check_span(x, neq = [])
478	518
479	519		Recursively checks whether the span of an expression equals the sum of the span
480	520		of its components. Returns a vector of failing expressions.
481	521		"""
482		-	function check_span(x::EXPR, neq=[])
483		-	(ispunctuation(x) \|\| isidentifier(x) \|\| iskw(x) \|\| isoperator(x) \|\| isliteral(x) \|\| typof(x) == StringH) && return neq
	522	+	function check_span(x::EXPR, neq = [])
	523	+	(ispunctuation(x) \|\| isidentifier(x) \|\| iskeyword(x) \|\| isoperator(x) \|\| isliteral(x) \|\| headof(x) == :string) && return neq
484	524
485	525		s = 0
486		-	for a in x.args
487		-	check_span(a, neq)
488		-	s += a.fullspan
	526	+	if x.args !== nothing
	527	+	for a in x.args
	528	+	check_span(a, neq)
	529	+	s += a.fullspan
	530	+	end
	531	+	end
	532	+	if hastrivia(x)
	533	+	for a in x.trivia
	534	+	check_span(a, neq)
	535	+	s += a.fullspan
	536	+	end
	537	+	end
	538	+	if x.head isa EXPR
	539	+	s += x.head.fullspan
489	540		end
490		-	if length(x.args) > 0 && s != x.fullspan
	541	+	if length(x) > 0 && s != x.fullspan
491	542		push!(neq, x)
492	543		end
493	544		neq

@@ -507,11 +558,11 @@

Attempt to get a string representation of a nodeless expression.
"""
function str_value(x)
    if typof(x) === IDENTIFIER || typof(x) === LITERAL
    if headof(x) === :IDENTIFIER || isliteral(x)
        return valof(x)
    elseif isidentifier(x)
        valof(x.args[2])
    elseif typof(x) === OPERATOR || typof(x) === MacroName
    elseif isoperator(x)
        return string(Expr(x))
    else
        return ""

@@ -627,12 +678,8 @@

    return true
end

"""
    is_getfield(x::EXPR)

Is this an expression of the form `a.b`.
"""
is_getfield(x::EXPR) = isbinarycall(x) && length(x) == 3 && kindof(x[2]) === Tokens.DOT


"""
    disallowednumberjuxt(ret::EXPR)

@@ -640,45 +687,33 @@

Does this number literal end in a decimal and so cannot precede a paren for
implicit multiplication?
"""
disallowednumberjuxt(ret::EXPR) = is_number(ret) && last(valof(ret)) == '.'
disallowednumberjuxt(ret::EXPR) = isnumber(ret) && last(valof(ret)) == '.'


nexttokenstartsdocstring(ps::ParseState) = isidentifier(ps.nt) && val(ps.nt, ps) == "doc" && (kindof(ps.nnt) === Tokens.STRING || kindof(ps.nnt) === Tokens.TRIPLE_STRING)

"""
    is_wrapped_assignment(x::EXPR)
    
Is `x` an assignment expression, ignoring any surrounding parentheses.
"""
is_wrapped_assignment(x::EXPR) = is_assignment(x) || (isbracketed(x) && is_wrapped_assignment(x.args[2]))

"""
    is_range(x::EXPR)

Is `x` a valid iterator for use in `for` loops or generators?
"""
is_range(x::EXPR) = isbinarycall(x) && (is_eq(x.args[2]) || is_in(x.args[2]) || is_elof(x.args[2]))

"""
    _do_kw_convert(ps::ParseState, a::EXPR)

Should `a` be converted to a keyword-argument expression?
"""
_do_kw_convert(ps::ParseState, a::EXPR) = !ps.closer.brace && is_assignment(a)
_do_kw_convert(ps::ParseState, a::EXPR) = !ps.closer.brace && isassignment(a)

"""
    _kw_convert(ps::ParseState, a::EXPR)

Converted an assignment expression to a keyword-argument expression.
"""
_kw_convert(a::EXPR) = EXPR(Kw, EXPR[a.args[1], a.args[2], a.args[3]], a.fullspan, a.span)
_kw_convert(x::EXPR) = EXPR(:kw, EXPR[x.args[1], x.args[2]], EXPR[x.head], x.fullspan, x.span)

"""
    convertsigtotuple(sig::EXPR)

When parsing a function or macro signature, should it be converted to a tuple?
"""
convertsigtotuple(sig::EXPR) = isbracketed(sig) && !(istuple(sig.args[2]) || (typof(sig.args[2]) === Block) || is_splat(sig.args[2]))
convertsigtotuple(sig::EXPR) = isbracketed(sig) && !(istuple(sig.args[1]) || (headof(sig.args[1]) === :block) || issplat(sig.args[1]))

"""
    docable(head)

@@ -686,16 +721,11 @@

When parsing a block of expressions, can documentation be attached? Prefixed docs at the
top-level are handled within `parse(ps::ParseState, cont = false)`.
"""
docable(head) = head === Begin || head === ModuleH || head === BareModule || head === Quote
docable(head) = head === :begin || head === :module || head === :baremodule || head === :quote


should_negate_number_literal(ps::ParseState, op::EXPR) = (is_plus(op) || is_minus(op)) && (kindof(ps.nt) === Tokens.INTEGER || kindof(ps.nt) === Tokens.FLOAT) && isemptyws(ps.ws) && kindof(ps.nnt) != Tokens.CIRCUMFLEX_ACCENT

isbracketed(x::EXPR) = typof(x) === InvisBrackets # Assumption that x has 3 args, doesn't need checking?

unwrapbracket(x::EXPR) = isbracketed(x) ? unwrapbracket(x[2]) : x

isbeginorblock(x::EXPR) = typof(x) === Begin || typof(unwrapbracket(x)) == Block

"""
    can_become_comparison(x::EXPR)

@@ -703,7 +733,7 @@

Is `x` a binary comparison call (e.g. `a < b`) that can be extended to include more
arguments?
"""
can_become_comparison(x::EXPR) = isbinarycall(x) && (precedence(x.args[2]) == ComparisonOp || is_issubt(x.args[2]) || is_issupt(x.args[2]))
can_become_comparison(x::EXPR) = (isoperator(x.head) && comp_prec(valof(x.head)) && length(x.args) > 1) || (x.head === :call && isoperator(x.args[1]) && comp_prec(valof(x.args[1])) && length(x.args) > 2 && !hastrivia(x))

"""
    can_become_chain(x::EXPR, op::EXPR)

@@ -711,10 +741,19 @@

Is `x` a binary call for `+` or `*` that can be extended to include more
arguments?
"""
can_become_chain(x::EXPR, op::EXPR) = isbinarycall(x) && (is_star(op) || is_plus(op)) && kindof(op) == kindof(x.args[2]) && !x.args[2].dot && x.args[2].span > 0
can_become_chain(x::EXPR, op::EXPR) = isbinarycall(x) && (is_star(op) || is_plus(op)) && valof(op) == valof(x.args[2]) && !isdotted(x.args[2]) && x.args[2].span > 0


macro cst_str(x)
    CSTParser.parse(x)
end

function issuffixableliteral(ps::ParseState, x::EXPR) 
    isidentifier(ps.nt) && isemptyws(ps.ws) && ismacrocall(x) && (valof(x.args[1]) isa String && (endswith(valof(x.args[1]), "_str") || endswith(valof(x.args[1]), "_cmd")))
end

function loop_check(ps, prevpos)
    if position(ps) <= prevpos
    if position(ps) <= prevpos && ps.nt.kind !== Tokens.ENDMARKER
        throw(CSTInfiniteLoop("Infinite loop at $ps"))
    else
        position(ps)

@@ -1,33 +1,21 @@

function Base.show(io::IO, x::EXPR, offset=0, d=0, er=false)
    T = typof(x)
    c =  T === ErrorToken || er ? :red : :normal
function Base.show(io::IO, x::EXPR, offset = 0, d = 0, er = false)
    T = headof(x)
    c =  T === :errortoken || er ? :red : :normal
    # Print span as 1-based range of the source string. This presentation is
    # simple to understand when strings are presented to CSTParser.parse().
    print(io, lpad(offset + 1, 3), ":", rpad(offset + x.fullspan, 3), " ")
    if isidentifier(x)
        printstyled(io, " "^d, typof(x) == NONSTDIDENTIFIER ? valof(x.args[2]) : valof(x), color=:yellow)
        printstyled(io, " "^d, headof(x) == :NONSTDIDENTIFIER ? valof(x.args[2]) : valof(x), color = :yellow)
        x.meta !== nothing && show(io, x.meta)
        println(io)
        print(io)
    elseif isoperator(x)
        printstyled(io, " "^d, "OP: ", kindof(x), "\n", color=c)
    elseif iskw(x)
        printstyled(io, " "^d, kindof(x), "\n", color=:magenta)
        printstyled(io, " "^d, "OP: ", valof(x), color = c)
    elseif iskeyword(x)
        printstyled(io, " "^d, headof(x), color = :magenta)
    elseif ispunctuation(x)
        if kindof(x) === Tokens.LPAREN
            printstyled(io, " "^d, "(\n", color=c)
        elseif kindof(x) === Tokens.RPAREN
            printstyled(io, " "^d, ")\n", color=c)
        elseif kindof(x) === Tokens.LSQUARE
            printstyled(io, " "^d, "[\n", color=c)
        elseif kindof(x) === Tokens.RSQUARE
            printstyled(io, " "^d, "]\n", color=c)
        elseif kindof(x) === Tokens.COMMA
            printstyled(io, " "^d, ",\n", color=c)
        else
            printstyled(io, " "^d, "PUNC: ", kindof(x), "\n", color=c)
        end
        printstyled(io, " "^d, punctuationprinting[headof(x)], color = c)
    elseif isliteral(x)
        printstyled(io, " "^d, "$(kindof(x)): ", valof(x), "\n", color=c)
        printstyled(io, " "^d, "$(headof(x)): ", valof(x) === nothing ? "nothing" : valof(x), color = c)
    else
        printstyled(io, " "^d, T, color=c)
        if x.meta !== nothing

@@ -35,15 +23,25 @@

            show(io, x.meta)
            print(io, ")")
        end
        println(io)
        x.args === nothing && return
        for a in x.args
            println(io)
            show(io, a, offset, d + 1, er)
            offset += a.fullspan
        end
    end
end

const punctuationprinting = Dict(:COMMA => ",",
:LPAREN => "(",
:RPAREN => ")",
:LSQUARE => "[",
:RSQUARE => "]",
:LBRACE => "{",
:RBRACE => "}",
:ATSIGN => "@",
:DOT => ".")

struct CSTInfiniteLoop <: Exception
    msg::AbstractString
end

@@ -10,14 +10,8 @@

        if kindof(ps.nt) ∈ term_c # error handling if an unexpected closer is hit
            if kindof(ps.nt) === Tokens.ENDMARKER
                break
            elseif kindof(ps.nt) === Tokens.RPAREN
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
            elseif kindof(ps.nt) === Tokens.RBRACE
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
            elseif kindof(ps.nt) === Tokens.RSQUARE
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
            else
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
                push!(ret, mErrorToken(ps, EXPR(next(ps)), UnexpectedToken))
            end
        else
            if docable

@@ -37,22 +31,40 @@

error expression if an invalid expression is parsed (anything other than
`=`, `in`, `∈`).
"""
function parse_iterator(ps::ParseState, outer=parse_outer(ps))
    arg = @closer ps :range @closer ps :ws parse_expression(ps)
function parse_iterator(ps::ParseState, outer = parse_outer(ps))
    arg = @closer ps :range @closer ps :ws @nocloser ps :wsop parse_expression(ps)
    if !is_range(arg)
        arg = mErrorToken(ps, arg, InvalidIterator)
    else 
        arg = adjust_iter(arg)
    end
    if outer !== nothing
        arg.args[1] = setparent!(EXPR(Outer, EXPR[outer, arg.args[1]]), arg)
        arg.args[1] = setparent!(EXPR(:outer, EXPR[arg.args[1]], EXPR[outer]), arg)
        arg.fullspan += outer.fullspan
        arg.span = outer.fullspan + arg.span
    end
    return arg
end

function adjust_iter(x::EXPR)
    # Assumes x is a valid iterator
    if x.head === :call # isoperator(x.args[1]) && x.args[1].val in ("in", "∈")
        EXPR(EXPR(:OPERATOR, 0, 0, "="), EXPR[x.args[2], x.args[3]], EXPR[x.args[1]])
    else 
        x
    end
end

"""
    is_range(x::EXPR)

Is `x` a valid iterator for use in `for` loops or generators?
"""
is_range(x::EXPR) = isassignment(x) || (x.head === :call && (is_in(x.args[1]) || is_elof(x.args[1])))

function parse_outer(ps)
    if kindof(ps.nt) === Tokens.OUTER && kindof(ps.nws) !== EmptyWS && !Tokens.isoperator(kindof(ps.nnt))
        outer = INSTANCE(next(ps))
        outer = EXPR(next(ps))
    end
end


@@ -62,23 +74,12 @@

Parses a group of iterators e.g. used in a `for` loop or generator. Can allow
for a succeeding `Filter` expression.
"""
function parse_iterators(ps::ParseState, allowfilter=false)
    arg = parse_iterator(ps)
    if iscomma(ps.nt) # we've hit a comma separated list of iterators.
        arg = EXPR(Block, EXPR[arg])
        prevpos = position(ps)
        while iscomma(ps.nt)
            accept_comma(ps, arg)
            nextarg = parse_iterator(ps)
            push!(arg, nextarg)
            prevpos = loop_check(ps, prevpos)
        end
    end

    if allowfilter
        arg = parse_filter(ps, arg)
function parse_iterators(ps::ParseState, iters, trivia)
    push!(iters, parse_iterator(ps))
    if iscomma(ps.nt)
        push!(trivia, accept_comma(ps))
        parse_iterators(ps, iters, trivia)
    end
    return arg
end

"""

@@ -88,14 +89,13 @@

"""
function parse_filter(ps::ParseState, arg)
    if kindof(ps.nt) === Tokens.IF # assumes we're inside a generator
        if typof(arg) === Block
            arg = EXPR(Filter, arg.args)
        trivia = EXPR[EXPR(next(ps))]
        cond = @closer ps :range parse_expression(ps)
        if headof(arg) === :block
            arg = EXPR(:filter, EXPR[cond; arg.args], trivia)
        else
            arg = EXPR(Filter, EXPR[arg])
            arg = EXPR(:filter, EXPR[cond, arg], trivia)
        end
        push!(arg, mKEYWORD(next(ps)))
        cond = @closer ps :range parse_expression(ps)
        push!(arg, cond)
    end
    return arg
end

@@ -110,27 +110,31 @@

        arg = @closer ps :unary @closer ps :inwhere @precedence ps PowerOp parse_expression(ps)
        if istuple(arg)
            pushfirst!(arg.args, ret)
            ret = EXPR(Call, arg.args)
        elseif iswherecall(arg) && istuple(arg.args[1])
            ret = mWhereOpCall(EXPR(Call, EXPR[ret; arg.args[1].args]), arg.args[2], arg.args[3:end])
            ret = EXPR(:call, arg.args, arg.trivia)
        else
            ret = mUnaryOpCall(ret, arg)
            ret = EXPR(:call, EXPR[ret, arg], nothing)
        end
    elseif is_and(ret) || is_decl(ret) || is_exor(ret)
        arg = @precedence ps 20 parse_expression(ps)
        if is_exor(ret) && istuple(arg) && length(arg) == 3 && is_splat(arg.args[2])
            arg = EXPR(InvisBrackets, arg.args)
        if is_exor(ret) && istuple(arg) && length(arg) == 3 && issplat(arg.args[2])
            arg = EXPR(:brackets, arg.args)
        end
        ret = mUnaryOpCall(ret, arg)
    elseif is_issubt(ret) || is_issupt(ret)
        arg = @precedence ps PowerOp parse_expression(ps)
        ret = EXPR(Call, EXPR[ret; arg.args])
        ret = EXPR(ret, EXPR[arg], nothing)
    else
        !ismacro && typof(ret) === MacroName && (ismacro = true)
        args = EXPR[ret, mPUNCTUATION(next(ps))]
        @closeparen ps @default ps parse_comma_sep(ps, args, !ismacro)
        accept_rparen(ps, args)
        ret = EXPR(ismacro ? MacroCall : Call, args)
        !ismacro && ismacroname(ret) && (ismacro = true)
        syntaxcall = is_issubt(ret) || is_issupt(ret)
        args = if syntaxcall
            EXPR[]
        elseif ismacro
            EXPR[ret, EXPR(:NOTHING, 0, 0)]
        else
            EXPR[ret]
        end
        # args = ismacro ? EXPR[ret, EXPR(:NOTHING, 0, 0)] : EXPR[ret] 
        trivia = EXPR[EXPR(next(ps))]
        @closeparen ps @default ps parse_comma_sep(ps, args, trivia, !ismacro, insert_params_at = ismacro ? 3 : 2)
        accept_rparen(ps, trivia)
        ret = EXPR(ismacro ? :macrocall : syntaxcall ? ret : :call, args, trivia)
    end
    return ret
end

@@ -139,7 +143,7 @@

Parses a comma separated list, optionally allowing for conversion of 
assignment (`=`) expressions to `Kw`.
"""
function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, kw=true, block=false, istuple=false)
function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, trivia::Vector{EXPR}, kw = true, block = false, istuple = false; insert_params_at = 2)
    prevpos = position(ps)
    @nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while !closer(ps)
        a = parse_expression(ps)

@@ -148,38 +152,45 @@

        end
        push!(args, a)
        if iscomma(ps.nt)
            accept_comma(ps, args)
        else# if kindof(ps.ws) == SemiColonWS
            if istuple
                # As soon as we hit a comma in a list we can no longer convert to a block expr
                block = false
            end
            accept_comma(ps, trivia)
        elseif kindof(ps.ws) == SemiColonWS
            break
        elseif !closer(ps)
            # We've not hit a closing token, nor separating punctuation so let's insert an error-wrapped comma
            push!(trivia, EXPR(:errortoken, EXPR[EXPR(:COMMA, 0, 0)], nothing))
        end
        prevpos = loop_check(ps, prevpos)
    end
    if istuple && length(args) > 2
    if istuple && length(args) > 1
        block = false
    end

    if kindof(ps.ws) == SemiColonWS
        if @nocloser ps :newline @closer ps :comma @nocloser ps :semicolon closer(ps)
            if block && !(length(args) == 1 && ispunctuation(args[1])) && !(typof(last(args)) === UnaryOpCall && is_dddot(last(args).args[2]))
                push!(args, EXPR(Block, EXPR[pop!(args)]))
            elseif kw && kindof(ps.nt) === Tokens.RPAREN
                push!(args, EXPR(Parameters, EXPR[], 0, 0))
        if @nocloser ps :inwhere @nocloser ps :newline @closer ps :comma @nocloser ps :semicolon closer(ps)
            if block && !(length(args) == 0 && ispunctuation(trivia[1])) && !(isunarycall(last(args)) && is_dddot(last(args).args[2]))
                push!(args, EXPR(:block, EXPR[pop!(args)]))
            else
                insert!(args, insert_params_at, EXPR(:parameters, EXPR[], nothing, 0, 0))
            end
        else
            a = @nocloser ps :newline @closer ps :comma @nocloser ps :inwhere parse_expression(ps)
            if block && !(length(args) == 1 && ispunctuation(args[1])) && !is_splat(last(args)) && !(istuple && iscomma(ps.nt))
            if block && !(length(args) == 0 && ispunctuation(trivia[1])) && !issplat(last(args)) && !(istuple && iscomma(ps.nt))
                args1 = EXPR[pop!(args), a]
                prevpos = position(ps)
                @nocloser ps :newline @closer ps :comma while @nocloser ps :semicolon !closer(ps)
                @nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while @nocloser ps :semicolon !closer(ps)
                    a = parse_expression(ps)
                    push!(args1, a)
                    prevpos = loop_check(ps, prevpos)
                end
                body = EXPR(Block, args1)
                body = EXPR(:block, args1)
                push!(args, body)
                args = body
            else
                parse_parameters(ps, args, EXPR[a])
                parse_parameters(ps, args, EXPR[a], insert_params_at)
            end
        end
    end

@@ -191,15 +202,12 @@


Parses parameter arguments for a function call (e.g. following a semicolon).
"""
function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR}=EXPR[]; usekw=true)
function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR} = EXPR[], insert_params_at = 2; usekw = true)
    trivia = EXPR[]
    isfirst = isempty(args1)
    prevpos = position(ps)
    @nocloser ps :inwhere @nocloser ps :newline  @closer ps :comma while !isfirst || (@nocloser ps :semicolon !closer(ps))
        if isfirst
            a = parse_expression(ps)
        else
            a = first(args1)
        end
        a = isfirst ? parse_expression(ps) : first(args1)
        if usekw && _do_kw_convert(ps, a)
            a = _kw_convert(a)
        end

@@ -210,55 +218,69 @@

            push!(args1, a)
        end
        if iscomma(ps.nt)
            accept_comma(ps, args1)
            accept_comma(ps, trivia)
        elseif kindof(ps.ws) !== SemiColonWS && !closer(ps)
            push!(trivia, EXPR(:errortoken, EXPR[EXPR(:COMMA, 0, 0)], nothing))
        end
        if kindof(ps.ws) == SemiColonWS
            parse_parameters(ps, args1; usekw=usekw)
        end
        if isfirst
            prevpos = loop_check(ps, prevpos)
        else
            prevpos = position(ps)
        end
        prevpos = isfirst ? loop_check(ps, prevpos) : position(ps)
        isfirst = true
    end
    if !isempty(args1)
        paras = EXPR(Parameters, args1)
        push!(args, paras)
        insert!(args, insert_params_at, EXPR(:parameters, args1, trivia))
    end
    return
end

function parse_macroname(ps)
    at = EXPR(ps)
    if !isemptyws(ps.ws)
        ws = ps.ws.endbyte - ps.ws.startbyte + 1
        mname = INSTANCE(next(ps))
        mname.val = valof(mname) isa String ? string("@", " "^ws, valof(mname)) : string("@", " "^ws)
        mname.span += ws
        mname.fullspan += ws
        mname = mErrorToken(ps, mname, UnexpectedWhiteSpace)
    else
        next(ps)
        # set span/fullspan min length at 1 to account for the case of a lonely '@'
        mname = EXPR(:IDENTIFIER, max(1, ps.nt.startbyte - ps.t.startbyte + 1), max(1, ps.t.endbyte - ps.t.startbyte + 2), string("@", val(ps.t, ps)))
    end
end

"""
    parse_macrocall(ps)

Parses a macro call. Expects to start on the `@`.
"""
function parse_macrocall(ps::ParseState)
    at = mPUNCTUATION(ps)
    if !isemptyws(ps.ws)
        mname = mErrorToken(ps, INSTANCE(next(ps)), UnexpectedWhiteSpace)
    else
        mname = EXPR(MacroName, EXPR[at, mIDENTIFIER(next(ps))])
    end

    mname = parse_macroname(ps)
    # Handle cases with @ at start of dotted expressions
    if kindof(ps.nt) === Tokens.DOT && isemptyws(ps.ws)
        prevpos = position(ps)
        while kindof(ps.nt) === Tokens.DOT
            op = mOPERATOR(next(ps))
            nextarg = mIDENTIFIER(next(ps))
            mname = mBinaryOpCall(mname, op, EXPR(Quotenode, EXPR[nextarg]))
            op = EXPR(:OPERATOR, next(ps))
            nextarg = EXPR(:IDENTIFIER, next(ps))
            mname = EXPR(op, EXPR[mname, EXPR(:quotenode, EXPR[nextarg], nothing)], nothing)
            prevpos = loop_check(ps, prevpos)
        end
    end

    if iscomma(ps.nt)
        return EXPR(MacroCall, EXPR[mname], mname.fullspan, mname.span)
        return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0)], nothing, mname.fullspan, mname.span)
    elseif isemptyws(ps.ws) && kindof(ps.nt) === Tokens.LPAREN
        return parse_call(ps, mname, true)
    elseif isemptyws(ps.ws) && kindof(ps.nt) === Tokens.LBRACE
        next(ps)
        return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0), @default ps @closebrace ps parse_braces(ps)], nothing)
    elseif isemptyws(ps.ws) && kindof(ps.nt) === Tokens.LSQUARE
        next(ps)
        return EXPR(:macrocall, EXPR[mname, EXPR(:NOTHING, 0, 0), @default ps parse_array(ps)], nothing)
    else
        args = EXPR[mname]
        #TODO add special hndling for @doc
        args = EXPR[mname, EXPR(:NOTHING, 0, 0)]
        insquare = ps.closer.insquare
        prevpos = position(ps)
        @default ps while !closer(ps)

@@ -273,7 +295,7 @@

            end
            prevpos = loop_check(ps, prevpos)
        end
        return EXPR(MacroCall, args)
        return EXPR(:macrocall, args, nothing)
    end
end


@@ -283,87 +305,98 @@

Having hit `for` not at the beginning of an expression return a generator.
Comprehensions are parsed as SQUAREs containing a generator.
"""
function parse_generator(ps::ParseState, ret::EXPR)
    kw = mKEYWORD(next(ps))
    ret = EXPR(Generator, EXPR[ret, kw])
    ranges = @closesquare ps parse_iterators(ps, true)

    if typof(ranges) === Block
        append!(ret, ranges)
function parse_generator(ps::ParseState, first::EXPR)
    kw = EXPR(next(ps))
    iters, trivia = EXPR[], EXPR[]
    @closesquare ps parse_iterators(ps, iters, trivia)
    if kindof(ps.nt) === Tokens.IF # filter
        push!(trivia, EXPR(next(ps)))
        cond = @closer ps :range parse_expression(ps)
        pushfirst!(iters, cond)
        iters = EXPR(:filter, iters, trivia)
        trivia = EXPR[]
    end
    if kindof(ps.nt) === Tokens.FOR
        return EXPR(:flatten, EXPR[EXPR(:generator, EXPR[parse_generator(ps, first); iters], [kw; trivia])], nothing)
    else
        push!(ret, ranges)
        return EXPR(:generator, EXPR[first; iters], EXPR[kw; trivia])
    end
end

    if typof(ret.args[1]) === Generator || typof(ret.args[1]) === Flatten
        ret = EXPR(Flatten, EXPR[ret])
    end

    return ret
function get_appropriate_child_to_expand(x)
    if headof(x) === :generator && !(headof(x.args[1]) in (:generator, :flatten))
        return x, x.args[1]
    elseif headof(x) === :flatten &&  headof(x.args[1]) === :generator && headof(x.args[1].args[1]) === :generator
        x.args[1], x.args[1].args[1]
    else
        get_appropriate_child_to_expand(x.args[1])
    end
end

function parse_importexport_item(ps, is_colon = false)
    if kindof(ps.nt) === Tokens.AT_SIGN
        mname = parse_macroname(next(ps))
    elseif kindof(ps.nt) === Tokens.LPAREN
        a = EXPR(:brackets, EXPR[], EXPR[EXPR(next(ps))])
        push!(a, @closeparen ps parse_expression(ps))
        pushtotrivia!(a, accept_rparen(ps))
        a
    elseif kindof(ps.nt) === Tokens.EX_OR
        @closer ps :comma parse_expression(ps)
    elseif !is_colon && isoperator(ps.nt)
        next(ps)
        EXPR(:OPERATOR, ps.nt.startbyte - ps.t.startbyte,  1 + ps.t.endbyte - ps.t.startbyte, val(ps.t, ps))
    elseif VERSION > v"1.3.0-" && isidentifier(ps.nt) && isemptyws(ps.nws) && (kindof(ps.nnt) === Tokens.STRING || kindof(ps.nnt) === Tokens.TRIPLE_STRING)
        EXPR(:NONSTDIDENTIFIER, EXPR[INSTANCE(next(ps)), INSTANCE(next(ps))])
        #TODO fix nonstdid handling
    else
        INSTANCE(next(ps))
    end
end
"""
Helper function for parsing import/using statements.
"""
function parse_dot_mod(ps::ParseState, is_colon=false)
    args = EXPR[]
function parse_dot_mod(ps::ParseState, is_colon = false, allow_as = false)
    ret = EXPR(EXPR(:OPERATOR, 0, 0, "."), EXPR[], EXPR[])

    prevpos = position(ps)
    while kindof(ps.nt) === Tokens.DOT || kindof(ps.nt) === Tokens.DDOT || kindof(ps.nt) === Tokens.DDDOT
        d = mOPERATOR(next(ps))
        d = EXPR(:OPERATOR, next(ps))
        trailing_ws = d.fullspan - d.span
        if is_dot(d)
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
            push!(ret, EXPR(:OPERATOR, 1 + trailing_ws, 1, "."))
        elseif is_ddot(d)
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
            push!(ret, EXPR(:OPERATOR, 1, 1, "."))
            push!(ret, EXPR(:OPERATOR, 1 + trailing_ws, 1, "."))
        elseif is_dddot(d)
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
            push!(ret, EXPR(:OPERATOR, 1, 1, "."))
            push!(ret, EXPR(:OPERATOR, 1, 1, "."))
            push!(ret, EXPR(:OPERATOR, 1 + trailing_ws, 1, "."))
        end
        prevpos = loop_check(ps, prevpos)
    end

    # import/export ..
    # TODO: Not clear what this is for?
    # if iscomma(ps.nt) || kindof(ps.ws) == NewLineWS || kindof(ps.nt) === Tokens.ENDMARKER
    #     if length(args) == 2
    #         return EXPR[INSTANCE(ps)]
    #     end
    # end

    prevpos = position(ps)
    while true
        if kindof(ps.nt) === Tokens.AT_SIGN
            at = mPUNCTUATION(next(ps))
            a = INSTANCE(next(ps))
            push!(args, EXPR(MacroName, EXPR[at, a]))
        elseif kindof(ps.nt) === Tokens.LPAREN
            a = EXPR(InvisBrackets, EXPR[mPUNCTUATION(next(ps))])
            push!(a, @closeparen ps parse_expression(ps))
            accept_rparen(ps, a)
            push!(args, a)
        elseif kindof(ps.nt) === Tokens.EX_OR
            a = @closer ps :comma parse_expression(ps)
            push!(args, a)
        elseif !is_colon && isoperator(ps.nt)
            next(ps)
            push!(args, mOPERATOR(ps.nt.startbyte - ps.t.startbyte,  1 + ps.t.endbyte - ps.t.startbyte, kindof(ps.t), false))
        elseif VERSION > v"1.3.0-" && isidentifier(ps.nt) && isemptyws(ps.nws) && (kindof(ps.nnt) === Tokens.STRING || kindof(ps.nnt) === Tokens.TRIPLE_STRING)
            push!(args, EXPR(NONSTDIDENTIFIER, EXPR[INSTANCE(next(ps)), INSTANCE(next(ps))]))
        else
            push!(args, INSTANCE(next(ps)))
        end
        push!(ret, parse_importexport_item(ps, is_colon))

        if kindof(ps.nt) === Tokens.DOT
            push!(args, mPUNCTUATION(next(ps)))
            pushtotrivia!(ret, EXPR(next(ps)))
        elseif isoperator(ps.nt) && (ps.nt.dotop || kindof(ps.nt) === Tokens.DOT)
            push!(args, mPUNCTUATION(Tokens.DOT, 1, 1))
            pushtotrivia!(ret, EXPR(:DOT, 1, 1))
            ps.nt = RawToken(kindof(ps.nt), ps.nt.startpos, ps.nt.endpos, ps.nt.startbyte + 1, ps.nt.endbyte, ps.nt.token_error, false, ps.nt.suffix)
        else
            @static if VERSION > v"1.6-"
                if allow_as && !isnewlinews(ps.ws) && isidentifier(ps.nt) && val(ps.nt, ps) == "as"
                    as = EXPR(:AS, next(ps))
                    as_val = parse_importexport_item(ps, is_colon)
                    ret = EXPR(:as, EXPR[ret, as_val], EXPR[as])
                end
            end
            break
        end
        prevpos = loop_check(ps, prevpos)
    end
    args
    ret
end

@@ -0,0 +1,793 @@

1	+	module Iterating
2	+	using ..CSTParser: EXPR, headof, hastrivia, isoperator, valof, isstringliteral, is_exor, is_lparen, is_rparen
3	+
4	+	function Base.getindex(x::EXPR, i)
5	+	try
6	+	a = _getindex(x, i)
7	+	if a === nothing
8	+	error("indexing error for $(x.head) expression at $i")
9	+	end
10	+	return a
11	+	catch err
12	+	@info typeof(x)
13	+	@info x
14	+	error("indexing error for $(x.head) expression at $i")
15	+	end
16	+	end
17	+
18	+	function _getindex(x::EXPR, i)
19	+	if headof(x) === :abstract
20	+	_abstract(x, i)
21	+	elseif headof(x) === :as
22	+	odda_event(x, i)
23	+	elseif headof(x) === :block
24	+	_block(x, i)
25	+	elseif headof(x) === :braces
26	+	_braces(x, i)
27	+	elseif headof(x) === :brackets
28	+	oddt_evena(x, i)
29	+	elseif headof(x) === :call
30	+	_call(x, i)
31	+	elseif headof(x) === :comparison \|\| headof(x) === :file
32	+	x.args[i]
33	+	elseif headof(x) === :comprehension
34	+	tat(x, i)
35	+	elseif headof(x) === :const
36	+	_const(x, i)
37	+	elseif headof(x) === :curly
38	+	_curly(x, i)
39	+	elseif headof(x) === :do
40	+	odda_event(x, i)
41	+	elseif headof(x) === :elseif
42	+	_elseif(x, i)
43	+	elseif headof(x) === :errortoken
44	+	x.args[i]
45	+	elseif headof(x) === :export
46	+	oddt_evena(x, i)
47	+	elseif headof(x) === :filter
48	+	_filter(x, i)
49	+	elseif headof(x) === :flatten
50	+	x.args[1]
51	+	elseif headof(x) === :for
52	+	taat(x, i)
53	+	elseif headof(x) === :function \|\| headof(x) === :macro
54	+	_function(x, i)
55	+	elseif headof(x) === :generator
56	+	odda_event(x, i)
57	+	elseif headof(x) === :global \|\| headof(x) === :local
58	+	_global(x, i)
59	+	elseif headof(x) === :if
60	+	if isoperator(first(x.trivia)) # ternary op
61	+	odda_event(x, i)
62	+	else
63	+	_if(x, i)
64	+	end
65	+	elseif headof(x) === :kw
66	+	_kw(x, i)
67	+	elseif headof(x) === :let
68	+	taat(x, i)
69	+	elseif headof(x) === :return
70	+	oddt_evena(x, i)
71	+	elseif headof(x) === :macrocall
72	+	_macrocall(x, i)
73	+	elseif headof(x) === :module
74	+	_module(x, i)
75	+	elseif headof(x) === :outer
76	+	ta(x, i)
77	+	elseif headof(x) === :parameters
78	+	if length(x.args) > 1 && headof(x.args[2]) === :parameters
79	+	if i == length(x)
80	+	x.args[2]
81	+	elseif i == 1
82	+	x.args[1]
83	+	elseif isodd(i)
84	+	x.args[div(i + 1, 2) + 1]
85	+	else
86	+	x.trivia[div(i, 2)]
87	+	end
88	+	else
89	+	odda_event(x, i)
90	+	end
91	+	# if hastrivia(x)
92	+	# odda_event(x, i)
93	+	# else
94	+	# x.args[i]
95	+	# end
96	+	elseif headof(x) === :primitive
97	+	_primitive(x, i)
98	+	elseif headof(x) === :quote
99	+	_quote(x, i)
100	+	elseif headof(x) === :quotenode
101	+	_quotenode(x, i)
102	+	elseif headof(x) === :ref
103	+	_call(x, i)
104	+	elseif headof(x) === :row
105	+	x.args[i]
106	+	elseif headof(x) === :string
107	+	_string(x, i)
108	+	elseif headof(x) === :struct
109	+	_struct(x, i)
110	+	elseif headof(x) === :toplevel
111	+	x.args[i]
112	+	elseif headof(x) === :try
113	+	_try(x, i)
114	+	elseif headof(x) === :tuple
115	+	_tuple(x, i)
116	+	elseif headof(x) === :typed_comprehension
117	+	odda_event(x, i)
118	+	elseif headof(x) === :typed_vcat \|\| headof(x) === :typed_hcat
119	+	_typed_vcat(x, i)
120	+	elseif headof(x) === :using \|\| headof(x) === :import
121	+	_using(x, i)
122	+	elseif headof(x) === :vcat \|\| headof(x) === :hcat \|\| headof(x) === :bracescat
123	+	_vcat(x, i)
124	+	elseif headof(x) === :vect
125	+	_vect(x, i)
126	+	elseif headof(x) === :where
127	+	_where(x, i)
128	+	elseif headof(x) === :while
129	+	taat(x, i)
130	+	elseif isoperator(headof(x))
131	+	if valof(headof(x)) == ":"
132	+	_colon_in_using(x, i)
133	+	elseif valof(headof(x)) == "."
134	+	_dot(x, i)
135	+	elseif valof(headof(x)) == "=" && hastrivia(x)
136	+	# a loop that has been lowered from in/∈ to = at parse time
137	+	if i == 1
138	+	x.args[1]
139	+	elseif i == 2
140	+	x.trivia[1]
141	+	elseif i == 3
142	+	x.args[2]
143	+	end
144	+	elseif valof(headof(x)) == "->" && headof(x).fullspan == 0
145	+	# anon function as used in a 'do' block
146	+	if i == 1
147	+	x.args[1]
148	+	else
149	+	x.args[2]
150	+	end
151	+	elseif !hastrivia(x)
152	+	if i == 1
153	+	x.args[1]
154	+	elseif i == 2
155	+	x.head
156	+	elseif i == 3
157	+	x.args[2]
158	+	end
159	+	else
160	+	if i == 1
161	+	x.head
162	+	elseif i == 2
163	+	x.trivia[1]
164	+	elseif i == length(x)
165	+	last(x.trivia)
166	+	elseif isodd(i)
167	+	x.args[div(i+1, 2) - 1]
168	+	else
169	+	x.trivia[div(i, 2)]
170	+	end
171	+	end
172	+	else
173	+	error("Indexing $(Expr(x)) at $i")
174	+	end
175	+	end
176	+	Base.iterate(x::EXPR) = length(x) == 0 ? nothing : (x[1], 1)
177	+	Base.iterate(x::EXPR, s) = s < length(x) ? (x[s + 1], s + 1) : nothing
178	+	Base.firstindex(x::EXPR) = 1
179	+	Base.lastindex(x::EXPR) = x.args === nothing ? 0 : length(x)
180	+
181	+	# Base.setindex!(x::EXPR, val, i) = Base.setindex!(x.args, val, i)
182	+	# Base.first(x::EXPR) = x.args === nothing ? nothing : first(x.args)
183	+	# Base.last(x::EXPR) = x.args === nothing ? nothing : last(x.args)
184	+
185	+
186	+	function ta(x, i)
187	+	if i == 1
188	+	x.trivia[1]
189	+	elseif i == 2
190	+	x.args[1]
191	+	end
192	+	end
193	+	function tat(x, i)
194	+	if i == 1
195	+	x.trivia[1]
196	+	elseif i == 2
197	+	x.args[1]
198	+	elseif i == 3
199	+	x.trivia[2]
200	+	end
201	+	end
202	+
203	+	function taat(x, i)
204	+	if i == 1
205	+	x.trivia[1]
206	+	elseif i == 2
207	+	x.args[1]
208	+	elseif i == 3
209	+	x.args[2]
210	+	elseif i == 4
211	+	x.trivia[2]
212	+	end
213	+	end
214	+
215	+
216	+	function oddt_evena(x, i)
217	+	if isodd(i)
218	+	x.trivia[div(i + 1, 2)]
219	+	else
220	+	x.args[div(i, 2)]
221	+	end
222	+	end
223	+
224	+	function odda_event(x, i)
225	+	if isodd(i)
226	+	x.args[div(i + 1, 2)]
227	+	else
228	+	x.trivia[div(i, 2)]
229	+	end
230	+	end
231	+
232	+
233	+	function _const(x, i)
234	+	if length(x.trivia) === 1
235	+	ta(x, i)
236	+	elseif length(x.trivia) === 2
237	+	#global const
238	+	if i < 3
239	+	x.trivia[i]
240	+	elseif i == 3
241	+	x.args[1]
242	+	end
243	+	end
244	+	end

627	678		return true
628	679		end
629	680
630		-	"""
631		-	is_getfield(x::EXPR)
632	681
633		-	Is this an expression of the form `a.b`.
634		-	"""
635		-	is_getfield(x::EXPR) = isbinarycall(x) && length(x) == 3 && kindof(x[2]) === Tokens.DOT
	682	+
636	683
637	684		"""
638	685		disallowednumberjuxt(ret::EXPR)

640	687		Does this number literal end in a decimal and so cannot precede a paren for
641	688		implicit multiplication?
642	689		"""
643		-	disallowednumberjuxt(ret::EXPR) = is_number(ret) && last(valof(ret)) == '.'
	690	+	disallowednumberjuxt(ret::EXPR) = isnumber(ret) && last(valof(ret)) == '.'
644	691
645	692
646	693		nexttokenstartsdocstring(ps::ParseState) = isidentifier(ps.nt) && val(ps.nt, ps) == "doc" && (kindof(ps.nnt) === Tokens.STRING \|\| kindof(ps.nnt) === Tokens.TRIPLE_STRING)
647	694
648		-	"""
649		-	is_wrapped_assignment(x::EXPR)
650		-
651		-	Is `x` an assignment expression, ignoring any surrounding parentheses.
652		-	"""
653		-	is_wrapped_assignment(x::EXPR) = is_assignment(x) \|\| (isbracketed(x) && is_wrapped_assignment(x.args[2]))
654		-
655		-	"""
656		-	is_range(x::EXPR)
657	695
658		-	Is `x` a valid iterator for use in `for` loops or generators?
659		-	"""
660		-	is_range(x::EXPR) = isbinarycall(x) && (is_eq(x.args[2]) \|\| is_in(x.args[2]) \|\| is_elof(x.args[2]))
661	696
662	697		"""
663	698		_do_kw_convert(ps::ParseState, a::EXPR)
664	699
665	700		Should `a` be converted to a keyword-argument expression?
666	701		"""
667		-	_do_kw_convert(ps::ParseState, a::EXPR) = !ps.closer.brace && is_assignment(a)
	702	+	_do_kw_convert(ps::ParseState, a::EXPR) = !ps.closer.brace && isassignment(a)
668	703
669	704		"""
670	705		_kw_convert(ps::ParseState, a::EXPR)
671	706
672	707		Converted an assignment expression to a keyword-argument expression.
673	708		"""
674		-	_kw_convert(a::EXPR) = EXPR(Kw, EXPR[a.args[1], a.args[2], a.args[3]], a.fullspan, a.span)
	709	+	_kw_convert(x::EXPR) = EXPR(:kw, EXPR[x.args[1], x.args[2]], EXPR[x.head], x.fullspan, x.span)
675	710
676	711		"""
677	712		convertsigtotuple(sig::EXPR)
678	713
679	714		When parsing a function or macro signature, should it be converted to a tuple?
680	715		"""
681		-	convertsigtotuple(sig::EXPR) = isbracketed(sig) && !(istuple(sig.args[2]) \|\| (typof(sig.args[2]) === Block) \|\| is_splat(sig.args[2]))
	716	+	convertsigtotuple(sig::EXPR) = isbracketed(sig) && !(istuple(sig.args[1]) \|\| (headof(sig.args[1]) === :block) \|\| issplat(sig.args[1]))
682	717
683	718		"""
684	719		docable(head)

686	721		When parsing a block of expressions, can documentation be attached? Prefixed docs at the
687	722		top-level are handled within `parse(ps::ParseState, cont = false)`.
688	723		"""
689		-	docable(head) = head === Begin \|\| head === ModuleH \|\| head === BareModule \|\| head === Quote
	724	+	docable(head) = head === :begin \|\| head === :module \|\| head === :baremodule \|\| head === :quote
690	725
691	726
692	727		should_negate_number_literal(ps::ParseState, op::EXPR) = (is_plus(op) \|\| is_minus(op)) && (kindof(ps.nt) === Tokens.INTEGER \|\| kindof(ps.nt) === Tokens.FLOAT) && isemptyws(ps.ws) && kindof(ps.nnt) != Tokens.CIRCUMFLEX_ACCENT
693	728
694		-	isbracketed(x::EXPR) = typof(x) === InvisBrackets # Assumption that x has 3 args, doesn't need checking?
695		-
696		-	unwrapbracket(x::EXPR) = isbracketed(x) ? unwrapbracket(x[2]) : x
697		-
698		-	isbeginorblock(x::EXPR) = typof(x) === Begin \|\| typof(unwrapbracket(x)) == Block
699	729
700	730		"""
701	731		can_become_comparison(x::EXPR)

703	733		Is `x` a binary comparison call (e.g. `a < b`) that can be extended to include more
704	734		arguments?
705	735		"""
706		-	can_become_comparison(x::EXPR) = isbinarycall(x) && (precedence(x.args[2]) == ComparisonOp \|\| is_issubt(x.args[2]) \|\| is_issupt(x.args[2]))
	736	+	can_become_comparison(x::EXPR) = (isoperator(x.head) && comp_prec(valof(x.head)) && length(x.args) > 1) \|\| (x.head === :call && isoperator(x.args[1]) && comp_prec(valof(x.args[1])) && length(x.args) > 2 && !hastrivia(x))
707	737
708	738		"""
709	739		can_become_chain(x::EXPR, op::EXPR)

711	741		Is `x` a binary call for `+` or `*` that can be extended to include more
712	742		arguments?
713	743		"""
714		-	can_become_chain(x::EXPR, op::EXPR) = isbinarycall(x) && (is_star(op) \|\| is_plus(op)) && kindof(op) == kindof(x.args[2]) && !x.args[2].dot && x.args[2].span > 0
	744	+	can_become_chain(x::EXPR, op::EXPR) = isbinarycall(x) && (is_star(op) \|\| is_plus(op)) && valof(op) == valof(x.args[2]) && !isdotted(x.args[2]) && x.args[2].span > 0
	745	+
	746	+
	747	+	macro cst_str(x)
	748	+	CSTParser.parse(x)
	749	+	end
	750	+
	751	+	function issuffixableliteral(ps::ParseState, x::EXPR)
	752	+	isidentifier(ps.nt) && isemptyws(ps.ws) && ismacrocall(x) && (valof(x.args[1]) isa String && (endswith(valof(x.args[1]), "_str") \|\| endswith(valof(x.args[1]), "_cmd")))
	753	+	end
715	754
716	755		function loop_check(ps, prevpos)
717		-	if position(ps) <= prevpos
	756	+	if position(ps) <= prevpos && ps.nt.kind !== Tokens.ENDMARKER
718	757		throw(CSTInfiniteLoop("Infinite loop at $ps"))
719	758		else
720	759		position(ps)

51	51		# (len < 45 \|\| (len == 45 && s <= "0o3777777777777777777777777777777777777777777")) && return Base.parse(UInt128, s)
52	52		# return Base.parse(BigInt, s)
53	53		(len < 45 \|\| (len == 45 && s <= "0o3777777777777777777777777777777777777777777")) && return Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), nothing, s)
54		-	return flisp_parse(s)
	54	+	return Meta.parse(s)
55	55		end
56	56
57	57		function _literal_expr(x)
58		-	if kindof(x) === Tokens.TRUE
	58	+	if headof(x) === :TRUE
59	59		return true
60		-	elseif kindof(x) === Tokens.FALSE
	60	+	elseif headof(x) === :FALSE
61	61		return false
62	62		elseif is_nothing(x)
63	63		return nothing
64		-	elseif kindof(x) === Tokens.INTEGER \|\| kindof(x) === Tokens.BIN_INT \|\| kindof(x) === Tokens.HEX_INT \|\| kindof(x) === Tokens.OCT_INT
	64	+	elseif headof(x) === :INTEGER \|\| headof(x) === :BININT \|\| headof(x) === :HEXINT \|\| headof(x) === :OCTINT
65	65		return Expr_int(x)
66		-	elseif kindof(x) === Tokens.FLOAT
	66	+	elseif isfloat(x)
67	67		return Expr_float(x)
68		-	elseif kindof(x) === Tokens.CHAR
	68	+	elseif ischar(x)
69	69		return Expr_char(x)
70		-	elseif kindof(x) === Tokens.MACRO
	70	+	elseif headof(x) === :MACRO
71	71		return Symbol(valof(x))
72		-	elseif kindof(x) === Tokens.STRING
	72	+	elseif headof(x) === :STRING \|\| headof(x) === :TRIPLESTRING
73	73		return valof(x)
74		-	elseif kindof(x) === Tokens.TRIPLE_STRING
75		-	return valof(x)
76		-	elseif kindof(x) === Tokens.CMD
	74	+	elseif headof(x) === :CMD
77	75		return Expr_cmd(x)
78		-	elseif kindof(x) === Tokens.TRIPLE_CMD
	76	+	elseif headof(x) === :TRIPLECMD
79	77		return Expr_tcmd(x)
80	78		end
81	79		end

95	93		is_oct && return sized_uint_oct_literal(val)
96	94		is_bin && return sized_uint_literal(val, 1)
97	95		# sizeof(val) <= sizeof(TYPEMAX_INT64_STR) && return Base.parse(Int64, val)
98		-	return flisp_parse(val)
	96	+	return Meta.parse(val)
99	97		# # val < TYPEMAX_INT64_STR && return Base.parse(Int64, val)
100	98		# sizeof(val) <= sizeof(TYPEMAX_INTval < TYPEMAX_INT128_STR128_STR) && return Base.parse(Int128, val)
101	99		# # val < TYPEMAX_INT128_STR && return Base.parse(Int128, val)

120	118
121	119		# Expressions
122	120
123		-	# Fallback
124	121		function Expr(x::EXPR)
125	122		if isidentifier(x)
126		-	if typof(x) === NONSTDIDENTIFIER
	123	+	if headof(x) === :NONSTDIDENTIFIER
127	124		Symbol(normalize_julia_identifier(valof(x.args[2])))
128	125		else
129	126		return Symbol(normalize_julia_identifier(valof(x)))
130	127		end
131		-	elseif iskw(x)
132		-	if kindof(x) === Tokens.BREAK
	128	+	elseif iskeyword(x)
	129	+	if headof(x) === :BREAK
133	130		return Expr(:break)
134		-	elseif kindof(x) === Tokens.CONTINUE
	131	+	elseif headof(x) === :CONTINUE
135	132		return Expr(:continue)
136	133		else
137		-	return Symbol(lowercase(string(kindof(x))))
	134	+	return Symbol(lowercase(string(headof(x))))
138	135		end
139	136		elseif isoperator(x)
140		-	ret = x.val isa String ? Symbol(valof(x)) : UNICODE_OPS_REVERSE[kindof(x)]
141		-	return x.dot ? Symbol(:., ret) : ret
	137	+	return Symbol(valof(x))
142	138		elseif ispunctuation(x)
143		-	return string(kindof(x))
	139	+	if headof(x) === :DOT
	140	+	return :(.)
	141	+	else
	142	+	error()
	143	+	end
144	144		elseif isliteral(x)
145	145		return _literal_expr(x)
146		-	elseif isunarycall(x)
147		-	return _unary_expr(x)
148		-	elseif isbinarycall(x)
149		-	return _binary_expr(x)
150		-	elseif iswherecall(x)
151		-	return _where_expr(x)
152		-	elseif typof(x) === ConditionalOpCall
153		-	return Expr(:if, Expr(x.args[1]), Expr(x.args[3]), Expr(x.args[5]))
154		-	elseif typof(x) === ErrorToken
155		-	ret = Expr(:error)
156		-	if x.args !== nothing
157		-	for a in x.args
158		-	if !(ispunctuation(a))
159		-	push!(ret.args, Expr(a))
160		-	end
161		-	end
162		-	end
163		-	return ret
164		-	elseif typof(x) === ChainOpCall
165		-	ret = Expr(:call, Expr(x.args[2]))
166		-	for i = 1:length(x.args)
167		-	if isodd(i)
168		-	push!(ret.args, Expr(x.args[i]))
169		-	end
170		-	end
171		-	return ret
172		-	elseif typof(x) === Comparison
173		-	ret = Expr(:comparison)
174		-	for a in x.args
175		-	if !(ispunctuation(a))
176		-	push!(ret.args, Expr(a))
177		-	end
178		-	end
179		-	return ret
180		-	elseif typof(x) === ColonOpCall
181		-	return Expr(:call, :(:), Expr(x.args[1]), Expr(x.args[3]), Expr(x.args[5]))
182		-	elseif typof(x) === TopLevel
183		-	ret = Expr(:toplevel)
184		-	for a in x.args
185		-	if !(ispunctuation(a))
186		-	push!(ret.args, Expr(a))
187		-	end
188		-	end
189		-	return ret
190		-	elseif typof(x) === MacroName
191		-	if isidentifier(x.args[2])
192		-	if valof(x.args[2]) == "."
193		-	return Symbol("@", "__dot__")
194		-	else
195		-	return Symbol("@", valof(x.args[2]))
196		-	end
197		-	elseif isoperator(x.args[2])
198		-	return Symbol("@", Expr(x.args[2]))
199		-
200		-	else
201		-	return Symbol("@")
202		-	end
203		-	elseif typof(x) === MacroCall
204		-	ret = Expr(:macrocall)
205		-	for a in x.args
206		-	if !(ispunctuation(a))
207		-	push!(ret.args, Expr(a))
208		-	end
209		-	end
210		-	insert!(ret.args, 2, nothing)
211		-	if ret.args[1] isa Expr && ret.args[1].head == :. && string(ret.args[1].args[2].value)[1] != '@'
212		-	clear_at!(ret.args[1])
213		-	ret.args[1].args[2] = QuoteNode(Symbol(string('@', ret.args[1].args[2].value)))
214		-	end
215		-	ret
216		-	elseif typof(x) === x_Str
217		-	if isbinarycall(x.args[1]) && issyntaxcall(x.args[1].args[2])
218		-	mname = Expr(x.args[1])
219		-	mname.args[2] = QuoteNode(Symbol("@", mname.args[2].value, "_str"))
220		-	ret = Expr(:macrocall, mname, nothing)
221		-	else
222		-	ret = Expr(:macrocall, Symbol("@", valof(x.args[1]), "_str"), nothing)
223		-	end
224		-	for i = 2:length(x.args)
225		-	push!(ret.args, valof(x.args[i]))
226		-	end
227		-	return ret
228		-	elseif typof(x) === x_Cmd
229		-	ret = Expr(:macrocall, Symbol("@", valof(x.args[1]), "_cmd"), nothing)
230		-	for i = 2:length(x.args)
231		-	push!(ret.args, valof(x.args[i]))
232		-	end
233		-	return ret
234		-	elseif typof(x) === Quotenode
235		-	return QuoteNode(Expr(x.args[end]))
236		-	elseif typof(x) === Call
237		-	if kindof(x.args[1]) === Tokens.ISSUBTYPE \|\| kindof(x.args[1]) === Tokens.ISSUPERTYPE
238		-	ret = Expr(Expr(x.args[1]))
239		-	for i in 2:length(x.args)
240		-	a = x.args[i]
241		-	if typof(a) === Parameters
242		-	insert!(ret.args, 2, Expr(a))
243		-	elseif !(ispunctuation(a))
244		-	push!(ret.args, Expr(a))
245		-	end
246		-	end
247		-	return ret
248		-	end
249		-	ret = Expr(:call)
250		-	for a in x.args
251		-	if typof(a) === Parameters
252		-	insert!(ret.args, 2, Expr(a))
253		-	elseif !(ispunctuation(a))
254		-	push!(ret.args, Expr(a))
255		-	end
256		-	end
257		-	return ret
258		-	elseif typof(x) === Braces
259		-	ret = Expr(:braces)
260		-	for a in x.args
261		-	if typof(a) === Parameters
262		-	insert!(ret.args, 1, Expr(a))
263		-	elseif !(ispunctuation(a))
264		-	push!(ret.args, Expr(a))
265		-	end
266		-	end
267		-	return ret
268		-	elseif typof(x) === BracesCat
269		-	ret = Expr(:bracescat)
270		-	for a in x.args
271		-	if typof(a) === Parameters
272		-	insert!(ret.args, 1, Expr(a))
273		-	elseif !(ispunctuation(a))
274		-	push!(ret.args, Expr(a))
275		-	end
276		-	end
277		-	return ret
278		-	elseif typof(x) === Struct
279		-	return Expr(:struct, false, Expr(x.args[2]), Expr(x.args[3]))
280		-	elseif typof(x) === Mutable
281		-	return length(x.args) == 4 ? Expr(:struct, true, Expr(x.args[2]), Expr(x.args[3])) : Expr(:struct, true, Expr(x.args[3]), Expr(x.args[4]))
282		-	elseif typof(x) === Abstract
283		-	return length(x.args) == 2 ? Expr(:abstract, Expr(x.args[2])) : Expr(:abstract, Expr(x.args[3]))
284		-	elseif typof(x) === Primitive
285		-	return Expr(:primitive, Expr(x.args[3]), Expr(x.args[4]))
286		-	elseif typof(x) === FunctionDef
287		-	ret = Expr(:function)
288		-	for a in x.args
289		-	if !(ispunctuation(a) \|\| iskw(a))
290		-	push!(ret.args, Expr(a))
291		-	end
292		-	end
293		-	return ret
294		-	elseif typof(x) === Macro
295		-	if length(x.args) == 3
296		-	Expr(:macro, Expr(x.args[2]))
297		-	else
298		-	Expr(:macro, Expr(x.args[2]), Expr(x.args[3]))
299		-	end
300		-	elseif typof(x) === ModuleH
301		-	return Expr(:module, true, Expr(x.args[2]), Expr(x.args[3]))
302		-	elseif typof(x) === BareModule
303		-	return Expr(:module, false, Expr(x.args[2]), Expr(x.args[3]))
304		-	elseif typof(x) === If
305		-	return _if_expr(x)
306		-	elseif typof(x) === Try
307		-	ret = Expr(:try)
308		-	for a in x.args
309		-	if !(ispunctuation(a) \|\| iskw(a))
310		-	push!(ret.args, Expr(a))
311		-	end
312		-	end
313		-	return ret
314		-	elseif typof(x) === Let
315		-	return _let_expr(x)
316		-	elseif typof(x) === Do
317		-	return Expr(:do, Expr(x.args[1]), Expr(:->, Expr(x.args[3]), Expr(x.args[4])))
318		-	elseif typof(x) === Outer
319		-	return Expr(:outer, Expr(x.args[2]))
320		-	elseif typof(x) === For
321		-	ret = Expr(:for)
322		-	if typof(x.args[2]) === Block
323		-	arg = Expr(:block)
324		-	for a in x.args[2].args
325		-	if !(ispunctuation(a))
326		-	push!(arg.args, fix_range(a))
327		-	end
328		-	end
329		-	push!(ret.args, arg)
330		-	else
331		-	push!(ret.args, fix_range(x.args[2]))
332		-	end
333		-	push!(ret.args, Expr(x.args[3]))
334		-	return ret
335		-	elseif typof(x) === While
336		-	ret = Expr(:while)
337		-	for a in x.args
338		-	if !(ispunctuation(a) \|\| iskw(a))
339		-	push!(ret.args, Expr(a))
340		-	end
341		-	end
342		-	return ret
343		-	elseif istuple(x)
344		-	ret = Expr(:tuple)
345		-	for a in x.args
346		-	if typof(a) == Parameters
347		-	insert!(ret.args, 1, Expr(a))
348		-	elseif !(ispunctuation(a))
349		-	push!(ret.args, Expr(a))
350		-	end
351		-	end
352		-	return ret
353		-	elseif typof(x) === Curly
354		-	ret = Expr(:curly)
355		-	for a in x.args
356		-	if typof(a) === Parameters
357		-	insert!(ret.args, 2, Expr(a))
358		-	elseif !(ispunctuation(a))
359		-	push!(ret.args, Expr(a))
360		-	end
361		-	end
362		-	return ret
363		-	elseif typof(x) === Vect
364		-	ret = Expr(:vect)
365		-	for a in x.args
366		-	if typof(a) === Parameters
367		-	pushfirst!(ret.args, Expr(a))
368		-	elseif !(ispunctuation(a))
369		-	push!(ret.args, Expr(a))
370		-	end
371		-	end
372		-	return ret
373		-	elseif typof(x) === Row
374		-	ret = Expr(:row)
375		-	for a in x.args
376		-	if !(ispunctuation(a))
377		-	push!(ret.args, Expr(a))
378		-	end
379		-	end
380		-	return ret
381		-	elseif typof(x) === Hcat
382		-	ret = Expr(:hcat)
383		-	for a in x.args
384		-	if !(ispunctuation(a))
385		-	push!(ret.args, Expr(a))
386		-	end
387		-	end
388		-	return ret
389		-	elseif typof(x) === Vcat
390		-	ret = Expr(:vcat)
391		-	for a in x.args
392		-	if !(ispunctuation(a))
393		-	push!(ret.args, Expr(a))
394		-	end
395		-	end
396		-	return ret
397		-	elseif typof(x) === Block
398		-	ret = Expr(:block)
399		-	for a in x.args
400		-	if !(ispunctuation(a))
401		-	push!(ret.args, Expr(a))
402		-	end
403		-	end
404		-	return ret
405		-	elseif typof(x) === Kw
406		-	return Expr(:kw, Expr(x.args[1]), Expr(x.args[3]))
407		-	elseif typof(x) === Parameters
408		-	ret = Expr(:parameters)
409		-	for a in x.args
410		-	if typof(a) === Parameters
411		-	insert!(ret.args, 2, Expr(a))
412		-	elseif !(ispunctuation(a))
413		-	push!(ret.args, Expr(a))
414		-	end
415		-	end
416		-	return ret
417		-	elseif typof(x) === Return
418		-	ret = Expr(:return)
419		-	for i = 2:length(x.args)
420		-	a = x.args[i]
421		-	push!(ret.args, Expr(a))
422		-	end
423		-	return ret
424	146		elseif isbracketed(x)
425		-	return Expr(x.args[2])
426		-	elseif typof(x) === Begin
427		-	return Expr(x.args[2])
428		-	elseif typof(x) === Quote
429		-	if length(x.args) == 1
430		-	return Expr(:quote, Expr(x.args[1]))
431		-	elseif isbracketed(x.args[2]) && (isoperator(x.args[2].args[2]) \|\| isliteral(x.args[2].args[2]) \|\| isidentifier(x.args[2].args[2]))
432		-	return QuoteNode(Expr(x.args[2]))
	147	+	return Expr(x.args[1])
	148	+	elseif x.head isa EXPR
	149	+	Expr(Expr(x.head), Expr.(x.args)...)
	150	+	elseif x.head === :quotenode
	151	+	QuoteNode(Expr(x.args[1]))
	152	+	elseif x.head === :globalrefdoc
	153	+	GlobalRef(Core, Symbol("@doc"))
	154	+	elseif x.head === :globalrefcmd
	155	+	if VERSION > v"1.1"
	156	+	GlobalRef(Core, Symbol("@cmd"))
433	157		else
434		-	return Expr(:quote, Expr(x.args[2]))
435		-	end
436		-	elseif typof(x) === Global
437		-	ret = Expr(:global)
438		-	if typof(x.args[2]) === Const
439		-	ret = Expr(:const, Expr(:global, Expr(x.args[2].args[2])))
440		-	elseif length(x.args) == 2 && istuple(x.args[2])
441		-	for a in x.args[2].args
442		-	if !(ispunctuation(a))
443		-	push!(ret.args, Expr(a))
444		-	end
445		-	end
446		-	else
447		-	for i = 2:length(x.args)
448		-	a = x.args[i]
449		-	push!(ret.args, Expr(a))
450		-	end
451		-	end
452		-	return ret
453		-	elseif typof(x) === Local
454		-	ret = Expr(:local)
455		-	if typof(x.args[2]) === Const
456		-	ret = Expr(:const, Expr(:global, Expr(x.args[2].args[2])))
457		-	elseif length(x.args) == 2 && istuple(x.args[2])
458		-	for a in x.args[2].args
459		-	if !(ispunctuation(a))
460		-	push!(ret.args, Expr(a))
461		-	end
462		-	end
463		-	else
464		-	for i = 2:length(x.args)
465		-	a = x.args[i]
466		-	push!(ret.args, Expr(a))
467		-	end
468		-	end
469		-	return ret
470		-	elseif typof(x) === Const
471		-	ret = Expr(:const)
472		-	for i = 2:length(x.args)
473		-	a = x.args[i]
474		-	push!(ret.args, Expr(a))
475		-	end
476		-	return ret
477		-	elseif typof(x) === GlobalRefDoc
478		-	return GlobalRef(Core, Symbol("@doc"))
479		-	elseif typof(x) === Ref
480		-	ret = Expr(:ref)
481		-	for a in x.args
482		-	if typof(a) === Parameters
483		-	insert!(ret.args, 2, Expr(a))
484		-	elseif !(ispunctuation(a))
485		-	push!(ret.args, Expr(a))
486		-	end
487		-	end
488		-	return ret
489		-	elseif typof(x) === TypedHcat
490		-	ret = Expr(:typed_hcat)
491		-	for a in x.args
492		-	if !(ispunctuation(a))
493		-	push!(ret.args, Expr(a))
494		-	end
495		-	end
496		-	return ret
497		-	elseif typof(x) === TypedVcat
498		-	ret = Expr(:typed_vcat)
499		-	for a in x.args
500		-	if typof(a) === Parameters
501		-	insert!(ret.args, 2, Expr(a))
502		-	elseif !(ispunctuation(a))
503		-	push!(ret.args, Expr(a))
504		-	end
505		-	end
506		-	return ret
507		-	elseif typof(x) === Comprehension \|\| typof(x) === DictComprehension
508		-	ret = Expr(:comprehension)
509		-	for a in x.args
510		-	if !(ispunctuation(a))
511		-	push!(ret.args, Expr(a))
512		-	end
513		-	end
514		-	return ret
515		-	elseif typof(x) === Flatten
516		-	iters, args = get_inner_gen(x)
517		-	i = popfirst!(iters)
518		-	ex = Expr(:generator, Expr(args[1]), convert_iter_assign(i[1]))
519		-	for i in iters
520		-	if length(i) == 1
521		-	ex = Expr(:generator, ex, convert_iter_assign(i[1]))
522		-	ex = Expr(:flatten, ex)
523		-	else
524		-	ex = Expr(:generator, ex)
525		-	for j in i
526		-	push!(ex.args, convert_iter_assign(j))
527		-	end
528		-	ex = Expr(:flatten, ex)
529		-	end
530		-	end
531		-	return ex
532		-	elseif typof(x) === Generator
533		-	ret = Expr(:generator, Expr(x.args[1]))
534		-	for i = 3:length(x.args)
535		-	a = x.args[i]
536		-	if !(ispunctuation(a))
537		-	push!(ret.args, convert_iter_assign(a))
538		-	end
539		-	end
540		-	return ret
541		-	elseif typof(x) === Filter
542		-	ret = Expr(:filter)
543		-	push!(ret.args, Expr(last(x.args)))
544		-	for i in 1:length(x.args) - 1
545		-	a = x.args[i]
546		-	if !(is_if(a) \|\| ispunctuation(a))
547		-	push!(ret.args, convert_iter_assign(a))
548		-	end
549		-	end
550		-	return ret
551		-	elseif typof(x) === TypedComprehension
552		-	ret = Expr(:typed_comprehension)
553		-	for a in x.args
554		-	if !(ispunctuation(a))
555		-	push!(ret.args, Expr(a))
556		-	end
557		-	end
558		-	return ret
559		-	elseif typof(x) === Import
560		-	return expr_import(x, :import)
561		-	elseif typof(x) === Using
562		-	return expr_import(x, :using)
563		-	elseif typof(x) === Export
564		-	ret = Expr(:export)
565		-	for i = 2:length(x.args)
566		-	a = x.args[i]
567		-	if !(ispunctuation(a))
568		-	push!(ret.args, Expr(a))
569		-	end
570		-	end
571		-	return ret
572		-	elseif typof(x) === FileH
573		-	ret = Expr(:file)
574		-	for a in x.args
575		-	push!(ret.args, Expr(a))
576		-	end
577		-	return ret
578		-	elseif typof(x) === StringH
579		-	ret = Expr(:string)
580		-	for (i, a) in enumerate(x.args)
581		-	if isunarycall(a)
582		-	a = a.args[2]
583		-	elseif isliteral(a) && kindof(a) === Tokens.STRING && span(a) == 0 \|\| ((i == 1 \|\| i == length(x.args)) && span(a) == 1) \|\| (valof(a) === nothing \|\| isempty(valof(a)))
584		-	continue
585		-	else isliteral(a) && kindof(a) === Tokens.TRIPLE_STRING && span(a) == 0 \|\| ((i == 1 \|\| i == length(x.args)) && span(a) == 3) \|\| (valof(a) === nothing \|\| isempty(valof(a)))
586		-	end
587		-	push!(ret.args, Expr(a))
588		-	end
589		-	return ret
	158	+	Symbol("@cmd")
	159	+	end
	160	+	elseif x.head === :macrocall && is_getfield_w_quotenode(x.args[1]) && !ismacroname(x.args[1].args[2].args[1])
	161	+	# Shift '@' to the right
	162	+	new_name = Expr(:., remove_at(x.args[1].args[1]), QuoteNode(Symbol("@", valof(x.args[1].args[2].args[1]))))
	163	+	Expr(:macrocall, new_name, Expr.(x.args[2:end])...)
	164	+	elseif x.head === :macrocall && isidentifier(x.args[1]) && valof(x.args[1]) == "@."
	165	+	Expr(:macrocall, Symbol("@__dot__"), Expr.(x.args[2:end])...)
	166	+	elseif x.head === :string && length(x.args) > 0 && (x.args[1].head === :STRING \|\| x.args[1].head === :TRIPLESTRING) && isempty(valof(x.args[1]))
	167	+	# Special conversion needed - the initial text section is treated as empty for the represented string following lowest-common-prefix adjustments, but exists in the source.
	168	+	Expr(:string, Expr.(x.args[2:end])...)
	169	+	elseif x.args === nothing
	170	+	Expr(Symbol(lowercase(String(x.head))))
590	171		else
591		-	ret = Expr(:call)
592		-	for a in x.args
593		-	if !(ispunctuation(a))
594		-	push!(ret.args, Expr(a))
595		-	end
596		-	end
597		-	return ret
	172	+	Expr(Symbol(lowercase(String(x.head))), Expr.(x.args)...)
598	173		end
599	174		end
600	175
601		-	# Op. expressions
602		-
603		-	function _unary_expr(x)
604		-	if isoperator(x.args[1]) && issyntaxunarycall(x.args[1])
605		-	Expr(Expr(x.args[1]), Expr(x.args[2]))
606		-	elseif isoperator(x.args[2]) && issyntaxunarycall(x.args[2])
607		-	Expr(Expr(x.args[2]), Expr(x.args[1]))
	176	+	function remove_at(x)
	177	+	if isidentifier(x) && valof(x) !== nothing && first(valof(x)) == '@'
	178	+	return Symbol(valof(x)[2:end])
	179	+	elseif is_getfield_w_quotenode(x)
	180	+	Expr(:., remove_at(x.args[1]), QuoteNode(remove_at(x.args[2].args[1])))
608	181		else
609		-	Expr(:call, Expr(x.args[1]), Expr(x.args[2]))
610		-	end
611		-	end
612		-	function _binary_expr(x)
613		-	if issyntaxcall(x.args[2]) && !(kindof(x.args[2]) in (Tokens.COLON,))
614		-	if kindof(x.args[2]) === Tokens.DOT
615		-	arg1, arg2 = Expr(x.args[1]), Expr(x.args[3])
616		-	if arg2 isa Expr && arg2.head === :macrocall && endswith(string(arg2.args[1]), "_cmd")
617		-	return Expr(:macrocall, Expr(:., arg1, QuoteNode(arg2.args[1])), nothing, arg2.args[3])
618		-	elseif arg2 isa Expr && arg2.head === :braces
619		-	return Expr(:., arg1, Expr(:quote, arg2))
620		-	end
621		-	end
622		-	Expr(Expr(x.args[2]), Expr(x.args[1]), Expr(x.args[3]))
623		-	else
624		-	Expr(:call, Expr(x.args[2]), Expr(x.args[1]), Expr(x.args[3]))
	182	+	Expr(x)
625	183		end
626	184		end
627	185
628		-	function _where_expr(x)
629		-	ret = Expr(:where, Expr(x.args[1]))
630		-	for i = 3:length(x.args)
631		-	a = x.args[i]
632		-	if typof(a) === Parameters
633		-	insert!(ret.args, 2, Expr(a))
634		-	elseif !(ispunctuation(a) \|\| iskw(a))
635		-	push!(ret.args, Expr(a))
636		-	end
637		-	end
638		-	return ret
639		-	end
640		-
641		-
642	186		# cross compatability for line number insertion in macrocalls
643	187		if VERSION > v"1.1-"
644	188		Expr_cmd(x) = Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), nothing, valof(x))

690	234		x
691	235		end
692	236
693		-	function _if_expr(x)
694		-	ret = Expr(:if)
695		-	iselseif = false
696		-	n = length(x.args)
697		-	i = 0
698		-	while i < n
699		-	i += 1
700		-	a = x.args[i]
701		-	if iskw(a) && kindof(a) === Tokens.ELSEIF
702		-	i += 1
703		-	r1 = Expr(x.args[i].args[1])
704		-	push!(ret.args, Expr(:elseif, r1.args...))
705		-	elseif !(ispunctuation(a) \|\| iskw(a))
706		-	push!(ret.args, Expr(a))
707		-	end
708		-	end
709		-	ret
710		-	end
711		-
712		-	function _let_expr(x)
713		-	ret = Expr(:let)
714		-	if length(x.args) == 3
715		-	push!(ret.args, Expr(:block))
716		-	push!(ret.args, Expr(x.args[2]))
717		-	return ret
718		-	elseif typof(x.args[2]) === Block
719		-	arg = Expr(:block)
720		-	for a in x.args[2].args
721		-	if !(ispunctuation(a))
722		-	push!(arg.args, fix_range(a))
723		-	end
724		-	end
725		-	push!(ret.args, arg)
726		-	else
727		-	push!(ret.args, fix_range(x.args[2]))
728		-	end
729		-	push!(ret.args, Expr(x.args[3]))
730		-	ret
731		-	end
732		-
733		-	function fix_range(a)
734		-	if isbinarycall(a) && (is_in(a.args[2]) \|\| is_elof(a.args[2]))
735		-	Expr(:(=), Expr(a.args[1]), Expr(a.args[3]))
736		-	else
737		-	Expr(a)
738		-	end
739		-	end
740		-
741		-	function get_inner_gen(x, iters=[], arg=[])
742		-	if typof(x) == Flatten
743		-	get_inner_gen(x.args[1], iters, arg)
744		-	elseif typof(x) === Generator
745		-	# push!(iters, get_iter(x))
746		-	get_iters(x, iters)
747		-	if typof(x.args[1]) === Generator \|\| typof(x.args[1]) === Flatten
748		-	get_inner_gen(x.args[1], iters, arg)
749		-	else
750		-	push!(arg, x.args[1])
751		-	end
752		-	end
753		-	return iters, arg
754		-	end
755		-
756		-	function get_iter(x)
757		-	if typof(x) === Generator
758		-	return x.args[3]
759		-	end
760		-	end
761		-
762		-	function get_iters(x, iters)
763		-	iters1 = []
764		-	if typof(x) === Generator
765		-	# return x.args[3]
766		-	for i = 3:length(x.args)
767		-	if typof(x.args[i]) !== PUNCTUATION
768		-	push!(iters1, x.args[i])
769		-	end
770		-	end
771		-	end
772		-	push!(iters, iters1)
773		-	end
774		-
775		-	function convert_iter_assign(a)
776		-	if isbinarycall(a) && (is_in(a.args[2]) \|\| is_elof(a.args[2]))
777		-	return Expr(:(=), Expr(a.args[1]), Expr(a.args[3]))
778		-	else
779		-	return Expr(a)
780		-	end
781		-	end
782		-
783		-	function _get_import_block(x, i, ret)
784		-	while is_dot(x.args[i + 1])
785		-	i += 1
786		-	push!(ret.args, :.)
787		-	end
788		-	while i < length(x.args) && !(is_comma(x.args[i + 1]))
789		-	i += 1
790		-	a = x.args[i]
791		-	if !(ispunctuation(a)) && !(is_dot(a) \|\| is_colon(a))
792		-	push!(ret.args, Expr(a))
793		-	end
794		-	end
795		-
796		-	return i
797		-	end
798		-
799		-	function expr_import(x, kw)
800		-	col = findall(a -> isoperator(a) && precedence(a) == ColonOp, x.args)
801		-	comma = findall(is_comma, x.args)
802		-
803		-	header = []
804		-	args = [Expr(:.)]
805		-	i = 1 # skip keyword
806		-	while i < length(x.args)
807		-	i += 1
808		-	a = x.args[i]
809		-	if is_colon(a)
810		-	push!(header, popfirst!(args))
811		-	push!(args, Expr(:.))
812		-	elseif is_comma(a)
813		-	push!(args, Expr(:.))
814		-	elseif !(ispunctuation(a))
815		-	push!(last(args).args, Expr(a))
816		-	end
817		-	end
818		-	if isempty(header)
819		-	return Expr(kw, args...)
820		-	else
821		-	return Expr(kw, Expr(:(:), header..., args...))
822		-	end
823		-	end
824		-
825		-
826		-
827		-	const UNICODE_OPS_REVERSE = Dict{Tokenize.Tokens.Kind,Symbol}()
828		-	for (k, v) in Tokenize.Tokens.UNICODE_OPS
829		-	UNICODE_OPS_REVERSE[v] = Symbol(k)
830		-	end
831	237
832		-	UNICODE_OPS_REVERSE[Tokens.EQ] = :(=)
833		-	UNICODE_OPS_REVERSE[Tokens.PLUS_EQ] = :(+=)
834		-	UNICODE_OPS_REVERSE[Tokens.MINUS_EQ] = :(-=)
835		-	UNICODE_OPS_REVERSE[Tokens.STAR_EQ] = :(*=)
836		-	UNICODE_OPS_REVERSE[Tokens.FWD_SLASH_EQ] = :(/=)
837		-	UNICODE_OPS_REVERSE[Tokens.FWDFWD_SLASH_EQ] = :(//=)
838		-	UNICODE_OPS_REVERSE[Tokens.OR_EQ] = :(\|=)
839		-	UNICODE_OPS_REVERSE[Tokens.CIRCUMFLEX_EQ] = :(^=)
840		-	UNICODE_OPS_REVERSE[Tokens.DIVISION_EQ] = :(÷=)
841		-	UNICODE_OPS_REVERSE[Tokens.REM_EQ] = :(%=)
842		-	UNICODE_OPS_REVERSE[Tokens.LBITSHIFT_EQ] = :(<<=)
843		-	UNICODE_OPS_REVERSE[Tokens.RBITSHIFT_EQ] = :(>>=)
844		-	UNICODE_OPS_REVERSE[Tokens.LBITSHIFT] = :(<<)
845		-	UNICODE_OPS_REVERSE[Tokens.RBITSHIFT] = :(>>)
846		-	UNICODE_OPS_REVERSE[Tokens.UNSIGNED_BITSHIFT] = :(>>>)
847		-	UNICODE_OPS_REVERSE[Tokens.UNSIGNED_BITSHIFT_EQ] = :(>>>=)
848		-	UNICODE_OPS_REVERSE[Tokens.BACKSLASH_EQ] = :(\=)
849		-	UNICODE_OPS_REVERSE[Tokens.AND_EQ] = :(&=)
850		-	UNICODE_OPS_REVERSE[Tokens.COLON_EQ] = :(:=)
851		-	UNICODE_OPS_REVERSE[Tokens.PAIR_ARROW] = :(=>)
852		-	UNICODE_OPS_REVERSE[Tokens.APPROX] = :(~)
853		-	UNICODE_OPS_REVERSE[Tokens.EX_OR_EQ] = :($=)
854		-	UNICODE_OPS_REVERSE[Tokens.XOR_EQ] = :(⊻=)
855		-	UNICODE_OPS_REVERSE[Tokens.RIGHT_ARROW] = :(-->)
856		-	UNICODE_OPS_REVERSE[Tokens.LAZY_OR] = :(\|\|)
857		-	UNICODE_OPS_REVERSE[Tokens.LAZY_AND] = :(&&)
858		-	UNICODE_OPS_REVERSE[Tokens.ISSUBTYPE] = :(<:)
859		-	UNICODE_OPS_REVERSE[Tokens.ISSUPERTYPE] = :(>:)
860		-	UNICODE_OPS_REVERSE[Tokens.GREATER] = :(>)
861		-	UNICODE_OPS_REVERSE[Tokens.LESS] = :(<)
862		-	UNICODE_OPS_REVERSE[Tokens.GREATER_EQ] = :(>=)
863		-	UNICODE_OPS_REVERSE[Tokens.GREATER_THAN_OR_EQUAL_TO] = :(≥)
864		-	UNICODE_OPS_REVERSE[Tokens.LESS_EQ] = :(<=)
865		-	UNICODE_OPS_REVERSE[Tokens.LESS_THAN_OR_EQUAL_TO] = :(≤)
866		-	UNICODE_OPS_REVERSE[Tokens.EQEQ] = :(==)
867		-	UNICODE_OPS_REVERSE[Tokens.EQEQEQ] = :(===)
868		-	UNICODE_OPS_REVERSE[Tokens.IDENTICAL_TO] = :(≡)
869		-	UNICODE_OPS_REVERSE[Tokens.NOT_EQ] = :(!=)
870		-	UNICODE_OPS_REVERSE[Tokens.NOT_EQUAL_TO] = :(≠)
871		-	UNICODE_OPS_REVERSE[Tokens.NOT_IS] = :(!==)
872		-	UNICODE_OPS_REVERSE[Tokens.NOT_IDENTICAL_TO] = :(≢)
873		-	UNICODE_OPS_REVERSE[Tokens.IN] = :(in)
874		-	UNICODE_OPS_REVERSE[Tokens.ISA] = :(isa)
875		-	UNICODE_OPS_REVERSE[Tokens.LPIPE] = :(<\|)
876		-	UNICODE_OPS_REVERSE[Tokens.RPIPE] = :(\|>)
877		-	UNICODE_OPS_REVERSE[Tokens.COLON] = :(:)
878		-	UNICODE_OPS_REVERSE[Tokens.DDOT] = :(..)
879		-	UNICODE_OPS_REVERSE[Tokens.EX_OR] = :($)
880		-	UNICODE_OPS_REVERSE[Tokens.PLUS] = :(+)
881		-	UNICODE_OPS_REVERSE[Tokens.MINUS] = :(-)
882		-	UNICODE_OPS_REVERSE[Tokens.PLUSPLUS] = :(++)
883		-	UNICODE_OPS_REVERSE[Tokens.OR] = :(\|)
884		-	UNICODE_OPS_REVERSE[Tokens.STAR] = :(*)
885		-	UNICODE_OPS_REVERSE[Tokens.FWD_SLASH] = :(/)
886		-	UNICODE_OPS_REVERSE[Tokens.REM] = :(%)
887		-	UNICODE_OPS_REVERSE[Tokens.BACKSLASH] = :(\)
888		-	UNICODE_OPS_REVERSE[Tokens.AND] = :(&)
889		-	UNICODE_OPS_REVERSE[Tokens.FWDFWD_SLASH] = :(//)
890		-	UNICODE_OPS_REVERSE[Tokens.CIRCUMFLEX_ACCENT] = :(^)
891		-	UNICODE_OPS_REVERSE[Tokens.DECLARATION] = :(::)
892		-	UNICODE_OPS_REVERSE[Tokens.CONDITIONAL] = :?
893		-	UNICODE_OPS_REVERSE[Tokens.DOT] = :(.)
894		-	UNICODE_OPS_REVERSE[Tokens.NOT] = :(!)
895		-	UNICODE_OPS_REVERSE[Tokens.PRIME] = Symbol(''')
896		-	UNICODE_OPS_REVERSE[Tokens.DDDOT] = :(...)
897		-	UNICODE_OPS_REVERSE[Tokens.TRANSPOSE] = Symbol(".'")
898		-	UNICODE_OPS_REVERSE[Tokens.ANON_FUNC] = :(->)
899		-	UNICODE_OPS_REVERSE[Tokens.WHERE] = :where

1	1		function accept_rparen(ps)
2	2		if kindof(ps.nt) === Tokens.RPAREN
3		-	return mPUNCTUATION(next(ps))
	3	+	return EXPR(next(ps))
4	4		else
5		-	return mErrorToken(ps, mPUNCTUATION(Tokens.RPAREN, 0, 0), UnexpectedToken)
	5	+	return mErrorToken(ps, EXPR(:RPAREN, 0, 0), UnexpectedToken)
6	6		end
7	7		end
8	8		accept_rparen(ps::ParseState, args) = push!(args, accept_rparen(ps))
9	9
10	10		function accept_rsquare(ps)
11	11		if kindof(ps.nt) === Tokens.RSQUARE
12		-	return mPUNCTUATION(next(ps))
	12	+	return EXPR(next(ps))
13	13		else
14		-	return mErrorToken(ps, mPUNCTUATION(Tokens.RSQUARE, 0, 0), UnexpectedToken)
	14	+	return mErrorToken(ps, EXPR(:RSQUARE, 0, 0), UnexpectedToken)
15	15		end
16	16		end
17	17		accept_rsquare(ps::ParseState, args) = push!(args, accept_rsquare(ps))
18	18
19	19		function accept_rbrace(ps)
20	20		if kindof(ps.nt) === Tokens.RBRACE
21		-	return mPUNCTUATION(next(ps))
	21	+	return EXPR(next(ps))
22	22		else
23		-	return mErrorToken(ps, mPUNCTUATION(Tokens.RBRACE, 0, 0), UnexpectedToken)
	23	+	return mErrorToken(ps, EXPR(:RBRACE, 0, 0), UnexpectedToken)
24	24		end
25	25		end
26	26		accept_rbrace(ps::ParseState, args) = push!(args, accept_rbrace(ps))
27	27
28	28		function accept_end(ps::ParseState)
29	29		if kindof(ps.nt) === Tokens.END
30		-	return mKEYWORD(next(ps))
	30	+	return EXPR(next(ps))
31	31		else
32		-	return mErrorToken(ps, mKEYWORD(Tokens.END, 0, 0), UnexpectedToken)
	32	+	return mErrorToken(ps, EXPR(:END, 0, 0), UnexpectedToken)
33	33		end
34	34		end
35	35		accept_end(ps::ParseState, args) = push!(args, accept_end(ps))
36	36
37	37		function accept_comma(ps)
38	38		if iscomma(ps.nt)
39		-	return mPUNCTUATION(next(ps))
	39	+	return EXPR(next(ps))
40	40		else
41		-	return mErrorToken(mPUNCTUATION(Tokens.COMMA, 0, 0), UnexpectedToken)
	41	+	return EXPR(:RPAREN, 0, 0)
42	42		end
43	43		end
44	44		accept_comma(ps::ParseState, args) = push!(args, accept_comma(ps))

52	52		end
53	53
54	54		function requires_no_ws(x, ps)
55		-	if !(ps.nt.kind === Tokens.RPAREN \|\|
56		-	ps.nt.kind === Tokens.RBRACE \|\|
57		-	ps.nt.kind === Tokens.RSQUARE) && x.span != x.fullspan
	55	+	if !(kindof(ps.nt) === Tokens.RPAREN \|\|
	56	+	kindof(ps.nt) === Tokens.RBRACE \|\|
	57	+	kindof(ps.nt) === Tokens.RSQUARE) && x.span != x.fullspan
58	58		return mErrorToken(ps, x, UnexpectedWhiteSpace)
59	59		else
60	60		return x

100	170		Should only be called when has_sig(x) == true.
101	171		"""
102	172		function get_sig(x::EXPR)
103		-	if isbinarycall(x)
	173	+	if headof(x) isa EXPR # headof(headof(x)) === :OPERATOR valof(headof(x)) == "="
104	174		return x.args[1]
105		-	elseif typof(x) === Struct \|\| typof(x) === FunctionDef \|\| typof(x) === Macro
	175	+	elseif headof(x) === :struct \|\| headof(x) === :mutable
106	176		return x.args[2]
107		-	elseif typof(x) === Mutable \|\| typof(x) === Abstract \|\| typof(x) === Primitive
108		-	return x.args[3]
	177	+	elseif headof(x) === :abstract \|\| headof(x) === :primitive \|\| headof(x) === :function \|\| headof(x) === :macro
	178	+	return x.args[1]
109	179		end
110	180		end
111	181
112	182		function get_name(x::EXPR)
113		-	if typof(x) === Struct \|\| typof(x) === Mutable \|\| typof(x) === Abstract \|\| typof(x) === Primitive
	183	+	if defines_datatype(x)
114	184		sig = get_sig(x)
115	185		sig = rem_subtype(sig)
116	186		sig = rem_wheres(sig)
117	187		sig = rem_subtype(sig)
118	188		sig = rem_curly(sig)
119		-	elseif typof(x) === ModuleH \|\| typof(x) === BareModule
	189	+	elseif defines_module(x)
120	190		sig = x.args[2]
121		-	elseif typof(x) === FunctionDef \|\| typof(x) === Macro
	191	+	elseif defines_function(x) \|\| defines_macro(x)
122	192		sig = get_sig(x)
123	193		sig = rem_wheres(sig)
124	194		sig = rem_decl(sig)
125	195		sig = rem_call(sig)
126	196		sig = rem_curly(sig)
127	197		sig = rem_invis(sig)
128		-	if isbinarycall(sig) && kindof(sig.args[2]) === Tokens.DOT
129		-	if length(sig.args) > 2 && sig.args[3].args isa Vector{EXPR} && length(sig.args[3].args) > 0
130		-	sig = sig.args[3].args[1]
131		-	end
	198	+	# if isbinarysyntax(sig) && is_dot(sig.head)
	199	+	if is_getfield_w_quotenode(sig)
	200	+	sig = sig.args[2].args[1]
132	201		end
133	202		return sig
	203	+	elseif is_getfield_w_quotenode(x)
	204	+	return x.args[2].args[1]
134	205		elseif isbinarycall(x)
135		-	length(x.args) < 2 && return x
136		-	if kindof(x.args[2]) === Tokens.DOT
137		-	if length(x.args) > 2 && typof(x.args[3]) === Quotenode && x.args[3].args isa Vector{EXPR} && length(x.args[3].args) > 0
138		-	return get_name(x.args[3].args[1])
139		-	else
140		-	return x
141		-	end
142		-	end
143	206		sig = x.args[1]
144	207		if isunarycall(sig)
145	208		return get_name(sig.args[1])

150	213		sig = rem_curly(sig)
151	214		sig = rem_invis(sig)
152	215		return get_name(sig)
	216	+	elseif isbinarysyntax(x) && valof(x.head) == "<:"
	217	+	return get_name(x.args[1])
	218	+	elseif isunarysyntax(x) && valof(x.head) == "..."
	219	+	return get_name(x.args[1])
153	220		else
154	221		sig = x
155	222		if isunarycall(sig)

96	96
97	97		Base.position(ps::ParseState) = ps.nt.startbyte
98	98
	99	+	current_line(ps::ParseState) = ps.nt.startpos[1]
99	100		"""
100	101		lex_ws_comment(l::Lexer, c)
101	102

178	179		@inline val(token::AbstractToken, ps::ParseState) = String(ps.l.io.data[token.startbyte + 1:token.endbyte + 1])
179	180		both_symbol_and_op(t::AbstractToken) = kindof(t) === Tokens.WHERE \|\| kindof(t) === Tokens.IN \|\| kindof(t) === Tokens.ISA
180	181		isprefixableliteral(t::AbstractToken) = (kindof(t) === Tokens.STRING \|\| kindof(t) === Tokens.TRIPLE_STRING \|\| kindof(t) === Tokens.CMD \|\| kindof(t) === Tokens.TRIPLE_CMD)
181		-	isassignment(t::AbstractToken) = Tokens.begin_assignments < kindof(t) < Tokens.end_assignments
	182	+	isassignmentop(t::AbstractToken) = Tokens.begin_assignments < kindof(t) < Tokens.end_assignments
182	183
183	184		isidentifier(t::AbstractToken) = kindof(t) === Tokens.IDENTIFIER
184	185		isliteral(t::AbstractToken) = Tokens.begin_literal < kindof(t) < Tokens.end_literal
185	186		isbool(t::AbstractToken) = Tokens.TRUE ≤ kindof(t) ≤ Tokens.FALSE
186	187		iscomma(t::AbstractToken) = kindof(t) === Tokens.COMMA
187	188		iscolon(t::AbstractToken) = kindof(t) === Tokens.COLON
188		-	iskw(t::AbstractToken) = Tokens.iskeyword(kindof(t))
189		-	isinstance(t::AbstractToken) = isidentifier(t) \|\| isliteral(t) \|\| isbool(t) \|\| iskw(t)
	189	+	iskeyword(t::AbstractToken) = Tokens.iskeyword(kindof(t))
	190	+	isinstance(t::AbstractToken) = isidentifier(t) \|\| isliteral(t) \|\| isbool(t) \|\| iskeyword(t)
190	191		ispunctuation(t::AbstractToken) = iscomma(t) \|\| kindof(t) === Tokens.END \|\| Tokens.LSQUARE ≤ kindof(t) ≤ Tokens.RPAREN \|\| kindof(t) === Tokens.AT_SIGN
191	192

7	7		const ComparisonOp = 6
8	8		const PipeOp = 7
9	9		const ColonOp = 8
	10	+	const PlusOp = 9
10	11		@static if Base.operator_precedence(:<<) == 12
11		-	const PlusOp = 9
12	12		const BitShiftOp = 10
13	13		const TimesOp = 11
14	14		const RationalOp = 12
15	15		else
16		-	const PlusOp = 9
17	16		const TimesOp = 10
18	17		const RationalOp = 11
19	18		const BitShiftOp = 12

26	25		const DddotOp = 7
27	26		const AnonFuncOp = 14
28	27
29		-	@enum(Head,IDENTIFIER,
30		-	NONSTDIDENTIFIER,
31		-	PUNCTUATION,
32		-	OPERATOR,
33		-	KEYWORD,
34		-	LITERAL,
35		-	NoHead,
36		-	Call,
37		-	UnaryOpCall,
38		-	BinaryOpCall,
39		-	WhereOpCall,
40		-	ConditionalOpCall,
41		-	ChainOpCall,
42		-	ColonOpCall,
43		-	Abstract,
44		-	Begin,
45		-	Block,
46		-	Braces,
47		-	BracesCat,
48		-	Const,
49		-	Comparison,
50		-	Curly,
51		-	Do,
52		-	Filter,
53		-	Flatten,
54		-	For,
55		-	FunctionDef,
56		-	Generator,
57		-	Global,
58		-	GlobalRefDoc,
59		-	If,
60		-	Kw,
61		-	Let,
62		-	Local,
63		-	Macro,
64		-	MacroCall,
65		-	MacroName,
66		-	Mutable,
67		-	Outer,
68		-	Parameters,
69		-	Primitive,
70		-	Quote,
71		-	Quotenode,
72		-	InvisBrackets,
73		-	StringH,
74		-	Struct,
75		-	Try,
76		-	TupleH,
77		-	FileH,
78		-	Return,
79		-	While,
80		-	x_Cmd,
81		-	x_Str,
82		-	ModuleH,
83		-	BareModule,
84		-	TopLevel,
85		-	Export,
86		-	Import,
87		-	Using,
88		-	Comprehension,
89		-	DictComprehension,
90		-	TypedComprehension,
91		-	Hcat,
92		-	TypedHcat,
93		-	Ref,
94		-	Row,
95		-	Vcat,
96		-	TypedVcat,
97		-	Vect,
98		-	ErrorToken)
99		-
100	28		@enum(ErrorKind,
101	29		UnexpectedToken,
102	30		CannotJuxtapose,

110	38		InvalidIterator,
111	39		StringInterpolationWithTrailingWhitespace,
112	40		TooLongChar,
113		-	Unknown)
114		-
115		-	const NoKind = Tokenize.Tokens.begin_keywords
	41	+	Unknown,
	42	+	SignatureOfFunctionDefIsNotACall)
116	43
117	44		mutable struct EXPR
118		-	typ::Head
	45	+	head::Union{Symbol,EXPR}
119	46		args::Union{Nothing,Vector{EXPR}}
	47	+	trivia::Union{Nothing,Vector{EXPR}}
120	48		fullspan::Int
121	49		span::Int
122	50		val::Union{Nothing,String}
123		-	kind::Tokenize.Tokens.Kind
124		-	dot::Bool
125	51		parent::Union{Nothing,EXPR}
126	52		meta
127	53		end
128	54
129		-	function EXPR(T::Head, args::Vector{EXPR}, fullspan::Int, span::Int)
130		-	ex = EXPR(T, args, fullspan, span, nothing, NoKind, false, nothing, nothing)
	55	+	function EXPR(head::Union{Symbol,EXPR}, args::Vector{EXPR}, trivia::Union{Vector{EXPR},Nothing}, fullspan::Int, span::Int)
	56	+	ex = EXPR(head, args, trivia, fullspan, span, nothing, nothing, nothing)
	57	+	if head isa EXPR
	58	+	setparent!(head, ex)
	59	+	end
131	60		for c in args
132	61		setparent!(c, ex)
133	62		end
	63	+	if trivia isa Vector{EXPR}
	64	+	for c in trivia
	65	+	setparent!(c, ex)
	66	+	end
	67	+	end
134	68		ex
135	69		end
136	70
137		-	function EXPR(T::Head, args::Vector{EXPR})
138		-	ret = EXPR(T, args, 0, 0)
	71	+	function EXPR(head::Union{Symbol,EXPR}, args::Vector{EXPR}, trivia::Union{Vector{EXPR},Nothing} = EXPR[])
	72	+	ret = EXPR(head, args, trivia, 0, 0)
139	73		update_span!(ret)
140	74		ret
141	75		end
142	76
	77	+	# These methods are for terminal/childless expressions.
	78	+	@noinline EXPR(head::Union{Symbol,EXPR}, fullspan::Int, span::Int, val = nothing) = EXPR(head, nothing, nothing, fullspan, span, val, nothing, nothing)
	79	+	@noinline EXPR(head::Union{Symbol,EXPR}, ps::ParseState) = EXPR(head, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps))
	80	+	@noinline EXPR(ps::ParseState) = EXPR(tokenkindtoheadmap(kindof(ps.t)), ps)
143	81
144		-
145		-
146		-	@noinline mIDENTIFIER(ps::ParseState) = EXPR(IDENTIFIER, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, nothing)
147		-
148		-	mPUNCTUATION(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(PUNCTUATION, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
149		-	@noinline mPUNCTUATION(ps::ParseState) = EXPR(PUNCTUATION, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)
150		-
151		-	mOPERATOR(fullspan::Int, span::Int, kind::Tokens.Kind, dotop::Bool) = EXPR(OPERATOR, nothing, fullspan, span, nothing, kind, dotop, nothing, nothing)
152		-	@noinline mOPERATOR(ps::ParseState) = EXPR(OPERATOR, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, ps.t.suffix ? val(ps.t, ps) : nothing, kindof(ps.t), ps.t.dotop, nothing, nothing)
153		-
154		-	mKEYWORD(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(KEYWORD, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
155		-	@noinline mKEYWORD(ps::ParseState) = EXPR(KEYWORD, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)
156		-
157		-	mLITERAL(fullspan::Int, span::Int, val::String, kind::Tokens.Kind) = EXPR(LITERAL, nothing, fullspan, span, val, kind, false, nothing, nothing)
158	82		@noinline function mLITERAL(ps::ParseState)
159		-
160	83		if kindof(ps.t) === Tokens.STRING \|\| kindof(ps.t) === Tokens.TRIPLE_STRING \|\|
161	84		kindof(ps.t) === Tokens.CMD \|\| kindof(ps.t) === Tokens.TRIPLE_CMD
162	85		return parse_string_or_cmd(ps)
163	86		else
164	87		v = val(ps.t, ps)
165	88		if kindof(ps.t) === Tokens.CHAR && length(v) > 3 && !(v[2] == '\\' && valid_escaped_seq(v[2:prevind(v, length(v))]))
166		-	return mErrorToken(ps, mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\''), kindof(ps.t)), TooLongChar)
	89	+	return mErrorToken(ps, EXPR(:CHAR, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\'')), TooLongChar)
167	90		end
168		-	return mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v, kindof(ps.t))
	91	+	return EXPR(literalmap(kindof(ps.t)), ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v)
169	92		end
170	93		end
171	94

174	97		span(x::EXPR) = x.span
175	98
176	99		function update_span!(x::EXPR)
177		-	(x.args isa Nothing \|\| isempty(x.args)) && return
	100	+	(x.args isa Nothing \|\| isempty(x.args)) && !hastrivia(x) && return
178	101		x.fullspan = 0
179	102		for i = 1:length(x.args)
180	103		x.fullspan += x.args[i].fullspan
181	104		end
182		-	x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
	105	+	if hastrivia(x)
	106	+	for i = 1:length(x.trivia)
	107	+	x.fullspan += x.trivia[i].fullspan
	108	+	end
	109	+	end
	110	+	if x.head isa EXPR
	111	+	x.fullspan += x.head.fullspan
	112	+	# TODO: special case for trailing unary ops?
	113	+	end
	114	+	if x.head isa EXPR && isoperator(x.head) && (is_dddot(x.head) \|\| is_prime(x.head))
	115	+	# trailing unary operator
	116	+	x.span = x.fullspan - x.head.fullspan + x.head.span
	117	+	elseif lastchildistrivia(x)
	118	+	x.span = x.fullspan - last(x.trivia).fullspan + last(x.trivia).span
	119	+	elseif !isempty(x.args)
	120	+	x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
	121	+	end
183	122		return
184	123		end
185	124

190	129		push!(e.args, arg)
191	130		end
192	131
	132	+	function pushtotrivia!(e::EXPR, arg::EXPR)
	133	+	e.span = e.fullspan + arg.span
	134	+	e.fullspan += arg.fullspan
	135	+	setparent!(arg, e)
	136	+	push!(e.trivia, arg)
	137	+	end
	138	+
193	139		function Base.pushfirst!(e::EXPR, arg::EXPR)
194	140		e.fullspan += arg.fullspan
195	141		setparent!(arg, e)

227	173
228	174		function INSTANCE(ps::ParseState)
229	175		if isidentifier(ps.t)
230		-	return mIDENTIFIER(ps)
	176	+	return EXPR(:IDENTIFIER, ps)
231	177		elseif isliteral(ps.t)
232	178		return mLITERAL(ps)
233		-	elseif iskw(ps.t)
234		-	return mKEYWORD(ps)
	179	+	elseif iskeyword(ps.t)
	180	+	return EXPR(ps)
235	181		elseif isoperator(ps.t)
236		-	return mOPERATOR(ps)
	182	+	return EXPR(:OPERATOR, ps)
237	183		elseif ispunctuation(ps.t)
238		-	return mPUNCTUATION(ps)
	184	+	return EXPR(ps)
239	185		elseif kindof(ps.t) === Tokens.ERROR
240	186		ps.errored = true
241		-	return EXPR(ErrorToken, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, Unknown)
	187	+	return EXPR(:errortoken, nothing, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), nothing, Unknown)
242	188		else
243	189		return mErrorToken(ps, Unknown)
244	190		end
245	191		end
246	192
247		-	function mUnaryOpCall(op::EXPR, arg::EXPR)
248		-	fullspan = op.fullspan + arg.fullspan
249		-	ex = EXPR(UnaryOpCall, EXPR[op, arg], fullspan, fullspan - arg.fullspan + arg.span)
250		-	setparent!(op, ex)
251		-	setparent!(op, ex)
252		-	return ex
253		-	end
254		-	function mBinaryOpCall(arg1::EXPR, op::EXPR, arg2::EXPR)
255		-	fullspan = arg1.fullspan + op.fullspan + arg2.fullspan
256		-	ex = EXPR(BinaryOpCall, EXPR[arg1, op, arg2], fullspan, fullspan - arg2.fullspan + arg2.span)
257		-	setparent!(arg1, ex)
258		-	setparent!(op, ex)
259		-	setparent!(arg2, ex)
260		-	return ex
261		-	end
262		-	function mWhereOpCall(arg1::EXPR, op::EXPR, args::Vector{EXPR})
263		-	ex = EXPR(WhereOpCall, EXPR[arg1; op; args], arg1.fullspan + op.fullspan, 0)
264		-	setparent!(arg1, ex)
265		-	setparent!(op, ex)
266		-	for a in args
267		-	ex.fullspan += a.fullspan
268		-	setparent!(a, ex)
269		-	end
270		-	ex.span = ex.fullspan - last(args).fullspan + last(args).span
271		-	return ex
272		-	end
273		-
274	193		function mErrorToken(ps::ParseState, k::ErrorKind)
275	194		ps.errored = true
276		-	return EXPR(ErrorToken, EXPR[], 0, 0, nothing, NoKind, false, nothing, k)
	195	+	return EXPR(:errortoken, EXPR[], nothing, 0, 0, nothing, nothing, k)
277	196		end
278	197		function mErrorToken(ps::ParseState, x::EXPR, k)
279	198		ps.errored = true
280		-	ret = EXPR(ErrorToken, EXPR[x], x.fullspan, x.span, nothing, NoKind, false, nothing, k)
281		-	setparent!(ret[1], ret)
	199	+	ret = EXPR(:errortoken, EXPR[x], nothing, x.fullspan, x.span, nothing, nothing, k)
	200	+	setparent!(ret.args[1], ret)
282	201		return ret
283	202		end
284	203
285		-	TRUE() = mLITERAL(0, 0, "", Tokens.TRUE)
286		-	FALSE() = mLITERAL(0, 0, "", Tokens.FALSE)
287		-	NOTHING() = mLITERAL(0, 0, "", Tokens.NOTHING)
288		-	GlobalRefDOC() = EXPR(GlobalRefDoc, EXPR[])
289	204
290		-	typof(x::EXPR) = x.typ
	205	+	headof(x::EXPR) = x.head
291	206		valof(x::EXPR) = x.val
292		-	kindof(x::EXPR) = x.kind
293	207		kindof(t::Tokens.AbstractToken) = t.kind
294	208		parentof(x::EXPR) = x.parent
295	209		errorof(x::EXPR) = errorof(x.meta)

74	74		end
75	75		end
76	76
77		-	# there are interpolations in the string
78		-	if prefixed != false \|\| iscmd
79		-	t_str = val(ps.t, ps)
	77	+	isinterpolated = false
	78	+
	79	+	t_str = val(ps.t, ps)
	80	+	if istrip && length(t_str) == 6
	81	+	if iscmd
	82	+	return wrapwithcmdmacro(EXPR(:TRIPLESTRING , sfullspan, sspan, ""))
	83	+	else
	84	+	return EXPR(:TRIPLESTRING, sfullspan, sspan, "")
	85	+	end
	86	+	elseif length(t_str) == 2
	87	+	if iscmd
	88	+	return wrapwithcmdmacro(EXPR(:STRING , sfullspan, sspan, ""))
	89	+	else
	90	+	return EXPR(:STRING, sfullspan, sspan, "")
	91	+	end
	92	+	elseif prefixed != false \|\| iscmd
80	93		_val = istrip ? t_str[4:prevind(t_str, sizeof(t_str), 3)] : t_str[2:prevind(t_str, sizeof(t_str))]
81	94		if iscmd
82	95		_val = replace(_val, "\\\\" => "\\")
83	96		_val = replace(_val, "\\`" => "`")
84	97		else
85		-	if endswith(_val, "\\\\")
86		-	_val = _val[1:end - 1]
87		-	end
88		-	_val = replace(_val, "\\\"" => "\"")
	98	+	_val = unescape_prefixed(_val)
89	99		end
90		-	expr = mLITERAL(sfullspan, sspan, _val, kindof(ps.t))
	100	+	expr = EXPR(istrip ? :TRIPLESTRING : :STRING, sfullspan, sspan, _val)
91	101		if istrip
92	102		adjust_lcp(expr)
93		-	ret = EXPR(StringH, EXPR[expr], sfullspan, sspan)
	103	+	ret = EXPR(:string, EXPR[expr], nothing, sfullspan, sspan)
94	104		else
95		-	return expr
	105	+	return iscmd ? wrapwithcmdmacro(expr) : expr
96	106		end
97	107		else
98		-	ret = EXPR(StringH, EXPR[], sfullspan, sspan)
99		-	str2 = val(ps.t, ps)
100		-	input = IOBuffer(str2)
	108	+	ret = EXPR(:string, EXPR[], EXPR[], sfullspan, sspan)
	109	+	input = IOBuffer(t_str)
101	110		startbytes = istrip ? 3 : 1
102	111		seek(input, startbytes)
103	112		b = IOBuffer()

109	118		write(b, c)
110	119		write(b, read(input, Char))
111	120		elseif c == '$'
	121	+	isinterpolated = true
112	122		lspan = position(b)
113	123		str = tostr(b)
114		-	ex = mLITERAL(lspan + startbytes, lspan + startbytes, str, Tokens.STRING)
115		-	push!(ret, ex)
	124	+	ex = EXPR(:STRING, lspan + startbytes, lspan + startbytes, str)
	125	+	if position(input) == (istrip ? 3 : 1) + 1
	126	+	# Need to add empty :STRING at start to account for \"
	127	+	pushtotrivia!(ret, ex)
	128	+	elseif !isempty(str)
	129	+	push!(ret, ex)
	130	+	end
116	131		istrip && adjust_lcp(ex)
117	132		startbytes = 0
118		-	op = mOPERATOR(1, 1, Tokens.EX_OR, false)
	133	+	op = EXPR(:OPERATOR, 1, 1, "\$")
119	134		if peekchar(input) == '('
120	135		skip(input, 1) # skip past '('
121	136		lpfullspan = -position(input)
122	137		if iswhitespace(peekchar(input)) \|\| peekchar(input) === '#'
123	138		read_ws_comment(input, readchar(input))
124	139		end
125		-	lparen = mPUNCTUATION(Tokens.LPAREN, lpfullspan + position(input) + 1, 1)
126		-	rparen = mPUNCTUATION(Tokens.RPAREN, 1, 1)
	140	+	lparen = EXPR(:LPAREN, lpfullspan + position(input) + 1, 1)
	141	+	rparen = EXPR(:RPAREN, 1, 1)
127	142
128	143		ps1 = ParseState(input)
129	144
130	145		if kindof(ps1.nt) === Tokens.RPAREN
131		-	call = mUnaryOpCall(op, EXPR(InvisBrackets, EXPR[lparen, rparen]))
132		-	push!(ret, call)
	146	+	push!(ret, EXPR(:ERRORTOKEN, EXPR[], nothing))
	147	+	pushtotrivia!(ret, op)
	148	+	pushtotrivia!(ret, lparen)
	149	+	pushtotrivia!(ret, rparen)
133	150		skip(input, 1)
134	151		else
135		-	interp = @closer ps1 :paren parse_expression(ps1)
136		-	call = mUnaryOpCall(op, EXPR(InvisBrackets, EXPR[lparen, interp, rparen]))
137		-	push!(ret, call)
	152	+	interp_val = @closer ps1 :paren parse_expression(ps1)
	153	+	push!(ret, interp_val)
	154	+	pushtotrivia!(ret, op)
	155	+	pushtotrivia!(ret, lparen)
	156	+	pushtotrivia!(ret, rparen)
138	157		seek(input, ps1.nt.startbyte + 1)
139	158		end
140	159		# Compared to flisp/JuliaParser, we have an extra lookahead token,
141	160		# so we need to back up one here
142	161		elseif Tokenize.Lexers.iswhitespace(peekchar(input)) \|\| peekchar(input) === '#'
143		-	push!(ret, mErrorToken(ps, op, StringInterpolationWithTrailingWhitespace))
	162	+	pushtotrivia!(ret, mErrorToken(ps, op, StringInterpolationWithTrailingWhitespace))
144	163		else
145	164		pos = position(input)
146	165		ps1 = ParseState(input)

152	171		end
153	172		# Attribute trailing whitespace to the string
154	173		t = adjustspan(t)
155		-	call = mUnaryOpCall(op, t)
156		-	push!(ret, call)
	174	+	push!(ret, t)
	175	+	pushtotrivia!(ret, op)
157	176		seek(input, pos + t.fullspan)
158	177		end
159	178		else

166	185		lspan = position(b)
167	186		if b.size == 0
168	187		ex = mErrorToken(ps, Unknown)
	188	+	push!(ret, ex)
169	189		else
170	190		str = tostr(b)
171	191		if istrip
172	192		str = str[1:prevind(str, lastindex(str), 3)]
173	193		# only mark non-interpolated triple strings
174		-	ex = mLITERAL(lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str, length(ret) == 0 ? Tokens.TRIPLE_STRING : Tokens.STRING)
	194	+	ex = EXPR(length(ret) == 0 ? :TRIPLESTRING : :STRING, lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str)
175	195		adjust_lcp(ex, true)
176	196		else
177	197		str = str[1:prevind(str, lastindex(str))]
178		-	ex = mLITERAL(lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str, Tokens.STRING)
	198	+	ex = EXPR(:STRING, lspan + ps.nt.startbyte - ps.t.endbyte - 1 + startbytes, lspan + startbytes, str)
	199	+	end
	200	+	if isempty(str)
	201	+	pushtotrivia!(ret, ex)
	202	+	else
	203	+	push!(ret, ex)
179	204		end
180	205		end
181		-	push!(ret, ex)
182		-
183	206		end
184	207
185		-	single_string_T = (Tokens.STRING, kindof(ps.t))
	208	+	single_string_T = (:STRING, :TRIPLESTRING, literalmap(kindof(ps.t)))
186	209		if istrip
187	210		if lcp !== nothing && !isempty(lcp)
188	211		for expr in exprs_to_adjust

195	218		end
196	219		end
197	220		# Drop leading newline
198		-	if isliteral(ret.args[1]) && kindof(ret.args[1]) in single_string_T &&
	221	+	if !isempty(ret.args) && isliteral(ret.args[1]) && headof(ret.args[1]) in single_string_T &&
199	222		!isempty(valof(ret.args[1])) && valof(ret.args[1])[1] == '\n'
200	223		ret.args[1] = dropleadlingnewline(ret.args[1])
201	224		end
202	225		end
203	226
204		-	if (length(ret.args) == 1 && isliteral(ret.args[1]) && kindof(ret.args[1]) in single_string_T)
	227	+	if (length(ret.args) == 1 && isliteral(ret.args[1]) && headof(ret.args[1]) in single_string_T) && !isinterpolated
205	228		ret = ret.args[1]
206	229		end
207	230		update_span!(ret)
208	231
209		-	return ret
	232	+	return iscmd ? wrapwithcmdmacro(ret) : ret
210	233		end
211	234
212	235		function adjustspan(x::EXPR)

13	13		(ps.closer.inwhere && kindof(ps.nt) === Tokens.WHERE) \|\|
14	14		(ps.closer.inwhere && ps.closer.ws && kindof(ps.t) === Tokens.RPAREN && isoperator(ps.nt) && precedence(ps.nt) < DeclarationOp) \|\|
15	15		(ps.closer.precedence > WhereOp && (
16		-	(kindof(ps.nt) === Tokens.LPAREN && !(ps.t.kind === Tokens.EX_OR)) \|\|
	16	+	(kindof(ps.nt) === Tokens.LPAREN && !(kindof(ps.t) === Tokens.EX_OR)) \|\|
17	17		kindof(ps.nt) === Tokens.LBRACE \|\|
18	18		kindof(ps.nt) === Tokens.LSQUARE \|\|
19	19		(kindof(ps.nt) === Tokens.STRING && isemptyws(ps.ws)) \|\|
20	20		((kindof(ps.nt) === Tokens.RPAREN \|\| kindof(ps.nt) === Tokens.RSQUARE) && isidentifier(ps.nt))
21	21		)) \|\|
22		-	(iscomma(ps.nt) && ps.closer.precedence > 0) \|\|
	22	+	(iscomma(ps.nt) && ps.closer.precedence > AssignmentOp) \|\|
	23	+	kindof(ps.nt) === Tokens.ENDMARKER \|\|
23	24		(ps.closer.comma && iscomma(ps.nt)) \|\|
24		-	(ps.closer.tuple && (iscomma(ps.nt) \|\| isassignment(ps.nt))) \|\|
	25	+	(ps.closer.tuple && (iscomma(ps.nt) \|\| isassignmentop(ps.nt))) \|\|
25	26		(kindof(ps.nt) === Tokens.FOR && ps.closer.precedence > -1) \|\|
26	27		(ps.closer.block && kindof(ps.nt) === Tokens.END) \|\|
27	28		(ps.closer.paren && kindof(ps.nt) === Tokens.RPAREN) \|\|

41	42		!(kindof(ps.nt) === Tokens.DO) &&
42	43		!(
43	44		(isbinaryop(ps.nt) && !(ps.closer.wsop && isemptyws(ps.nws) && isunaryop(ps.nt) && precedence(ps.nt) > 7)) \|\|
44		-	(isunaryop(ps.t) && kindof(ps.ws) == WS && ps.lt.kind !== CSTParser.Tokens.COLON)
	45	+	(isunaryop(ps.t) && kindof(ps.ws) == WS && kindof(ps.lt) !== CSTParser.Tokens.COLON)
45	46		)) \|\|
46	47		(ps.closer.unary && (kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT, Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE) && isidentifier(ps.nt)))
47	48		end

201	202		end
202	203
203	204
204		-
205		-	isidentifier(x::EXPR) = typof(x) === IDENTIFIER \|\| typof(x) === NONSTDIDENTIFIER
206		-
207		-	isunarycall(x::EXPR) = typof(x) === UnaryOpCall
208		-	isbinarycall(x::EXPR) = typof(x) === BinaryOpCall
209		-	iswherecall(x::EXPR) = typof(x) === WhereOpCall
210		-	isdeclaration(x::EXPR) = isbinarycall(x) && is_decl(x[2])
211		-	isinterpolant(x::EXPR) = isunarycall(x) && is_exor(x[1])
212		-	istuple(x::EXPR) = typof(x) === TupleH
213		-	is_either_id_op_interp(x::EXPR) = isidentifier(x) \|\| isoperator(x) \|\| isinterpolant(x)
214		-	is_splat(x::EXPR) = isunarycall(x) && is_dddot(x[2])
215		-
216		-
217		-	isliteral(x::EXPR) = typof(x) === LITERAL
218		-	iskw(x::EXPR) = typof(x) === KEYWORD # TODO: should change to `iskeyword`
219		-	ispunctuation(x::EXPR) = typof(x) === PUNCTUATION
220		-
221		-	isstring(x) = typof(x) === StringH \|\| (isliteral(x) && (kindof(x) === Tokens.STRING \|\| kindof(x) === Tokens.TRIPLE_STRING))
222		-	is_integer(x) = isliteral(x) && kindof(x) === Tokens.INTEGER
223		-	is_float(x) = isliteral(x) && kindof(x) === Tokens.FLOAT
224		-	is_number(x) = isliteral(x) && (kindof(x) === Tokens.INTEGER \|\| kindof(x) === Tokens.FLOAT)
225		-	is_nothing(x) = isliteral(x) && kindof(x) === Tokens.NOTHING
226		-
227		-	isajuxtaposition(ps::ParseState, ret::EXPR) = ((is_number(ret) && (isidentifier(ps.nt) \|\| kindof(ps.nt) === Tokens.LPAREN \|\| kindof(ps.nt) === Tokens.CMD \|\| kindof(ps.nt) === Tokens.STRING \|\| kindof(ps.nt) === Tokens.TRIPLE_STRING)) \|\|
228		-	((typof(ret) === UnaryOpCall && is_prime(ret.args[2]) && isidentifier(ps.nt)) \|\|
	205	+	isajuxtaposition(ps::ParseState, ret::EXPR) = ((isnumber(ret) && (isidentifier(ps.nt) \|\| kindof(ps.nt) === Tokens.LPAREN \|\| kindof(ps.nt) === Tokens.CMD \|\| kindof(ps.nt) === Tokens.STRING \|\| kindof(ps.nt) === Tokens.TRIPLE_STRING)) \|\|
	206	+	((is_prime(ret.head) && isidentifier(ps.nt)) \|\|
229	207		((kindof(ps.t) === Tokens.RPAREN \|\| kindof(ps.t) === Tokens.RSQUARE) && (isidentifier(ps.nt) \|\| kindof(ps.nt) === Tokens.CMD)) \|\|
230	208		((kindof(ps.t) === Tokens.STRING \|\| kindof(ps.t) === Tokens.TRIPLE_STRING) && (kindof(ps.nt) === Tokens.STRING \|\| kindof(ps.nt) === Tokens.TRIPLE_STRING)))) \|\| ((kindof(ps.t) in (Tokens.INTEGER, Tokens.FLOAT) \|\| kindof(ps.t) in (Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE)) && isidentifier(ps.nt))
231	209

237	215		`ParseState`, or exists as a (sub) expression of `x`.
238	216		"""
239	217		function has_error(x::EXPR)
240		-	return typof(x) == ErrorToken \|\| (x.args !== nothing && any(has_error, x.args))
	218	+	return headof(x) == :errortoken \|\| (x.args !== nothing && any(has_error, x.args)) \|\| (x.trivia !== nothing && any(has_error, x.trivia))
241	219		end
242	220		has_error(ps::ParseState) = ps.errored
243	221
244		-	# When using the FancyDiagnostics package, Meta.parse is the
245		-	# same as CSTParser.parse. Manually call the flisp parser here
246		-	# to make sure we test what we want, even when people load the
247		-	# FancyDiagnostics package.
248		-	function flisp_parse(str::AbstractString, pos::Int; greedy::Bool=true, raise::Bool=true)
249		-	if VERSION < v"1.6-DEV"
250		-	bstr = String(str)
251		-	ex, pos = ccall(:jl_parse_string, Any,
252		-	(Ptr{UInt8}, Csize_t, Int32, Int32),
253		-	bstr, sizeof(bstr), pos - 1, greedy ? 1 : 0)
254		-	else
255		-	filename = "none"
256		-	rule = greedy ? :statement : :atom
257		-	ex, pos = Core.Compiler.fl_parse(str, filename, pos - 1, rule)
258		-	end
259		-	if raise && isa(ex, Expr) && ex.head === :error
260		-	throw(Meta.ParseError(ex.args[1]))
261		-	end
262		-	if ex === ()
263		-	raise && throw(Meta.ParseError("end of input"))
264		-	ex = Expr(:error, "end of input")
265		-	end
266		-	# pos is zero-based byte offset
267		-	return ex, pos + 1
268		-	end
269		-
270		-	function flisp_parse(str::AbstractString; raise::Bool=true)
271		-	ex, pos = flisp_parse(str, 1, greedy=true, raise=raise)
272		-	if isa(ex, Expr) && ex.head === :error
273		-	return ex
274		-	end
275		-	if !(pos > ncodeunits(str))
276		-	raise && throw(Meta.ParseError("extra token after end of expression"))
277		-	return Expr(:error, "extra token after end of expression")
278		-	end
279		-	return ex
280		-	end
281		-
282		-	function flisp_parse(stream::IO; greedy::Bool=true, raise::Bool=true)
283		-	pos = position(stream)
284		-	ex, Δ = flisp_parse(read(stream, String), 1, greedy=greedy, raise=raise)
285		-	seek(stream, pos + Δ - 1)
286		-	return ex
287		-	end
288		-
289	222		using Base.Meta
290	223
291	224		function norm_ast(a::Any)

328	261		return a
329	262		end
330	263
331		-	function flisp_parsefile(str, display=true)
332		-	io = IOBuffer(str)
	264	+	function flisp_parsefile(str, display = true)
	265	+	pos = 1
333	266		failed = false
334	267		x1 = Expr(:file)
335	268		try
336		-	while !eof(io)
337		-	push!(x1.args, flisp_parse(io))
	269	+	while pos <= sizeof(str)
	270	+	x, pos = Meta.parse(str, pos)
	271	+	push!(x1.args, x)
338	272		end
339	273		catch er
340	274		isa(er, InterruptException) && rethrow(er)

359	293		if length(x.args) > 0 && is_nothing(x.args[1])
360	294		popfirst!(x.args)
361	295		end
362		-	if length(x.args) > 0 && is_nothing(x.args[end])
363		-	pop!(x.args)
364		-	end
	296	+	# if length(x.args) > 0 && is_nothing(x.args[end])
	297	+	# pop!(x.args)
	298	+	# end
365	299		x0 = norm_ast(Expr(x))
366	300		x0, has_error(ps), sp
367	301		end

386	320		cumfail = 0
387	321		printstyled(file, color=:green)
388	322		println()
389		-	c0, c1 = CSTParser.compare(x0, x1)
390		-	printstyled(string(" ", c0), bold=true, color=:ligth_red)
	323	+	c0, c1 = compare(x0, x1)
	324	+	printstyled(string(" ", c0), bold = true, color = :light_red)
391	325		println()
392	326		printstyled(string(" ", c1), bold=true, color=:light_green)
393	327		println()

507	558		Attempt to get a string representation of a nodeless expression.
508	559		"""
509	560		function str_value(x)
510		-	if typof(x) === IDENTIFIER \|\| typof(x) === LITERAL
	561	+	if headof(x) === :IDENTIFIER \|\| isliteral(x)
511	562		return valof(x)
512	563		elseif isidentifier(x)
513	564		valof(x.args[2])
514		-	elseif typof(x) === OPERATOR \|\| typof(x) === MacroName
	565	+	elseif isoperator(x)
515	566		return string(Expr(x))
516	567		else
517	568		return ""

1		-	function Base.show(io::IO, x::EXPR, offset=0, d=0, er=false)
2		-	T = typof(x)
3		-	c = T === ErrorToken \|\| er ? :red : :normal
	1	+	function Base.show(io::IO, x::EXPR, offset = 0, d = 0, er = false)
	2	+	T = headof(x)
	3	+	c = T === :errortoken \|\| er ? :red : :normal
4	4		# Print span as 1-based range of the source string. This presentation is
5	5		# simple to understand when strings are presented to CSTParser.parse().
6	6		print(io, lpad(offset + 1, 3), ":", rpad(offset + x.fullspan, 3), " ")
7	7		if isidentifier(x)
8		-	printstyled(io, " "^d, typof(x) == NONSTDIDENTIFIER ? valof(x.args[2]) : valof(x), color=:yellow)
	8	+	printstyled(io, " "^d, headof(x) == :NONSTDIDENTIFIER ? valof(x.args[2]) : valof(x), color = :yellow)
9	9		x.meta !== nothing && show(io, x.meta)
10		-	println(io)
	10	+	print(io)
11	11		elseif isoperator(x)
12		-	printstyled(io, " "^d, "OP: ", kindof(x), "\n", color=c)
13		-	elseif iskw(x)
14		-	printstyled(io, " "^d, kindof(x), "\n", color=:magenta)
	12	+	printstyled(io, " "^d, "OP: ", valof(x), color = c)
	13	+	elseif iskeyword(x)
	14	+	printstyled(io, " "^d, headof(x), color = :magenta)
15	15		elseif ispunctuation(x)
16		-	if kindof(x) === Tokens.LPAREN
17		-	printstyled(io, " "^d, "(\n", color=c)
18		-	elseif kindof(x) === Tokens.RPAREN
19		-	printstyled(io, " "^d, ")\n", color=c)
20		-	elseif kindof(x) === Tokens.LSQUARE
21		-	printstyled(io, " "^d, "[\n", color=c)
22		-	elseif kindof(x) === Tokens.RSQUARE
23		-	printstyled(io, " "^d, "]\n", color=c)
24		-	elseif kindof(x) === Tokens.COMMA
25		-	printstyled(io, " "^d, ",\n", color=c)
26		-	else
27		-	printstyled(io, " "^d, "PUNC: ", kindof(x), "\n", color=c)
28		-	end
	16	+	printstyled(io, " "^d, punctuationprinting[headof(x)], color = c)
29	17		elseif isliteral(x)
30		-	printstyled(io, " "^d, "$(kindof(x)): ", valof(x), "\n", color=c)
	18	+	printstyled(io, " "^d, "$(headof(x)): ", valof(x) === nothing ? "nothing" : valof(x), color = c)
31	19		else
32	20		printstyled(io, " "^d, T, color=c)
33	21		if x.meta !== nothing