internals.jl at fc030f1 ⋅ julia-vscode/CSTParser.jl

const term_c = (Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE, Tokens.END, Tokens.ELSE, Tokens.ELSEIF, Tokens.CATCH, Tokens.FINALLY, Tokens.ENDMARKER)

"""
Continue parsing statements until an element of `closers` is hit (usually
`end`). Statements are grouped in a `Block` EXPR.
"""
function parse_block(ps::ParseState, ret::Vector{EXPR}=EXPR[], closers=(Tokens.END,), docable=false)
    prevpos = position(ps)
    while kindof(ps.nt) ∉ closers # loop until an expected closer is hit
        if kindof(ps.nt) ∈ term_c # error handling if an unexpected closer is hit
            if kindof(ps.nt) === Tokens.ENDMARKER
                break
            elseif kindof(ps.nt) === Tokens.RPAREN
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
            elseif kindof(ps.nt) === Tokens.RBRACE
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
            elseif kindof(ps.nt) === Tokens.RSQUARE
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
            else
                push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
            end
        else
            if docable
                a = parse_doc(ps)
            else
                a = parse_expression(ps)
            end
            push!(ret, a)
        end
        prevpos = loop_check(ps, prevpos)
    end
    return ret
end

"""
Parses an iterator, allowing for the preceding keyword `outer`. Returns an
error expression if an invalid expression is parsed (anything other than
`=`, `in`, `∈`).
"""
function parse_iterator(ps::ParseState, outer=parse_outer(ps))
    arg = @closer ps :range @closer ps :ws parse_expression(ps)
    if !is_range(arg)
        arg = mErrorToken(ps, arg, InvalidIterator)
    end
    if outer !== nothing
        arg.args[1] = setparent!(EXPR(Outer, EXPR[outer, arg.args[1]]), arg)
        arg.fullspan += outer.fullspan
        arg.span = outer.fullspan + arg.span
    end
    return arg
end

function parse_outer(ps)
    if kindof(ps.nt) === Tokens.OUTER && kindof(ps.nws) !== EmptyWS && !Tokens.isoperator(kindof(ps.nnt))
        outer = INSTANCE(next(ps))
    end
end

"""
    parse_iterators(ps::ParseState, allowfilter = false)

Parses a group of iterators e.g. used in a `for` loop or generator. Can allow
for a succeeding `Filter` expression.
"""
function parse_iterators(ps::ParseState, allowfilter=false)
    arg = parse_iterator(ps)
    if iscomma(ps.nt) # we've hit a comma separated list of iterators.
        arg = EXPR(Block, EXPR[arg])
        prevpos = position(ps)
        while iscomma(ps.nt)
            accept_comma(ps, arg)
            nextarg = parse_iterator(ps)
            push!(arg, nextarg)
            prevpos = loop_check(ps, prevpos)
        end
    end

    if allowfilter
        arg = parse_filter(ps, arg)
    end
    return arg
end

"""
parse_filter(ps::ParseState, arg)

Parse a conditional filter following a generator.
"""
function parse_filter(ps::ParseState, arg)
    if kindof(ps.nt) === Tokens.IF # assumes we're inside a generator
        if typof(arg) === Block
            arg = EXPR(Filter, arg.args)
        else
            arg = EXPR(Filter, EXPR[arg])
        end
        push!(arg, mKEYWORD(next(ps)))
        cond = @closer ps :range parse_expression(ps)
        push!(arg, cond)
    end
    return arg
end

"""
    parse_call(ps, ret)

Parses a function call. Expects to start before the opening parentheses and is passed the expression declaring the function name, `ret`.
"""
function parse_call(ps::ParseState, ret::EXPR, ismacro=false)
    if is_minus(ret) || is_not(ret)
        arg = @closer ps :unary @closer ps :inwhere @precedence ps PowerOp parse_expression(ps)
        if istuple(arg)
            pushfirst!(arg.args, ret)
            ret = EXPR(Call, arg.args)
        elseif iswherecall(arg) && istuple(arg.args[1])
            ret = mWhereOpCall(EXPR(Call, EXPR[ret; arg.args[1].args]), arg.args[2], arg.args[3:end])
        else
            ret = mUnaryOpCall(ret, arg)
        end
    elseif is_and(ret) || is_decl(ret) || is_exor(ret)
        arg = @precedence ps 20 parse_expression(ps)
        if is_exor(ret) && istuple(arg) && length(arg) == 3 && is_splat(arg.args[2])
            arg = EXPR(InvisBrackets, arg.args)
        end
        ret = mUnaryOpCall(ret, arg)
    elseif is_issubt(ret) || is_issupt(ret)
        arg = @precedence ps PowerOp parse_expression(ps)
        ret = EXPR(Call, EXPR[ret; arg.args])
    else
        !ismacro && typof(ret) === MacroName && (ismacro = true)
        args = EXPR[ret, mPUNCTUATION(next(ps))]
        @closeparen ps @default ps parse_comma_sep(ps, args, !ismacro)
        accept_rparen(ps, args)
        ret = EXPR(ismacro ? MacroCall : Call, args)
    end
    return ret
end

"""
Parses a comma separated list, optionally allowing for conversion of 
assignment (`=`) expressions to `Kw`.
"""
function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, kw=true, block=false, istuple=false)
    prevpos = position(ps)
    @nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while !closer(ps)
        a = parse_expression(ps)
        if kw && _do_kw_convert(ps, a)
            a = _kw_convert(a)
        end
        push!(args, a)
        if iscomma(ps.nt)
            accept_comma(ps, args)
        else# if kindof(ps.ws) == SemiColonWS
            break
        end
        prevpos = loop_check(ps, prevpos)
    end
    if istuple && length(args) > 2
        block = false
    end

    if kindof(ps.ws) == SemiColonWS
        if @nocloser ps :newline @closer ps :comma @nocloser ps :semicolon closer(ps)
            if block && !(length(args) == 1 && ispunctuation(args[1])) && !(typof(last(args)) === UnaryOpCall && is_dddot(last(args).args[2]))
                push!(args, EXPR(Block, EXPR[pop!(args)]))
            elseif kw && kindof(ps.nt) === Tokens.RPAREN
                push!(args, EXPR(Parameters, EXPR[], 0, 0))
            end
        else
            a = @nocloser ps :newline @closer ps :comma @nocloser ps :inwhere parse_expression(ps)
            if block && !(length(args) == 1 && ispunctuation(args[1])) && !is_splat(last(args)) && !(istuple && iscomma(ps.nt))
                args1 = EXPR[pop!(args), a]
                prevpos = position(ps)
                @nocloser ps :newline @closer ps :comma while @nocloser ps :semicolon !closer(ps)
                    a = parse_expression(ps)
                    push!(args1, a)
                    prevpos = loop_check(ps, prevpos)
                end
                body = EXPR(Block, args1)
                push!(args, body)
                args = body
            else
                parse_parameters(ps, args, EXPR[a])
            end
        end
    end
    return
end

"""
    parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR} = EXPR[]; usekw = true)

Parses parameter arguments for a function call (e.g. following a semicolon).
"""
function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR}=EXPR[]; usekw=true)
    isfirst = isempty(args1)
    prevpos = position(ps)
    @nocloser ps :inwhere @nocloser ps :newline  @closer ps :comma while !isfirst || (@nocloser ps :semicolon !closer(ps))
        if isfirst
            a = parse_expression(ps)
        else
            a = first(args1)
        end
        if usekw && _do_kw_convert(ps, a)
            a = _kw_convert(a)
        end
        if isfirst
            push!(args1, a)
        else
            pop!(args1)
            push!(args1, a)
        end
        if iscomma(ps.nt)
            accept_comma(ps, args1)
        end
        if kindof(ps.ws) == SemiColonWS
            parse_parameters(ps, args1; usekw=usekw)
        end
        if isfirst
            prevpos = loop_check(ps, prevpos)
        else
            prevpos = position(ps)
        end
        isfirst = true
    end
    if !isempty(args1)
        paras = EXPR(Parameters, args1)
        push!(args, paras)
    end
    return
end

"""
    parse_macrocall(ps)

Parses a macro call. Expects to start on the `@`.
"""
function parse_macrocall(ps::ParseState)
    at = mPUNCTUATION(ps)
    if !isemptyws(ps.ws)
        mname = mErrorToken(ps, INSTANCE(next(ps)), UnexpectedWhiteSpace)
    else
        mname = EXPR(MacroName, EXPR[at, mIDENTIFIER(next(ps))])
    end

    # Handle cases with @ at start of dotted expressions
    if kindof(ps.nt) === Tokens.DOT && isemptyws(ps.ws)
        prevpos = position(ps)
        while kindof(ps.nt) === Tokens.DOT
            op = mOPERATOR(next(ps))
            nextarg = mIDENTIFIER(next(ps))
            mname = mBinaryOpCall(mname, op, EXPR(Quotenode, EXPR[nextarg]))
            prevpos = loop_check(ps, prevpos)
        end
    end

    if iscomma(ps.nt)
        return EXPR(MacroCall, EXPR[mname], mname.fullspan, mname.span)
    elseif isemptyws(ps.ws) && kindof(ps.nt) === Tokens.LPAREN
        return parse_call(ps, mname, true)
    else
        args = EXPR[mname]
        insquare = ps.closer.insquare
        prevpos = position(ps)
        @default ps while !closer(ps)
            if insquare
                a = @closer ps :insquare @closer ps :inmacro @closer ps :ws @closer ps :wsop parse_expression(ps)
            else
                a = @closer ps :inmacro @closer ps :ws @closer ps :wsop parse_expression(ps)
            end
            push!(args, a)
            if insquare && kindof(ps.nt) === Tokens.FOR
                break
            end
            prevpos = loop_check(ps, prevpos)
        end
        return EXPR(MacroCall, args)
    end
end

"""
parse_generator(ps)

Having hit `for` not at the beginning of an expression return a generator.
Comprehensions are parsed as SQUAREs containing a generator.
"""
function parse_generator(ps::ParseState, ret::EXPR)
    kw = mKEYWORD(next(ps))
    ret = EXPR(Generator, EXPR[ret, kw])
    ranges = @closesquare ps parse_iterators(ps, true)

    if typof(ranges) === Block
        append!(ret, ranges)
    else
        push!(ret, ranges)
    end

    if typof(ret.args[1]) === Generator || typof(ret.args[1]) === Flatten
        ret = EXPR(Flatten, EXPR[ret])
    end

    return ret
end

"""
Helper function for parsing import/using statements.
"""
function parse_dot_mod(ps::ParseState, is_colon=false)
    args = EXPR[]

    prevpos = position(ps)
    while kindof(ps.nt) === Tokens.DOT || kindof(ps.nt) === Tokens.DDOT || kindof(ps.nt) === Tokens.DDDOT
        d = mOPERATOR(next(ps))
        trailing_ws = d.fullspan - d.span
        if is_dot(d)
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
        elseif is_ddot(d)
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
        elseif is_dddot(d)
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
            push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
            push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
        end
        prevpos = loop_check(ps, prevpos)
    end

    # import/export ..
    # TODO: Not clear what this is for?
    # if iscomma(ps.nt) || kindof(ps.ws) == NewLineWS || kindof(ps.nt) === Tokens.ENDMARKER
    #     if length(args) == 2
    #         return EXPR[INSTANCE(ps)]
    #     end
    # end

    prevpos = position(ps)
    while true
        if kindof(ps.nt) === Tokens.AT_SIGN
            at = mPUNCTUATION(next(ps))
            a = INSTANCE(next(ps))
            push!(args, EXPR(MacroName, EXPR[at, a]))
        elseif kindof(ps.nt) === Tokens.LPAREN
            a = EXPR(InvisBrackets, EXPR[mPUNCTUATION(next(ps))])
            push!(a, @closeparen ps parse_expression(ps))
            accept_rparen(ps, a)
            push!(args, a)
        elseif kindof(ps.nt) === Tokens.EX_OR
            a = @closer ps :comma parse_expression(ps)
            push!(args, a)
        elseif !is_colon && isoperator(ps.nt)
            next(ps)
            push!(args, mOPERATOR(ps.nt.startbyte - ps.t.startbyte,  1 + ps.t.endbyte - ps.t.startbyte, kindof(ps.t), false))
        elseif VERSION > v"1.3.0-" && isidentifier(ps.nt) && isemptyws(ps.nws) && (kindof(ps.nnt) === Tokens.STRING || kindof(ps.nnt) === Tokens.TRIPLE_STRING)
            push!(args, EXPR(NONSTDIDENTIFIER, EXPR[INSTANCE(next(ps)), INSTANCE(next(ps))]))
        else
            push!(args, INSTANCE(next(ps)))
        end

        if kindof(ps.nt) === Tokens.DOT
            push!(args, mPUNCTUATION(next(ps)))
        elseif isoperator(ps.nt) && (ps.nt.dotop || kindof(ps.nt) === Tokens.DOT)
            push!(args, mPUNCTUATION(Tokens.DOT, 1, 1))
            ps.nt = RawToken(kindof(ps.nt), ps.nt.startpos, ps.nt.endpos, ps.nt.startbyte + 1, ps.nt.endbyte, ps.nt.token_error, false, ps.nt.suffix)
        else
            break
        end
        prevpos = loop_check(ps, prevpos)
    end
    args
end

Read our documentation on viewing source code .

Navigation	Overlay
`t` Navigate files	`h` Toggle hits
`y` Change url to tip of branch	`m` Toggle misses
`b / v` Jump to prev/next hit line	`p` Toggle partial
`z / x` Jump to prev/next missed or partial line	`1..9` Toggle flags
`shift + o` Open current page in GitHub	`a` Toggle all on
`/ or ?` Show keyboard shortcuts dialog	`c` Toggle context lines or commits

1		const term_c = (Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE, Tokens.END, Tokens.ELSE, Tokens.ELSEIF, Tokens.CATCH, Tokens.FINALLY, Tokens.ENDMARKER)
2
3		"""
4		Continue parsing statements until an element of `closers` is hit (usually
5		`end`). Statements are grouped in a `Block` EXPR.
6		"""
7		function parse_block(ps::ParseState, ret::Vector{EXPR}=EXPR[], closers=(Tokens.END,), docable=false)
8	23	prevpos = position(ps)
9	23	while kindof(ps.nt) ∉ closers # loop until an expected closer is hit
10	23	if kindof(ps.nt) ∈ term_c # error handling if an unexpected closer is hit
11	0	if kindof(ps.nt) === Tokens.ENDMARKER
12	0	break
13	0	elseif kindof(ps.nt) === Tokens.RPAREN
14	0	push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
15	0	elseif kindof(ps.nt) === Tokens.RBRACE
16	0	push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
17	0	elseif kindof(ps.nt) === Tokens.RSQUARE
18	0	push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
19		else
20	0	push!(ret, mErrorToken(ps, INSTANCE(next(ps)), UnexpectedToken))
21		end
22		else
23	23	if docable
24	23	a = parse_doc(ps)
25		else
26	23	a = parse_expression(ps)
27		end
28	23	push!(ret, a)
29		end
30	23	prevpos = loop_check(ps, prevpos)
31		end
32	23	return ret
33		end
34
35		"""
36		Parses an iterator, allowing for the preceding keyword `outer`. Returns an
37		error expression if an invalid expression is parsed (anything other than
38		`=`, `in`, `∈`).
39		"""
40		function parse_iterator(ps::ParseState, outer=parse_outer(ps))
41	23	arg = @closer ps :range @closer ps :ws parse_expression(ps)
42	23	if !is_range(arg)
43	0	arg = mErrorToken(ps, arg, InvalidIterator)
44		end
45	23	if outer !== nothing
46	23	arg.args[1] = setparent!(EXPR(Outer, EXPR[outer, arg.args[1]]), arg)
47	21	arg.fullspan += outer.fullspan
48	21	arg.span = outer.fullspan + arg.span
49		end
50	23	return arg
51		end
52
53		function parse_outer(ps)
54	23	if kindof(ps.nt) === Tokens.OUTER && kindof(ps.nws) !== EmptyWS && !Tokens.isoperator(kindof(ps.nnt))
55	23	outer = INSTANCE(next(ps))
56		end
57		end
58
59		"""
60		parse_iterators(ps::ParseState, allowfilter = false)
61
62		Parses a group of iterators e.g. used in a `for` loop or generator. Can allow
63		for a succeeding `Filter` expression.
64		"""
65		function parse_iterators(ps::ParseState, allowfilter=false)
66	23	arg = parse_iterator(ps)
67	23	if iscomma(ps.nt) # we've hit a comma separated list of iterators.
68	23	arg = EXPR(Block, EXPR[arg])
69	21	prevpos = position(ps)
70	23	while iscomma(ps.nt)
71	21	accept_comma(ps, arg)
72	23	nextarg = parse_iterator(ps)
73	23	push!(arg, nextarg)
74	23	prevpos = loop_check(ps, prevpos)
75		end
76		end
77
78	23	if allowfilter
79	23	arg = parse_filter(ps, arg)
80		end
81	23	return arg
82		end
83
84		"""
85		parse_filter(ps::ParseState, arg)
86
87		Parse a conditional filter following a generator.
88		"""
89		function parse_filter(ps::ParseState, arg)
90	23	if kindof(ps.nt) === Tokens.IF # assumes we're inside a generator
91	23	if typof(arg) === Block
92	23	arg = EXPR(Filter, arg.args)
93		else
94	23	arg = EXPR(Filter, EXPR[arg])
95		end
96	23	push!(arg, mKEYWORD(next(ps)))
97	21	cond = @closer ps :range parse_expression(ps)
98	23	push!(arg, cond)
99		end
100	23	return arg
101		end
102
103		"""
104		parse_call(ps, ret)
105
106		Parses a function call. Expects to start before the opening parentheses and is passed the expression declaring the function name, `ret`.
107		"""
108		function parse_call(ps::ParseState, ret::EXPR, ismacro=false)
109	23	if is_minus(ret) \|\| is_not(ret)
110	21	arg = @closer ps :unary @closer ps :inwhere @precedence ps PowerOp parse_expression(ps)
111	23	if istuple(arg)
112	23	pushfirst!(arg.args, ret)
113	23	ret = EXPR(Call, arg.args)
114	23	elseif iswherecall(arg) && istuple(arg.args[1])
115	0	ret = mWhereOpCall(EXPR(Call, EXPR[ret; arg.args[1].args]), arg.args[2], arg.args[3:end])
116		else
117	23	ret = mUnaryOpCall(ret, arg)
118		end
119	23	elseif is_and(ret) \|\| is_decl(ret) \|\| is_exor(ret)
120	21	arg = @precedence ps 20 parse_expression(ps)
121	23	if is_exor(ret) && istuple(arg) && length(arg) == 3 && is_splat(arg.args[2])
122	0	arg = EXPR(InvisBrackets, arg.args)
123		end
124	23	ret = mUnaryOpCall(ret, arg)
125	23	elseif is_issubt(ret) \|\| is_issupt(ret)
126	21	arg = @precedence ps PowerOp parse_expression(ps)
127	23	ret = EXPR(Call, EXPR[ret; arg.args])
128		else
129	23	!ismacro && typof(ret) === MacroName && (ismacro = true)
130	23	args = EXPR[ret, mPUNCTUATION(next(ps))]
131	21	@closeparen ps @default ps parse_comma_sep(ps, args, !ismacro)
132	21	accept_rparen(ps, args)
133	23	ret = EXPR(ismacro ? MacroCall : Call, args)
134		end
135	23	return ret
136		end
137
138		"""
139		Parses a comma separated list, optionally allowing for conversion of
140		assignment (`=`) expressions to `Kw`.
141		"""
142		function parse_comma_sep(ps::ParseState, args::Vector{EXPR}, kw=true, block=false, istuple=false)
143	23	prevpos = position(ps)
144	21	@nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while !closer(ps)
145	23	a = parse_expression(ps)
146	23	if kw && _do_kw_convert(ps, a)
147	23	a = _kw_convert(a)
148		end
149	23	push!(args, a)
150	23	if iscomma(ps.nt)
151	23	accept_comma(ps, args)
152		else# if kindof(ps.ws) == SemiColonWS
153	0	break
154		end
155	21	prevpos = loop_check(ps, prevpos)
156		end
157	23	if istuple && length(args) > 2
158	0	block = false
159		end
160
161	23	if kindof(ps.ws) == SemiColonWS
162	23	if @nocloser ps :newline @closer ps :comma @nocloser ps :semicolon closer(ps)
163	23	if block && !(length(args) == 1 && ispunctuation(args[1])) && !(typof(last(args)) === UnaryOpCall && is_dddot(last(args).args[2]))
164	23	push!(args, EXPR(Block, EXPR[pop!(args)]))
165	23	elseif kw && kindof(ps.nt) === Tokens.RPAREN
166	23	push!(args, EXPR(Parameters, EXPR[], 0, 0))
167		end
168		else
169	21	a = @nocloser ps :newline @closer ps :comma @nocloser ps :inwhere parse_expression(ps)
170	23	if block && !(length(args) == 1 && ispunctuation(args[1])) && !is_splat(last(args)) && !(istuple && iscomma(ps.nt))
171	23	args1 = EXPR[pop!(args), a]
172	21	prevpos = position(ps)
173	21	@nocloser ps :newline @closer ps :comma while @nocloser ps :semicolon !closer(ps)
174	23	a = parse_expression(ps)
175	21	push!(args1, a)
176	21	prevpos = loop_check(ps, prevpos)
177		end
178	23	body = EXPR(Block, args1)
179	23	push!(args, body)
180	23	args = body
181		else
182	23	parse_parameters(ps, args, EXPR[a])
183		end
184		end
185		end
186	23	return
187		end
188
189		"""
190		parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR} = EXPR[]; usekw = true)
191
192		Parses parameter arguments for a function call (e.g. following a semicolon).
193		"""
194		function parse_parameters(ps::ParseState, args::Vector{EXPR}, args1::Vector{EXPR}=EXPR[]; usekw=true)
195	23	isfirst = isempty(args1)
196	21	prevpos = position(ps)
197	21	@nocloser ps :inwhere @nocloser ps :newline @closer ps :comma while !isfirst \|\| (@nocloser ps :semicolon !closer(ps))
198	23	if isfirst
199	23	a = parse_expression(ps)
200		else
201	21	a = first(args1)
202		end
203	23	if usekw && _do_kw_convert(ps, a)
204	23	a = _kw_convert(a)
205		end
206	23	if isfirst
207	23	push!(args1, a)
208		else
209	21	pop!(args1)
210	21	push!(args1, a)
211		end
212	23	if iscomma(ps.nt)
213	21	accept_comma(ps, args1)
214		end
215	23	if kindof(ps.ws) == SemiColonWS
216	0	parse_parameters(ps, args1; usekw=usekw)
217		end
218	23	if isfirst
219	23	prevpos = loop_check(ps, prevpos)
220		else
221	21	prevpos = position(ps)
222		end
223	23	isfirst = true
224		end
225	23	if !isempty(args1)
226	23	paras = EXPR(Parameters, args1)
227	21	push!(args, paras)
228		end
229	23	return
230		end
231
232		"""
233		parse_macrocall(ps)
234
235		Parses a macro call. Expects to start on the `@`.
236		"""
237		function parse_macrocall(ps::ParseState)
238	23	at = mPUNCTUATION(ps)
239	23	if !isemptyws(ps.ws)
240	0	mname = mErrorToken(ps, INSTANCE(next(ps)), UnexpectedWhiteSpace)
241		else
242	23	mname = EXPR(MacroName, EXPR[at, mIDENTIFIER(next(ps))])
243		end
244
245		# Handle cases with @ at start of dotted expressions
246	23	if kindof(ps.nt) === Tokens.DOT && isemptyws(ps.ws)
247	21	prevpos = position(ps)
248	21	while kindof(ps.nt) === Tokens.DOT
249	21	op = mOPERATOR(next(ps))
250	21	nextarg = mIDENTIFIER(next(ps))
251	21	mname = mBinaryOpCall(mname, op, EXPR(Quotenode, EXPR[nextarg]))
252	21	prevpos = loop_check(ps, prevpos)
253		end
254		end
255
256	23	if iscomma(ps.nt)
257	23	return EXPR(MacroCall, EXPR[mname], mname.fullspan, mname.span)
258	23	elseif isemptyws(ps.ws) && kindof(ps.nt) === Tokens.LPAREN
259	23	return parse_call(ps, mname, true)
260		else
261	21	args = EXPR[mname]
262	21	insquare = ps.closer.insquare
263	21	prevpos = position(ps)
264	21	@default ps while !closer(ps)
265	23	if insquare
266	23	a = @closer ps :insquare @closer ps :inmacro @closer ps :ws @closer ps :wsop parse_expression(ps)
267		else
268	21	a = @closer ps :inmacro @closer ps :ws @closer ps :wsop parse_expression(ps)
269		end
270	23	push!(args, a)
271	23	if insquare && kindof(ps.nt) === Tokens.FOR
272	23	break
273		end
274	21	prevpos = loop_check(ps, prevpos)
275		end
276	23	return EXPR(MacroCall, args)
277		end
278		end
279
280		"""
281		parse_generator(ps)
282
283		Having hit `for` not at the beginning of an expression return a generator.
284		Comprehensions are parsed as SQUAREs containing a generator.
285		"""
286		function parse_generator(ps::ParseState, ret::EXPR)
287	23	kw = mKEYWORD(next(ps))
288	23	ret = EXPR(Generator, EXPR[ret, kw])
289	21	ranges = @closesquare ps parse_iterators(ps, true)
290
291	23	if typof(ranges) === Block
292	23	append!(ret, ranges)
293		else
294	23	push!(ret, ranges)
295		end
296
297	23	if typof(ret.args[1]) === Generator \|\| typof(ret.args[1]) === Flatten
298	23	ret = EXPR(Flatten, EXPR[ret])
299		end
300
301	23	return ret
302		end
303
304		"""
305		Helper function for parsing import/using statements.
306		"""
307		function parse_dot_mod(ps::ParseState, is_colon=false)
308	23	args = EXPR[]
309
310	21	prevpos = position(ps)
311	23	while kindof(ps.nt) === Tokens.DOT \|\| kindof(ps.nt) === Tokens.DDOT \|\| kindof(ps.nt) === Tokens.DDDOT
312	23	d = mOPERATOR(next(ps))
313	21	trailing_ws = d.fullspan - d.span
314	23	if is_dot(d)
315	23	push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
316	23	elseif is_ddot(d)
317	23	push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
318	23	push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
319	23	elseif is_dddot(d)
320	23	push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
321	23	push!(args, mOPERATOR(1, 1, Tokens.DOT, false))
322	23	push!(args, mOPERATOR(1 + trailing_ws, 1, Tokens.DOT, false))
323		end
324	23	prevpos = loop_check(ps, prevpos)
325		end
326
327		# import/export ..
328		# TODO: Not clear what this is for?
329		# if iscomma(ps.nt) \|\| kindof(ps.ws) == NewLineWS \|\| kindof(ps.nt) === Tokens.ENDMARKER
330		# if length(args) == 2
331		# return EXPR[INSTANCE(ps)]
332		# end
333		# end
334
335	21	prevpos = position(ps)
336	23	while true
337	23	if kindof(ps.nt) === Tokens.AT_SIGN
338	23	at = mPUNCTUATION(next(ps))
339	23	a = INSTANCE(next(ps))
340	23	push!(args, EXPR(MacroName, EXPR[at, a]))
341	23	elseif kindof(ps.nt) === Tokens.LPAREN
342	23	a = EXPR(InvisBrackets, EXPR[mPUNCTUATION(next(ps))])
343	23	push!(a, @closeparen ps parse_expression(ps))
344	21	accept_rparen(ps, a)
345	23	push!(args, a)
346	23	elseif kindof(ps.nt) === Tokens.EX_OR
347	0	a = @closer ps :comma parse_expression(ps)
348	0	push!(args, a)
349	23	elseif !is_colon && isoperator(ps.nt)
350	23	next(ps)
351	23	push!(args, mOPERATOR(ps.nt.startbyte - ps.t.startbyte, 1 + ps.t.endbyte - ps.t.startbyte, kindof(ps.t), false))
352	12	elseif VERSION > v"1.3.0-" && isidentifier(ps.nt) && isemptyws(ps.nws) && (kindof(ps.nnt) === Tokens.STRING \|\| kindof(ps.nnt) === Tokens.TRIPLE_STRING)
353	4	push!(args, EXPR(NONSTDIDENTIFIER, EXPR[INSTANCE(next(ps)), INSTANCE(next(ps))]))
354		else
355	23	push!(args, INSTANCE(next(ps)))
356		end
357
358	23	if kindof(ps.nt) === Tokens.DOT
359	23	push!(args, mPUNCTUATION(next(ps)))
360	23	elseif isoperator(ps.nt) && (ps.nt.dotop \|\| kindof(ps.nt) === Tokens.DOT)
361	23	push!(args, mPUNCTUATION(Tokens.DOT, 1, 1))
362	23	ps.nt = RawToken(kindof(ps.nt), ps.nt.startpos, ps.nt.endpos, ps.nt.startbyte + 1, ps.nt.endbyte, ps.nt.token_error, false, ps.nt.suffix)
363		else
364	23	break
365		end
366	23	prevpos = loop_check(ps, prevpos)
367		end
368	23	args
369		end