spec.jl at fc030f1 ⋅ julia-vscode/CSTParser.jl

# Operator hierarchy
const AssignmentOp  = 1
const ConditionalOp = 2
const ArrowOp       = 3
const LazyOrOp      = 4
const LazyAndOp     = 5
const ComparisonOp  = 6
const PipeOp        = 7
const ColonOp       = 8
@static if Base.operator_precedence(:<<) == 12
    const PlusOp        = 9
    const BitShiftOp    = 10
    const TimesOp       = 11
    const RationalOp    = 12
else
    const PlusOp        = 9
    const TimesOp       = 10
    const RationalOp    = 11
    const BitShiftOp    = 12
end
const PowerOp       = 13
const DeclarationOp = 14
const WhereOp       = 15
const DotOp         = 16
const PrimeOp       = 16
const DddotOp       = 7
const AnonFuncOp    = 14

@enum(Head,IDENTIFIER,
NONSTDIDENTIFIER,
PUNCTUATION,
OPERATOR,
KEYWORD,
LITERAL,
NoHead,
Call,
UnaryOpCall,
BinaryOpCall,
WhereOpCall,
ConditionalOpCall,
ChainOpCall,
ColonOpCall,
Abstract,
Begin,
Block,
Braces,
BracesCat,
Const,
Comparison,
Curly,
Do,
Filter,
Flatten,
For,
FunctionDef,
Generator,
Global,
GlobalRefDoc,
If,
Kw,
Let,
Local,
Macro,
MacroCall,
MacroName,
Mutable,
Outer,
Parameters,
Primitive,
Quote,
Quotenode,
InvisBrackets,
StringH,
Struct,
Try,
TupleH,
FileH,
Return,
While,
x_Cmd,
x_Str,
ModuleH,
BareModule,
TopLevel,
Export,
Import,
Using,
Comprehension,
DictComprehension,
TypedComprehension,
Hcat,
TypedHcat,
Ref,
Row,
Vcat,
TypedVcat,
Vect,
ErrorToken)

@enum(ErrorKind,
    UnexpectedToken,
    CannotJuxtapose,
    UnexpectedWhiteSpace,
    UnexpectedNewLine,
    ExpectedAssignment,
    UnexpectedAssignmentOp,
    MissingConditional,
    MissingCloser,
    MissingColon, # We didn't get a colon (`:`) when we expected to while parsing a `?` expression.
    InvalidIterator,
    StringInterpolationWithTrailingWhitespace,
    TooLongChar,
    Unknown)

const NoKind = Tokenize.Tokens.begin_keywords

mutable struct EXPR
    typ::Head
    args::Union{Nothing,Vector{EXPR}}
    fullspan::Int
    span::Int
    val::Union{Nothing,String}
    kind::Tokenize.Tokens.Kind
    dot::Bool
    parent::Union{Nothing,EXPR}
    meta
end

function EXPR(T::Head, args::Vector{EXPR}, fullspan::Int, span::Int)
    ex = EXPR(T, args, fullspan, span, nothing, NoKind, false, nothing, nothing)
    for c in args
        setparent!(c, ex)
    end
    ex
end

function EXPR(T::Head, args::Vector{EXPR})
    ret = EXPR(T, args, 0, 0)
    update_span!(ret)
    ret
end




@noinline mIDENTIFIER(ps::ParseState) = EXPR(IDENTIFIER, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, nothing)

mPUNCTUATION(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(PUNCTUATION, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
@noinline mPUNCTUATION(ps::ParseState) = EXPR(PUNCTUATION, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)

mOPERATOR(fullspan::Int, span::Int, kind::Tokens.Kind, dotop::Bool) = EXPR(OPERATOR, nothing, fullspan, span, nothing, kind, dotop, nothing, nothing)
@noinline mOPERATOR(ps::ParseState) = EXPR(OPERATOR, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, ps.t.suffix ? val(ps.t, ps) : nothing, kindof(ps.t), ps.t.dotop, nothing, nothing)

mKEYWORD(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(KEYWORD, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
@noinline mKEYWORD(ps::ParseState) = EXPR(KEYWORD, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)

mLITERAL(fullspan::Int, span::Int, val::String, kind::Tokens.Kind) = EXPR(LITERAL, nothing, fullspan, span, val, kind, false, nothing, nothing)
@noinline function mLITERAL(ps::ParseState)

    if kindof(ps.t) === Tokens.STRING || kindof(ps.t) === Tokens.TRIPLE_STRING ||
        kindof(ps.t) === Tokens.CMD || kindof(ps.t) === Tokens.TRIPLE_CMD
        return parse_string_or_cmd(ps)
    else
        v = val(ps.t, ps)
        if kindof(ps.t) === Tokens.CHAR && length(v) > 3 && !(v[2] == '\\' && valid_escaped_seq(v[2:prevind(v, length(v))]))
            return mErrorToken(ps, mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\''), kindof(ps.t)), TooLongChar)
        end
        return mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v, kindof(ps.t))
    end
end



span(x::EXPR) = x.span

function update_span!(x::EXPR)
    (x.args isa Nothing || isempty(x.args)) && return
    x.fullspan = 0
    for i = 1:length(x.args)
        x.fullspan += x.args[i].fullspan
    end
    x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
    return
end

function Base.push!(e::EXPR, arg::EXPR)
    e.span = e.fullspan + arg.span
    e.fullspan += arg.fullspan
    setparent!(arg, e)
    push!(e.args, arg)
end

function Base.pushfirst!(e::EXPR, arg::EXPR)
    e.fullspan += arg.fullspan
    setparent!(arg, e)
    pushfirst!(e.args, arg)
end

function Base.pop!(e::EXPR)
    arg = pop!(e.args)
    e.fullspan -= arg.fullspan
    if isempty(e.args)
        e.span = 0
    else
        e.span = e.fullspan - last(e.args).fullspan + last(e.args).span
    end
    arg
end

function Base.append!(e::EXPR, args::Vector{EXPR})
    append!(e.args, args)
    for arg in args
        setparent!(arg, e)
    end
    update_span!(e)
end

function Base.append!(a::EXPR, b::EXPR)
    append!(a.args, b.args)
    for arg in b.args
        setparent!(arg, a)
    end
    a.fullspan += b.fullspan
    a.span = a.fullspan + last(b.span)
end


function INSTANCE(ps::ParseState)
    if isidentifier(ps.t)
        return mIDENTIFIER(ps)
    elseif isliteral(ps.t)
        return mLITERAL(ps)
    elseif iskw(ps.t)
        return mKEYWORD(ps)
    elseif isoperator(ps.t)
        return mOPERATOR(ps)
    elseif ispunctuation(ps.t)
        return mPUNCTUATION(ps)
    elseif kindof(ps.t) === Tokens.ERROR
        ps.errored = true
        return EXPR(ErrorToken, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, Unknown)
    else
        return mErrorToken(ps, Unknown)
    end
end

function mUnaryOpCall(op::EXPR, arg::EXPR)
    fullspan = op.fullspan + arg.fullspan
    ex = EXPR(UnaryOpCall, EXPR[op, arg], fullspan, fullspan - arg.fullspan + arg.span)
    setparent!(op, ex)
    setparent!(op, ex)
    return ex
end
function mBinaryOpCall(arg1::EXPR, op::EXPR, arg2::EXPR)
    fullspan = arg1.fullspan + op.fullspan + arg2.fullspan
    ex = EXPR(BinaryOpCall, EXPR[arg1, op, arg2], fullspan, fullspan - arg2.fullspan + arg2.span)
    setparent!(arg1, ex)
    setparent!(op, ex)
    setparent!(arg2, ex)
    return ex
end
function mWhereOpCall(arg1::EXPR, op::EXPR, args::Vector{EXPR})
    ex = EXPR(WhereOpCall, EXPR[arg1; op; args], arg1.fullspan + op.fullspan, 0)
    setparent!(arg1, ex)
    setparent!(op, ex)
    for a in args
        ex.fullspan += a.fullspan
        setparent!(a, ex)
    end
    ex.span = ex.fullspan - last(args).fullspan + last(args).span
    return ex
end

function mErrorToken(ps::ParseState, k::ErrorKind)
    ps.errored = true
    return EXPR(ErrorToken, EXPR[], 0, 0, nothing, NoKind, false, nothing, k)
end
function mErrorToken(ps::ParseState, x::EXPR, k)
    ps.errored = true
    ret = EXPR(ErrorToken, EXPR[x], x.fullspan, x.span, nothing, NoKind, false, nothing, k)
    setparent!(ret[1], ret)
    return ret
end

TRUE() = mLITERAL(0, 0, "", Tokens.TRUE)
FALSE() = mLITERAL(0, 0, "", Tokens.FALSE)
NOTHING() = mLITERAL(0, 0, "", Tokens.NOTHING)
GlobalRefDOC() = EXPR(GlobalRefDoc, EXPR[])

typof(x::EXPR) = x.typ
valof(x::EXPR) = x.val
kindof(x::EXPR) = x.kind
kindof(t::Tokens.AbstractToken) = t.kind
parentof(x::EXPR) = x.parent
errorof(x::EXPR) = errorof(x.meta)
errorof(x) = x

function setparent!(c, p)
    c.parent = p
    return c
end


Base.iterate(x::EXPR) = length(x) == 0 ? nothing : (x.args[1], 1)
Base.iterate(x::EXPR, s) = s < length(x) ? (x.args[s + 1], s + 1) : nothing
Base.length(x::EXPR) = x.args isa Nothing ? 0 : length(x.args)
Base.firstindex(x::EXPR) = 1
Base.lastindex(x::EXPR) = x.args === nothing ? 0 : lastindex(x.args)
Base.getindex(x::EXPR, i) = x.args[i]
Base.setindex!(x::EXPR, val, i) = Base.setindex!(x.args, val, i)
Base.first(x::EXPR) = x.args === nothing ? nothing : first(x.args)
Base.last(x::EXPR) = x.args === nothing ? nothing : last(x.args)

Read our documentation on viewing source code .

Navigation	Overlay
`t` Navigate files	`h` Toggle hits
`y` Change url to tip of branch	`m` Toggle misses
`b / v` Jump to prev/next hit line	`p` Toggle partial
`z / x` Jump to prev/next missed or partial line	`1..9` Toggle flags
`shift + o` Open current page in GitHub	`a` Toggle all on
`/ or ?` Show keyboard shortcuts dialog	`c` Toggle context lines or commits

1		# Operator hierarchy
2		const AssignmentOp = 1
3		const ConditionalOp = 2
4		const ArrowOp = 3
5		const LazyOrOp = 4
6		const LazyAndOp = 5
7		const ComparisonOp = 6
8		const PipeOp = 7
9		const ColonOp = 8
10		@static if Base.operator_precedence(:<<) == 12
11		const PlusOp = 9
12		const BitShiftOp = 10
13		const TimesOp = 11
14		const RationalOp = 12
15		else
16		const PlusOp = 9
17		const TimesOp = 10
18		const RationalOp = 11
19		const BitShiftOp = 12
20		end
21		const PowerOp = 13
22		const DeclarationOp = 14
23		const WhereOp = 15
24		const DotOp = 16
25		const PrimeOp = 16
26		const DddotOp = 7
27		const AnonFuncOp = 14
28
29		@enum(Head,IDENTIFIER,
30		NONSTDIDENTIFIER,
31		PUNCTUATION,
32		OPERATOR,
33		KEYWORD,
34		LITERAL,
35		NoHead,
36		Call,
37		UnaryOpCall,
38		BinaryOpCall,
39		WhereOpCall,
40		ConditionalOpCall,
41		ChainOpCall,
42		ColonOpCall,
43		Abstract,
44		Begin,
45		Block,
46		Braces,
47		BracesCat,
48		Const,
49		Comparison,
50		Curly,
51		Do,
52		Filter,
53		Flatten,
54		For,
55		FunctionDef,
56		Generator,
57		Global,
58		GlobalRefDoc,
59		If,
60		Kw,
61		Let,
62		Local,
63		Macro,
64		MacroCall,
65		MacroName,
66		Mutable,
67		Outer,
68		Parameters,
69		Primitive,
70		Quote,
71		Quotenode,
72		InvisBrackets,
73		StringH,
74		Struct,
75		Try,
76		TupleH,
77		FileH,
78		Return,
79		While,
80		x_Cmd,
81		x_Str,
82		ModuleH,
83		BareModule,
84		TopLevel,
85		Export,
86		Import,
87		Using,
88		Comprehension,
89		DictComprehension,
90		TypedComprehension,
91		Hcat,
92		TypedHcat,
93		Ref,
94		Row,
95		Vcat,
96		TypedVcat,
97		Vect,
98		ErrorToken)
99
100		@enum(ErrorKind,
101		UnexpectedToken,
102		CannotJuxtapose,
103		UnexpectedWhiteSpace,
104		UnexpectedNewLine,
105		ExpectedAssignment,
106		UnexpectedAssignmentOp,
107		MissingConditional,
108		MissingCloser,
109		MissingColon, # We didn't get a colon (`:`) when we expected to while parsing a `?` expression.
110		InvalidIterator,
111		StringInterpolationWithTrailingWhitespace,
112		TooLongChar,
113		Unknown)
114
115		const NoKind = Tokenize.Tokens.begin_keywords
116
117		mutable struct EXPR
118	23	typ::Head
119		args::Union{Nothing,Vector{EXPR}}
120		fullspan::Int
121		span::Int
122		val::Union{Nothing,String}
123		kind::Tokenize.Tokens.Kind
124		dot::Bool
125		parent::Union{Nothing,EXPR}
126		meta
127		end
128
129		function EXPR(T::Head, args::Vector{EXPR}, fullspan::Int, span::Int)
130	23	ex = EXPR(T, args, fullspan, span, nothing, NoKind, false, nothing, nothing)
131	23	for c in args
132	23	setparent!(c, ex)
133		end
134	23	ex
135		end
136
137		function EXPR(T::Head, args::Vector{EXPR})
138	21	ret = EXPR(T, args, 0, 0)
139	23	update_span!(ret)
140	23	ret
141		end
142
143
144
145
146	23	@noinline mIDENTIFIER(ps::ParseState) = EXPR(IDENTIFIER, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, nothing)
147
148	23	mPUNCTUATION(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(PUNCTUATION, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
149	23	@noinline mPUNCTUATION(ps::ParseState) = EXPR(PUNCTUATION, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)
150
151	23	mOPERATOR(fullspan::Int, span::Int, kind::Tokens.Kind, dotop::Bool) = EXPR(OPERATOR, nothing, fullspan, span, nothing, kind, dotop, nothing, nothing)
152	23	@noinline mOPERATOR(ps::ParseState) = EXPR(OPERATOR, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, ps.t.suffix ? val(ps.t, ps) : nothing, kindof(ps.t), ps.t.dotop, nothing, nothing)
153
154	0	mKEYWORD(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(KEYWORD, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
155	23	@noinline mKEYWORD(ps::ParseState) = EXPR(KEYWORD, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)
156
157	23	mLITERAL(fullspan::Int, span::Int, val::String, kind::Tokens.Kind) = EXPR(LITERAL, nothing, fullspan, span, val, kind, false, nothing, nothing)
158		@noinline function mLITERAL(ps::ParseState)
159
160	23	if kindof(ps.t) === Tokens.STRING \|\| kindof(ps.t) === Tokens.TRIPLE_STRING \|\|
161		kindof(ps.t) === Tokens.CMD \|\| kindof(ps.t) === Tokens.TRIPLE_CMD
162	23	return parse_string_or_cmd(ps)
163		else
164	21	v = val(ps.t, ps)
165	23	if kindof(ps.t) === Tokens.CHAR && length(v) > 3 && !(v[2] == '\\' && valid_escaped_seq(v[2:prevind(v, length(v))]))
166	0	return mErrorToken(ps, mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\''), kindof(ps.t)), TooLongChar)
167		end
168	23	return mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v, kindof(ps.t))
169		end
170		end
171
172
173
174	21	span(x::EXPR) = x.span
175
176		function update_span!(x::EXPR)
177	23	(x.args isa Nothing \|\| isempty(x.args)) && return
178	21	x.fullspan = 0
179	23	for i = 1:length(x.args)
180	23	x.fullspan += x.args[i].fullspan
181		end
182	23	x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
183	23	return
184		end
185
186		function Base.push!(e::EXPR, arg::EXPR)
187	23	e.span = e.fullspan + arg.span
188	21	e.fullspan += arg.fullspan
189	21	setparent!(arg, e)
190	23	push!(e.args, arg)
191		end
192
193		function Base.pushfirst!(e::EXPR, arg::EXPR)
194	23	e.fullspan += arg.fullspan
195	21	setparent!(arg, e)
196	23	pushfirst!(e.args, arg)
197		end
198
199	0	function Base.pop!(e::EXPR)
200	0	arg = pop!(e.args)
201	0	e.fullspan -= arg.fullspan
202	0	if isempty(e.args)
203	0	e.span = 0
204		else
205	0	e.span = e.fullspan - last(e.args).fullspan + last(e.args).span
206		end
207	0	arg
208		end
209
210		function Base.append!(e::EXPR, args::Vector{EXPR})
211	23	append!(e.args, args)
212	23	for arg in args
213	23	setparent!(arg, e)
214		end
215	23	update_span!(e)
216		end
217
218		function Base.append!(a::EXPR, b::EXPR)
219	23	append!(a.args, b.args)
220	23	for arg in b.args
221	23	setparent!(arg, a)
222		end
223	21	a.fullspan += b.fullspan
224	23	a.span = a.fullspan + last(b.span)
225		end
226
227
228		function INSTANCE(ps::ParseState)
229	23	if isidentifier(ps.t)
230	23	return mIDENTIFIER(ps)
231	23	elseif isliteral(ps.t)
232	23	return mLITERAL(ps)
233	23	elseif iskw(ps.t)
234	23	return mKEYWORD(ps)
235	23	elseif isoperator(ps.t)
236	23	return mOPERATOR(ps)
237	23	elseif ispunctuation(ps.t)
238	23	return mPUNCTUATION(ps)
239	0	elseif kindof(ps.t) === Tokens.ERROR
240	0	ps.errored = true
241	0	return EXPR(ErrorToken, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, Unknown)
242		else
243	0	return mErrorToken(ps, Unknown)
244		end
245		end
246
247		function mUnaryOpCall(op::EXPR, arg::EXPR)
248	23	fullspan = op.fullspan + arg.fullspan
249	23	ex = EXPR(UnaryOpCall, EXPR[op, arg], fullspan, fullspan - arg.fullspan + arg.span)
250	21	setparent!(op, ex)
251	21	setparent!(op, ex)
252	23	return ex
253		end
254		function mBinaryOpCall(arg1::EXPR, op::EXPR, arg2::EXPR)
255	23	fullspan = arg1.fullspan + op.fullspan + arg2.fullspan
256	23	ex = EXPR(BinaryOpCall, EXPR[arg1, op, arg2], fullspan, fullspan - arg2.fullspan + arg2.span)
257	21	setparent!(arg1, ex)
258	21	setparent!(op, ex)
259	21	setparent!(arg2, ex)
260	23	return ex
261		end
262		function mWhereOpCall(arg1::EXPR, op::EXPR, args::Vector{EXPR})
263	23	ex = EXPR(WhereOpCall, EXPR[arg1; op; args], arg1.fullspan + op.fullspan, 0)
264	21	setparent!(arg1, ex)
265	21	setparent!(op, ex)
266	23	for a in args
267	21	ex.fullspan += a.fullspan
268	23	setparent!(a, ex)
269		end
270	21	ex.span = ex.fullspan - last(args).fullspan + last(args).span
271	23	return ex
272		end
273
274		function mErrorToken(ps::ParseState, k::ErrorKind)
275	0	ps.errored = true
276	0	return EXPR(ErrorToken, EXPR[], 0, 0, nothing, NoKind, false, nothing, k)
277		end
278		function mErrorToken(ps::ParseState, x::EXPR, k)
279	21	ps.errored = true
280	23	ret = EXPR(ErrorToken, EXPR[x], x.fullspan, x.span, nothing, NoKind, false, nothing, k)
281	23	setparent!(ret[1], ret)
282	0	return ret
283		end
284
285	0	TRUE() = mLITERAL(0, 0, "", Tokens.TRUE)
286	23	FALSE() = mLITERAL(0, 0, "", Tokens.FALSE)
287	23	NOTHING() = mLITERAL(0, 0, "", Tokens.NOTHING)
288	23	GlobalRefDOC() = EXPR(GlobalRefDoc, EXPR[])
289
290	23	typof(x::EXPR) = x.typ
291	23	valof(x::EXPR) = x.val
292	23	kindof(x::EXPR) = x.kind
293	21	kindof(t::Tokens.AbstractToken) = t.kind
294	0	parentof(x::EXPR) = x.parent
295	23	errorof(x::EXPR) = errorof(x.meta)
296	23	errorof(x) = x
297
298		function setparent!(c, p)
299	21	c.parent = p
300	0	return c
301		end
302
303
304	0	Base.iterate(x::EXPR) = length(x) == 0 ? nothing : (x.args[1], 1)
305	0	Base.iterate(x::EXPR, s) = s < length(x) ? (x.args[s + 1], s + 1) : nothing
306	23	Base.length(x::EXPR) = x.args isa Nothing ? 0 : length(x.args)
307	0	Base.firstindex(x::EXPR) = 1
308	0	Base.lastindex(x::EXPR) = x.args === nothing ? 0 : lastindex(x.args)
309	23	Base.getindex(x::EXPR, i) = x.args[i]
310	0	Base.setindex!(x::EXPR, val, i) = Base.setindex!(x.args, val, i)
311	0	Base.first(x::EXPR) = x.args === nothing ? nothing : first(x.args)
312	0	Base.last(x::EXPR) = x.args === nothing ? nothing : last(x.args)