1
# Operator hierarchy
2
const AssignmentOp  = 1
3
const ConditionalOp = 2
4
const ArrowOp       = 3
5
const LazyOrOp      = 4
6
const LazyAndOp     = 5
7
const ComparisonOp  = 6
8
const PipeOp        = 7
9
const ColonOp       = 8
10
@static if Base.operator_precedence(:<<) == 12
11
    const PlusOp        = 9
12
    const BitShiftOp    = 10
13
    const TimesOp       = 11
14
    const RationalOp    = 12
15
else
16
    const PlusOp        = 9
17
    const TimesOp       = 10
18
    const RationalOp    = 11
19
    const BitShiftOp    = 12
20
end
21
const PowerOp       = 13
22
const DeclarationOp = 14
23
const WhereOp       = 15
24
const DotOp         = 16
25
const PrimeOp       = 16
26
const DddotOp       = 7
27
const AnonFuncOp    = 14
28

29
@enum(Head,IDENTIFIER,
30
NONSTDIDENTIFIER,
31
PUNCTUATION,
32
OPERATOR,
33
KEYWORD,
34
LITERAL,
35
NoHead,
36
Call,
37
UnaryOpCall,
38
BinaryOpCall,
39
WhereOpCall,
40
ConditionalOpCall,
41
ChainOpCall,
42
ColonOpCall,
43
Abstract,
44
Begin,
45
Block,
46
Braces,
47
BracesCat,
48
Const,
49
Comparison,
50
Curly,
51
Do,
52
Filter,
53
Flatten,
54
For,
55
FunctionDef,
56
Generator,
57
Global,
58
GlobalRefDoc,
59
If,
60
Kw,
61
Let,
62
Local,
63
Macro,
64
MacroCall,
65
MacroName,
66
Mutable,
67
Outer,
68
Parameters,
69
Primitive,
70
Quote,
71
Quotenode,
72
InvisBrackets,
73
StringH,
74
Struct,
75
Try,
76
TupleH,
77
FileH,
78
Return,
79
While,
80
x_Cmd,
81
x_Str,
82
ModuleH,
83
BareModule,
84
TopLevel,
85
Export,
86
Import,
87
Using,
88
Comprehension,
89
DictComprehension,
90
TypedComprehension,
91
Hcat,
92
TypedHcat,
93
Ref,
94
Row,
95
Vcat,
96
TypedVcat,
97
Vect,
98
ErrorToken)
99

100
@enum(ErrorKind,
101
    UnexpectedToken,
102
    CannotJuxtapose,
103
    UnexpectedWhiteSpace,
104
    UnexpectedNewLine,
105
    ExpectedAssignment,
106
    UnexpectedAssignmentOp,
107
    MissingConditional,
108
    MissingCloser,
109
    MissingColon, # We didn't get a colon (`:`) when we expected to while parsing a `?` expression.
110
    InvalidIterator,
111
    StringInterpolationWithTrailingWhitespace,
112
    TooLongChar,
113
    Unknown)
114

115
const NoKind = Tokenize.Tokens.begin_keywords
116

117
mutable struct EXPR
118 23
    typ::Head
119
    args::Union{Nothing,Vector{EXPR}}
120
    fullspan::Int
121
    span::Int
122
    val::Union{Nothing,String}
123
    kind::Tokenize.Tokens.Kind
124
    dot::Bool
125
    parent::Union{Nothing,EXPR}
126
    meta
127
end
128

129
function EXPR(T::Head, args::Vector{EXPR}, fullspan::Int, span::Int)
130 23
    ex = EXPR(T, args, fullspan, span, nothing, NoKind, false, nothing, nothing)
131 23
    for c in args
132 23
        setparent!(c, ex)
133
    end
134 23
    ex
135
end
136

137
function EXPR(T::Head, args::Vector{EXPR})
138 21
    ret = EXPR(T, args, 0, 0)
139 23
    update_span!(ret)
140 23
    ret
141
end
142

143

144

145

146 23
@noinline mIDENTIFIER(ps::ParseState) = EXPR(IDENTIFIER, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, nothing)
147

148 23
mPUNCTUATION(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(PUNCTUATION, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
149 23
@noinline mPUNCTUATION(ps::ParseState) = EXPR(PUNCTUATION, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)
150

151 23
mOPERATOR(fullspan::Int, span::Int, kind::Tokens.Kind, dotop::Bool) = EXPR(OPERATOR, nothing, fullspan, span, nothing, kind, dotop, nothing, nothing)
152 23
@noinline mOPERATOR(ps::ParseState) = EXPR(OPERATOR, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, ps.t.suffix ? val(ps.t, ps) : nothing, kindof(ps.t), ps.t.dotop, nothing, nothing)
153

154 0
mKEYWORD(kind::Tokens.Kind, fullspan::Int, span::Int) = EXPR(KEYWORD, nothing, fullspan, span, nothing, kind, false, nothing, nothing)
155 23
@noinline mKEYWORD(ps::ParseState) = EXPR(KEYWORD, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, nothing, kindof(ps.t), false, nothing, nothing)
156

157 23
mLITERAL(fullspan::Int, span::Int, val::String, kind::Tokens.Kind) = EXPR(LITERAL, nothing, fullspan, span, val, kind, false, nothing, nothing)
158
@noinline function mLITERAL(ps::ParseState)
159

160 23
    if kindof(ps.t) === Tokens.STRING || kindof(ps.t) === Tokens.TRIPLE_STRING ||
161
        kindof(ps.t) === Tokens.CMD || kindof(ps.t) === Tokens.TRIPLE_CMD
162 23
        return parse_string_or_cmd(ps)
163
    else
164 21
        v = val(ps.t, ps)
165 23
        if kindof(ps.t) === Tokens.CHAR && length(v) > 3 && !(v[2] == '\\' && valid_escaped_seq(v[2:prevind(v, length(v))]))
166 0
            return mErrorToken(ps, mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, string(v[1:2], '\''), kindof(ps.t)), TooLongChar)
167
        end
168 23
        return mLITERAL(ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, v, kindof(ps.t))
169
    end
170
end
171

172

173

174 21
span(x::EXPR) = x.span
175

176
function update_span!(x::EXPR)
177 23
    (x.args isa Nothing || isempty(x.args)) && return
178 21
    x.fullspan = 0
179 23
    for i = 1:length(x.args)
180 23
        x.fullspan += x.args[i].fullspan
181
    end
182 23
    x.span = x.fullspan - last(x.args).fullspan + last(x.args).span
183 23
    return
184
end
185

186
function Base.push!(e::EXPR, arg::EXPR)
187 23
    e.span = e.fullspan + arg.span
188 21
    e.fullspan += arg.fullspan
189 21
    setparent!(arg, e)
190 23
    push!(e.args, arg)
191
end
192

193
function Base.pushfirst!(e::EXPR, arg::EXPR)
194 23
    e.fullspan += arg.fullspan
195 21
    setparent!(arg, e)
196 23
    pushfirst!(e.args, arg)
197
end
198

199 0
function Base.pop!(e::EXPR)
200 0
    arg = pop!(e.args)
201 0
    e.fullspan -= arg.fullspan
202 0
    if isempty(e.args)
203 0
        e.span = 0
204
    else
205 0
        e.span = e.fullspan - last(e.args).fullspan + last(e.args).span
206
    end
207 0
    arg
208
end
209

210
function Base.append!(e::EXPR, args::Vector{EXPR})
211 23
    append!(e.args, args)
212 23
    for arg in args
213 23
        setparent!(arg, e)
214
    end
215 23
    update_span!(e)
216
end
217

218
function Base.append!(a::EXPR, b::EXPR)
219 23
    append!(a.args, b.args)
220 23
    for arg in b.args
221 23
        setparent!(arg, a)
222
    end
223 21
    a.fullspan += b.fullspan
224 23
    a.span = a.fullspan + last(b.span)
225
end
226

227

228
function INSTANCE(ps::ParseState)
229 23
    if isidentifier(ps.t)
230 23
        return mIDENTIFIER(ps)
231 23
    elseif isliteral(ps.t)
232 23
        return mLITERAL(ps)
233 23
    elseif iskw(ps.t)
234 23
        return mKEYWORD(ps)
235 23
    elseif isoperator(ps.t)
236 23
        return mOPERATOR(ps)
237 23
    elseif ispunctuation(ps.t)
238 23
        return mPUNCTUATION(ps)
239 0
    elseif kindof(ps.t) === Tokens.ERROR
240 0
        ps.errored = true
241 0
        return EXPR(ErrorToken, nothing, ps.nt.startbyte - ps.t.startbyte, ps.t.endbyte - ps.t.startbyte + 1, val(ps.t, ps), NoKind, false, nothing, Unknown)
242
    else
243 0
        return mErrorToken(ps, Unknown)
244
    end
245
end
246

247
function mUnaryOpCall(op::EXPR, arg::EXPR)
248 23
    fullspan = op.fullspan + arg.fullspan
249 23
    ex = EXPR(UnaryOpCall, EXPR[op, arg], fullspan, fullspan - arg.fullspan + arg.span)
250 21
    setparent!(op, ex)
251 21
    setparent!(op, ex)
252 23
    return ex
253
end
254
function mBinaryOpCall(arg1::EXPR, op::EXPR, arg2::EXPR)
255 23
    fullspan = arg1.fullspan + op.fullspan + arg2.fullspan
256 23
    ex = EXPR(BinaryOpCall, EXPR[arg1, op, arg2], fullspan, fullspan - arg2.fullspan + arg2.span)
257 21
    setparent!(arg1, ex)
258 21
    setparent!(op, ex)
259 21
    setparent!(arg2, ex)
260 23
    return ex
261
end
262
function mWhereOpCall(arg1::EXPR, op::EXPR, args::Vector{EXPR})
263 23
    ex = EXPR(WhereOpCall, EXPR[arg1; op; args], arg1.fullspan + op.fullspan, 0)
264 21
    setparent!(arg1, ex)
265 21
    setparent!(op, ex)
266 23
    for a in args
267 21
        ex.fullspan += a.fullspan
268 23
        setparent!(a, ex)
269
    end
270 21
    ex.span = ex.fullspan - last(args).fullspan + last(args).span
271 23
    return ex
272
end
273

274
function mErrorToken(ps::ParseState, k::ErrorKind)
275 0
    ps.errored = true
276 0
    return EXPR(ErrorToken, EXPR[], 0, 0, nothing, NoKind, false, nothing, k)
277
end
278
function mErrorToken(ps::ParseState, x::EXPR, k)
279 21
    ps.errored = true
280 23
    ret = EXPR(ErrorToken, EXPR[x], x.fullspan, x.span, nothing, NoKind, false, nothing, k)
281 23
    setparent!(ret[1], ret)
282 0
    return ret
283
end
284

285 0
TRUE() = mLITERAL(0, 0, "", Tokens.TRUE)
286 23
FALSE() = mLITERAL(0, 0, "", Tokens.FALSE)
287 23
NOTHING() = mLITERAL(0, 0, "", Tokens.NOTHING)
288 23
GlobalRefDOC() = EXPR(GlobalRefDoc, EXPR[])
289

290 23
typof(x::EXPR) = x.typ
291 23
valof(x::EXPR) = x.val
292 23
kindof(x::EXPR) = x.kind
293 21
kindof(t::Tokens.AbstractToken) = t.kind
294 0
parentof(x::EXPR) = x.parent
295 23
errorof(x::EXPR) = errorof(x.meta)
296 23
errorof(x) = x
297

298
function setparent!(c, p)
299 21
    c.parent = p
300 0
    return c
301
end
302

303

304 0
Base.iterate(x::EXPR) = length(x) == 0 ? nothing : (x.args[1], 1)
305 0
Base.iterate(x::EXPR, s) = s < length(x) ? (x.args[s + 1], s + 1) : nothing
306 23
Base.length(x::EXPR) = x.args isa Nothing ? 0 : length(x.args)
307 0
Base.firstindex(x::EXPR) = 1
308 0
Base.lastindex(x::EXPR) = x.args === nothing ? 0 : lastindex(x.args)
309 23
Base.getindex(x::EXPR, i) = x.args[i]
310 0
Base.setindex!(x::EXPR, val, i) = Base.setindex!(x.args, val, i)
311 0
Base.first(x::EXPR) = x.args === nothing ? nothing : first(x.args)
312 0
Base.last(x::EXPR) = x.args === nothing ? nothing : last(x.args)

Read our documentation on viewing source code .

Loading