#155 storage mechanism

Merged ZacLN
Coverage Reach

Flags

Flags have been temporarily removed from this view while the flagging feature is refactored for better performance and user experience.

You can still use flags when viewing individual files. Flag-level thresholds will also remain on pull and merge requests in your repository provider.

More information can be found in our documentation.


@@ -129,12 +129,13 @@
Loading
129 129
        T
130 130
        N
131 131
        FakeTypeofVararg() = new()
132 -
        FakeTypeofVararg(T) = new(T)
132 +
        FakeTypeofVararg(T) = (new(T))
133 133
        FakeTypeofVararg(T, N) = new(T, N)
134 134
    end
135 135
    function FakeTypeofVararg(va::Core.TypeofVararg)
136 136
        if isdefined(va, :N)
137 -
            FakeTypeofVararg(FakeTypeName(va.T; justname=true), va.N)
137 +
            vaN = va.N isa TypeVar ? FakeTypeVar(va.N) : va.N
138 +
            FakeTypeofVararg(FakeTypeName(va.T; justname=true), vaN) # This should be FakeTypeName(va.N) but seems to crash inference.
138 139
        elseif isdefined(va, :T)
139 140
            FakeTypeofVararg(FakeTypeName(va.T; justname=true))
140 141
        else

@@ -2,13 +2,15 @@
Loading
2 2
3 3
export SymbolServerInstance, getstore
4 4
5 -
using Serialization, Pkg, SHA
5 +
using Pkg, SHA
6 6
using Base: UUID, Process
7 7
import Sockets, UUIDs
8 8
9 9
include("faketypes.jl")
10 10
include("symbols.jl")
11 11
include("utils.jl")
12 +
include("serialize.jl")
13 +
using .CacheStore
12 14
13 15
mutable struct SymbolServerInstance
14 16
    process::Union{Nothing,Base.Process}
@@ -169,7 +171,7 @@
Loading
169 171
    if isfile(cache_path)
170 172
        try
171 173
            package_data = open(cache_path) do io
172 -
                deserialize(io)
174 +
                CacheStore.read(io)
173 175
            end
174 176
            store[Symbol(pe_name)] = package_data.val
175 177
            for dep in deps(pe)

@@ -27,12 +27,14 @@
Loading
27 27
# Make sure we can load stdlibs
28 28
!in("@stdlib", LOAD_PATH) && push!(LOAD_PATH, "@stdlib")
29 29
30 -
using Serialization, Pkg, SHA
30 +
using Pkg, SHA
31 31
using Base: UUID
32 32
33 33
include("faketypes.jl")
34 34
include("symbols.jl")
35 35
include("utils.jl")
36 +
include("serialize.jl")
37 +
using .CacheStore
36 38
37 39
store_path = length(ARGS) > 0 ? ARGS[1] : abspath(joinpath(@__DIR__, "..", "store"))
38 40
@@ -55,6 +57,7 @@
Loading
55 57
function load_package(c::Pkg.Types.Context, uuid, conn)
56 58
    isinmanifest(c, uuid isa String ? Base.UUID(uuid) : uuid) || return
57 59
    pe_name = packagename(c, uuid)
60 +
58 61
    pid = Base.PkgId(uuid isa String ? Base.UUID(uuid) : uuid, pe_name)
59 62
    if pid in keys(Base.loaded_modules)
60 63
        conn !== nothing && println(conn, "PROCESSPKG;$pe_name;$uuid;noversion")
@@ -74,7 +77,7 @@
Loading
74 77
75 78
function write_cache(name, pkg)
76 79
    open(joinpath(server.storedir, name), "w") do io
77 -
        serialize(io, pkg)
80 +
        CacheStore.write(io, pkg)
78 81
    end
79 82
end
80 83
@@ -96,6 +99,7 @@
Loading
96 99
toplevel_pkgs = deps(project(ctx))
97 100
packages_to_load = []
98 101
# Next make sure the cache is up-to-date for all of these
102 +
99 103
for (pk_name, uuid) in toplevel_pkgs
100 104
    file_name = get_filename_from_name(ctx.env.manifest, uuid)
101 105
    # We sometimes have UUIDs in the project file that are not in the
@@ -107,14 +111,18 @@
Loading
107 111
108 112
    if isfile(cache_path)
109 113
        if is_package_deved(ctx.env.manifest, uuid)
110 -
            cached_version = open(cache_path) do io
111 -
                deserialize(io)
112 -
            end
113 -
            if sha_pkg(frommanifest(ctx.env.manifest, uuid)) != cached_version.sha
114 -
                @info "Outdated sha, will recache package $pk_name ($uuid)"
115 -
                push!(packages_to_load, uuid)
116 -
            else
117 -
                @info "Package $pk_name ($uuid) is cached."
114 +
            try
115 +
                cached_version = open(cache_path) do io
116 +
                    CacheStore.read(io)
117 +
                end
118 +
                if sha_pkg(frommanifest(ctx.env.manifest, uuid)) != cached_version.sha
119 +
                    @info "Outdated sha, will recache package $pk_name ($uuid)"
120 +
                    push!(packages_to_load, uuid)
121 +
                else
122 +
                    @info "Package $pk_name ($uuid) is cached."
123 +
                end
124 +
            catch err
125 +
                @info "Couldn't load $pk_name ($uuid) from file, will recache."
118 126
            end
119 127
        else
120 128
            @info "Package $pk_name ($uuid) is cached."
@@ -145,15 +153,15 @@
Loading
145 153
    end
146 154
end
147 155
148 -
symbols(env_symbols, nothing, SymbolServer.getallns(), visited)
156 +
symbols(env_symbols, nothing, getallns(), visited)
149 157
150 158
# Wrap the `ModuleStore`s as `Package`s.
151 159
for (pkg_name, cache) in env_symbols
152 160
    pkg_name = String(pkg_name)
153 161
    !isinmanifest(ctx, pkg_name) && continue
154 162
    uuid = packageuuid(ctx, String(pkg_name))
155 163
    pe = frommanifest(ctx, uuid)
156 -
    server.depot[uuid] = Package(String(pkg_name), cache, version(pe), uuid, sha_pkg(pe))
164 +
    server.depot[uuid] = Package(String(pkg_name), cache, uuid, sha_pkg(pe))
157 165
end
158 166
159 167
# Write to disc

@@ -26,11 +26,10 @@
Loading
26 26
struct Package
27 27
    name::String
28 28
    val::ModuleStore
29 -
    ver::Any
30 29
    uuid::Base.UUID
31 -
    sha
30 +
    sha::Union{Vector{UInt8},Nothing}
32 31
end
33 -
Package(name::String, val::ModuleStore, ver, uuid::String, sha) = Package(name, val, ver, Base.UUID(uuid), sha)
32 +
Package(name::String, val::ModuleStore, uuid::String, sha) = Package(name, val, Base.UUID(uuid), sha)
34 33
35 34
struct MethodStore
36 35
    name::Symbol
@@ -150,7 +149,17 @@
Loading
150 149
    Method[x[3]::Method for x in ms]
151 150
end
152 151
153 -
function cache_methods(@nospecialize(f), name, env)
152 +
function sparam_syms(meth::Method)
153 +
    s = Symbol[]
154 +
    sig = meth.sig
155 +
    while sig isa UnionAll
156 +
        push!(s, Symbol(sig.var.name))
157 +
        sig = sig.body
158 +
    end
159 +
    return s
160 +
end
161 +
162 +
function cache_methods(@nospecialize(f), name, env, get_return_type)
154 163
    if isa(f, Core.Builtin)
155 164
        return MethodStore[]
156 165
    end
@@ -165,8 +174,23 @@
Loading
165 174
    ind_of_method_w_kws = Int[] # stores the index of methods with kws.
166 175
    i = 1
167 176
    for m in methods0
177 +
        # Get inferred method return type
178 +
        if get_return_type
179 +
            sparams = Core.svec(sparam_syms(m[3])...)
180 +
            rt = try 
181 +
                @static if isdefined(Core.Compiler, :NativeInterpreter)
182 +
                Core.Compiler.typeinf_type(Core.Compiler.NativeInterpreter(), m[3], m[3].sig, sparams)
183 +
            else
184 +
                Core.Compiler.typeinf_type(m[3], m[3].sig, sparams, Core.Compiler.Params(world))
185 +
            end
186 +
            catch e
187 +
                Any
188 +
            end
189 +
        else
190 +
            rt = Any
191 +
        end
168 192
        file = maybe_fix_path(String(m[3].file))
169 -
        MS = MethodStore(m[3].name, nameof(m[3].module), file, m[3].line, [], Symbol[], FakeTypeName(Any))
193 +
        MS = MethodStore(m[3].name, nameof(m[3].module), file, m[3].line, [], Symbol[], FakeTypeName(rt))
170 194
        # Get signature
171 195
        sig = Base.unwrap_unionall(m[1])
172 196
        argnames = getargnames(m[3])
@@ -376,7 +400,7 @@
Loading
376 400
    symbols
377 401
end
378 402
379 -
function symbols(env::EnvStore, m::Union{Module,Nothing} = nothing, allnames::Base.IdSet{Symbol} = getallns(), visited = Base.IdSet{Module}())
403 +
function symbols(env::EnvStore, m::Union{Module,Nothing} = nothing, allnames::Base.IdSet{Symbol} = getallns(), visited = Base.IdSet{Module}();  get_return_type = false)
380 404
    if m isa Module
381 405
        cache = _lookup(VarRef(m), env, true)
382 406
        cache === nothing && return
@@ -388,19 +412,19 @@
Loading
388 412
            if Base.unwrap_unionall(x) isa DataType # Unions aren't handled here.
389 413
                if parentmodule((x)) === m
390 414
                    cache[s] = DataTypeStore(x, m, s in getnames(m))
391 -
                    cache_methods(x, s, env)
415 +
                    cache_methods(x, s, env, get_return_type)
392 416
                elseif nameof(x) !== s
393 417
                    # This needs some finessing.
394 418
                    cache[s] = DataTypeStore(x, m, s in getnames(m))
395 -
                    cache_methods(x, s, env)
419 +
                    cache_methods(x, s, env, get_return_type)
396 420
                else
397 421
                    # These are imported variables that are reexported.
398 422
                    cache[s] = VarRef(VarRef(parentmodule(x)), nameof(x))
399 423
                end
400 424
            elseif x isa Function
401 425
                if parentmodule(x) === m || (x isa Core.IntrinsicFunction && m === Core.Intrinsics)
402 426
                    cache[s] = FunctionStore(x, m, s in getnames(m))
403 -
                    cache_methods(x, s, env)
427 +
                    cache_methods(x, s, env, get_return_type)
404 428
                elseif !haskey(cache, s)
405 429
                    # This will be replaced at a later point by a FunctionStore if methods for `x` are defined within `m`.
406 430
                    if x isa Core.IntrinsicFunction
@@ -437,7 +461,7 @@
Loading
437 461
end
438 462
439 463
440 -
function load_core()
464 +
function load_core(; get_return_type = false)
441 465
    c = Pkg.Types.Context()
442 466
    cache = getenvtree([:Core,:Base])
443 467
    symbols(cache)
@@ -526,10 +550,10 @@
Loading
526 550
        true)
527 551
    push!(cache[:Core].exportednames, :ccall)
528 552
    cache[:Core][Symbol("@__doc__")] = FunctionStore(VarRef(VarRef(Core), Symbol("@__doc__")), [], "", VarRef(VarRef(Core), Symbol("@__doc__")), true)
529 -
    cache_methods(getfield(Core, Symbol("@__doc__")), Symbol("@__doc__"), cache)
553 +
    cache_methods(getfield(Core, Symbol("@__doc__")), Symbol("@__doc__"), cache, false)
530 554
    # Accounts for the dd situation where Base.rand only has methods from Random which doesn't appear to be explicitly used.
531 555
    # append!(cache[:Base][:rand].methods, cache_methods(Base.rand, cache))
532 -
    for m in cache_methods(Base.rand, :rand, cache)
556 +
    for m in cache_methods(Base.rand, :rand, cache, get_return_type)
533 557
        push!(cache[:Base][:rand].methods, m[2])
534 558
    end
535 559
@@ -557,7 +581,7 @@
Loading
557 581
    end
558 582
end
559 583
560 -
getallns() = let allns = Base.IdSet{Symbol}(); SymbolServer.oneverything((m, s, x, state) -> push!(allns, s)); allns end
584 +
getallns() = let allns = Base.IdSet{Symbol}(); oneverything((m, s, x, state) -> push!(allns, s)); allns end
561 585
562 586
"""
563 587
    split_module_names(m::Module, allns)

@@ -352,6 +352,8 @@
Loading
352 352
# - its care to protect against circular depenency graphs
353 353
# When you don't need to worry about cycles, you can do much better by defining your own function.
354 354
355 +
recursive_copy(x) = deepcopy(x)
356 +
355 357
recursive_copy(::Nothing) = nothing
356 358
357 359
recursive_copy(s::Symbol) = s
@@ -396,7 +398,6 @@
Loading
396 398
397 399
recursive_copy(p::Package) = Package(p.name,
398 400
                                     recursive_copy(p.val),
399 -
                                     recursive_copy(p.ver),
400 401
                                     p.uuid,
401 402
                                     recursive_copy(p.sha))
402 403
@@ -427,3 +428,34 @@
Loading
427 428
                                                recursive_copy(gs.typ),
428 429
                                                gs.doc,
429 430
                                                gs.exported)
431 +
432 +
433 +
# Tools for modifying source location
434 +
# env = getenvtree([:somepackage])
435 +
# symbols(env, somepackage)
436 +
# m = env[:somepackage]
437 +
# To strip actual src path:
438 +
# modify_dirs(m, f -> modify_dir(f, pkg_src_dir(somepackage), "PLACEHOLDER"))
439 +
# To replace the placeholder:
440 +
# modify_dirs(m, f -> modify_dir(f, "PLACEHOLDER", new_src_dir))
441 +
function modify_dirs(m::ModuleStore, f)
442 +
    for (k, v) in m.vals
443 +
        if v isa FunctionStore
444 +
            m.vals[k] = FunctionStore(v.name, MethodStore[MethodStore(m.name, m.mod, f(m.file), m.line, m.sig, m.kws, m.rt) for m in v.methods], v.doc, v.extends, v.exported)
445 +
        elseif v isa DataTypeStore
446 +
            m.vals[k] = DataTypeStore(v.name, v.super, v.parameters, v.types, v.fieldnames, MethodStore[MethodStore(m.name, m.mod, f(m.file), m.line, m.sig, m.kws, m.rt) for m in v.methods], v.doc, v.exported)
447 +
        elseif v isa ModuleStore
448 +
            modify_dirs(v, f)
449 +
        end
450 +
    end
451 +
end
452 +
453 +
pkg_src_dir(m::Module) = dirname(pathof(m))
454 +
    
455 +
456 +
457 +
# replace s1 with s2 at the start of a string
458 +
function modify_dir(f, s1, s2)
459 +
    @assert startswith(f, s1)
460 +
    string(s2, f[length(s1)+1:end])
461 +
end

Click to load this diff.
Loading diff...

Learn more Showing 2 files with coverage changes found.

Changes in src/utils.jl
-2
+2
Loading file...
New file src/serialize.jl
New
Loading file...
Files Coverage
src 0.53% 71.46%
Project Totals (6 files) 71.46%
Loading