Compare 88240a4 ... +9 ... 3e53e96

Coverage Reach

Flags

Flags have been temporarily removed from this view while the flagging feature is refactored for better performance and user experience.

You can still use flags when viewing individual files. Flag-level thresholds will also remain on pull and merge requests in your repository provider.

More information can be found in our documentation.


@@ -27,12 +27,14 @@
Loading
27 27
# Make sure we can load stdlibs
28 28
!in("@stdlib", LOAD_PATH) && push!(LOAD_PATH, "@stdlib")
29 29
30 -
using Serialization, Pkg, SHA
30 +
using Pkg, SHA
31 31
using Base: UUID
32 32
33 33
include("faketypes.jl")
34 34
include("symbols.jl")
35 35
include("utils.jl")
36 +
include("serialize.jl")
37 +
using .CacheStore
36 38
37 39
store_path = length(ARGS) > 0 ? ARGS[1] : abspath(joinpath(@__DIR__, "..", "store"))
38 40
@@ -55,6 +57,7 @@
Loading
55 57
function load_package(c::Pkg.Types.Context, uuid, conn)
56 58
    isinmanifest(c, uuid isa String ? Base.UUID(uuid) : uuid) || return
57 59
    pe_name = packagename(c, uuid)
60 +
58 61
    pid = Base.PkgId(uuid isa String ? Base.UUID(uuid) : uuid, pe_name)
59 62
    if pid in keys(Base.loaded_modules)
60 63
        conn !== nothing && println(conn, "PROCESSPKG;$pe_name;$uuid;noversion")
@@ -74,7 +77,7 @@
Loading
74 77
75 78
function write_cache(name, pkg)
76 79
    open(joinpath(server.storedir, name), "w") do io
77 -
        serialize(io, pkg)
80 +
        CacheStore.write(io, pkg)
78 81
    end
79 82
end
80 83
@@ -96,6 +99,7 @@
Loading
96 99
toplevel_pkgs = deps(project(ctx))
97 100
packages_to_load = []
98 101
# Next make sure the cache is up-to-date for all of these
102 +
99 103
for (pk_name, uuid) in toplevel_pkgs
100 104
    file_name = get_filename_from_name(ctx.env.manifest, uuid)
101 105
    # We sometimes have UUIDs in the project file that are not in the
@@ -108,7 +112,7 @@
Loading
108 112
    if isfile(cache_path)
109 113
        if is_package_deved(ctx.env.manifest, uuid)
110 114
            cached_version = open(cache_path) do io
111 -
                deserialize(io)
115 +
                CacheStore.read(io)
112 116
            end
113 117
            if sha_pkg(frommanifest(ctx.env.manifest, uuid)) != cached_version.sha
114 118
                @info "Outdated sha, will recache package $pk_name ($uuid)"
@@ -145,15 +149,15 @@
Loading
145 149
    end
146 150
end
147 151
148 -
symbols(env_symbols, nothing, SymbolServer.getallns(), visited)
152 +
symbols(env_symbols, nothing, getallns(), visited)
149 153
150 154
# Wrap the `ModuleStore`s as `Package`s.
151 155
for (pkg_name, cache) in env_symbols
152 156
    pkg_name = String(pkg_name)
153 157
    !isinmanifest(ctx, pkg_name) && continue
154 158
    uuid = packageuuid(ctx, String(pkg_name))
155 159
    pe = frommanifest(ctx, uuid)
156 -
    server.depot[uuid] = Package(String(pkg_name), cache, version(pe), uuid, sha_pkg(pe))
160 +
    server.depot[uuid] = Package(String(pkg_name), cache, uuid, sha_pkg(pe))
157 161
end
158 162
159 163
# Write to disc

@@ -26,11 +26,10 @@
Loading
26 26
struct Package
27 27
    name::String
28 28
    val::ModuleStore
29 -
    ver::Any
30 29
    uuid::Base.UUID
31 -
    sha
30 +
    sha::Union{Vector{UInt8},Nothing}
32 31
end
33 -
Package(name::String, val::ModuleStore, ver, uuid::String, sha) = Package(name, val, ver, Base.UUID(uuid), sha)
32 +
Package(name::String, val::ModuleStore, uuid::String, sha) = Package(name, val, Base.UUID(uuid), sha)
34 33
35 34
struct MethodStore
36 35
    name::Symbol
@@ -548,7 +547,7 @@
Loading
548 547
    end
549 548
end
550 549
551 -
getallns() = let allns = Base.IdSet{Symbol}(); SymbolServer.oneverything((m, s, x, state) -> push!(allns, s)); allns end
550 +
getallns() = let allns = Base.IdSet{Symbol}(); oneverything((m, s, x, state) -> push!(allns, s)); allns end
552 551
553 552
"""
554 553
    split_module_names(m::Module, allns)

@@ -0,0 +1,254 @@
Loading
1 +
module CacheStore
2 +
using ..SymbolServer: VarRef, FakeTypeName, FakeTypeofBottom, FakeTypeVar, FakeUnion, FakeUnionAll
3 +
using ..SymbolServer: ModuleStore, Package, FunctionStore, MethodStore, DataTypeStore, GenericStore
4 +
5 +
const NothingHeader = 0x01
6 +
const SymbolHeader = 0x02
7 +
const CharHeader = 0x03
8 +
const IntegerHeader = 0x04
9 +
const StringHeader = 0x05
10 +
const VarRefHeader = 0x06
11 +
const FakeTypeNameHeader = 0x07
12 +
const FakeTypeofBottomHeader = 0x08
13 +
const FakeTypeVarHeader = 0x09
14 +
const FakeUnionHeader = 0x0a
15 +
const FakeUnionAllHeader = 0xb
16 +
const ModuleStoreHeader = 0x0c
17 +
const MethodStoreHeader = 0x0d
18 +
const FunctionStoreHeader = 0x0e
19 +
const DataTypeStoreHeader = 0x0f
20 +
const GenericStoreHeader = 0x10
21 +
const PackageHeader = 0x11
22 +
const TrueHeader = 0x12
23 +
const FalseHeader = 0x13
24 +
const TupleHeader = 0x14
25 +
26 +
27 +
function write(io, x::VarRef)
28 +
    Base.write(io, VarRefHeader)
29 +
    write(io, x.parent)
30 +
    write(io, x.name)
31 +
end
32 +
function write(io, x::Nothing)
33 +
    Base.write(io, NothingHeader)
34 +
end
35 +
function write(io, x::Char)
36 +
    Base.write(io, CharHeader)
37 +
    Base.write(io, UInt32(x))
38 +
end
39 +
function write(io, x::Bool)
40 +
    x ? Base.write(io, TrueHeader) : Base.write(io, FalseHeader)
41 +
end
42 +
function write(io, x::Int)
43 +
    Base.write(io, IntegerHeader)
44 +
    Base.write(io, x)
45 +
end
46 +
function write(io, x::Symbol)
47 +
    Base.write(io, SymbolHeader)
48 +
    Base.write(io, sizeof(x))
49 +
    Base.write(io, String(x))
50 +
end
51 +
function write(io, x::NTuple{N,Any}) where N
52 +
    Base.write(io, TupleHeader)
53 +
    Base.write(io, N)
54 +
    for i = 1:N
55 +
        write(io, x[i])
56 +
    end
57 +
end
58 +
function write(io, x::String)
59 +
    Base.write(io, StringHeader)
60 +
    Base.write(io, sizeof(x))
61 +
    Base.write(io, x)
62 +
end
63 +
function write(io, x::FakeTypeName)
64 +
    Base.write(io, FakeTypeNameHeader)
65 +
    write(io, x.name)
66 +
    write_vector(io, x.parameters)
67 +
end
68 +
write(io, x::FakeTypeofBottom) = Base.write(io, FakeTypeofBottomHeader)
69 +
function write(io, x::FakeTypeVar)
70 +
    Base.write(io, FakeTypeVarHeader)
71 +
    write(io, x.name)
72 +
    write(io, x.lb)
73 +
    write(io, x.ub)
74 +
end
75 +
function write(io, x::FakeUnion)
76 +
    Base.write(io, FakeUnionHeader)
77 +
    write(io, x.a)
78 +
    write(io, x.b)
79 +
end
80 +
function write(io, x::FakeUnionAll)
81 +
    Base.write(io, FakeUnionAllHeader)
82 +
    write(io, x.var)
83 +
    write(io, x.body)
84 +
end
85 +
86 +
function write(io, x::MethodStore)
87 +
    Base.write(io, MethodStoreHeader)
88 +
    write(io, x.name)
89 +
    write(io, x.mod)
90 +
    write(io, x.file)
91 +
    Base.write(io, x.line)
92 +
    Base.write(io, length(x.sig))
93 +
    for p in x.sig
94 +
        write(io, p[1])
95 +
        write(io, p[2])
96 +
    end
97 +
    write_vector(io, x.kws)
98 +
    write(io, x.rt)
99 +
end
100 +
101 +
function write(io, x::FunctionStore)
102 +
    Base.write(io, FunctionStoreHeader)
103 +
    write(io, x.name)
104 +
    write_vector(io, x.methods)
105 +
    write(io, x.doc)
106 +
    write(io, x.extends)
107 +
    write(io, x.exported)
108 +
end
109 +
110 +
function write(io, x::DataTypeStore)
111 +
    Base.write(io, DataTypeStoreHeader)
112 +
    write(io, x.name)
113 +
    write(io, x.super)
114 +
    write_vector(io, x.parameters)
115 +
    write_vector(io, x.types)
116 +
    write_vector(io, x.fieldnames)
117 +
    write_vector(io, x.methods)
118 +
    write(io, x.doc)
119 +
    write(io, x.exported)
120 +
end
121 +
122 +
function write(io, x::GenericStore)
123 +
    Base.write(io, GenericStoreHeader)
124 +
    write(io, x.name)
125 +
    write(io, x.typ)
126 +
    write(io, x.doc)
127 +
    write(io, x.exported)
128 +
end
129 +
130 +
function write(io, x::ModuleStore)
131 +
    Base.write(io, ModuleStoreHeader)
132 +
    write(io, x.name)
133 +
    Base.write(io, length(x.vals))
134 +
    for p in x.vals
135 +
        write(io, p[1])
136 +
        write(io, p[2])
137 +
    end
138 +
    write(io, x.doc)
139 +
    write(io, x.exported)
140 +
    write_vector(io, x.exportednames)
141 +
    write_vector(io, x.used_modules)
142 +
end
143 +
144 +
function write(io, x::Package)
145 +
    Base.write(io, PackageHeader)
146 +
    write(io, x.name)
147 +
    write(io, x.val)
148 +
    Base.write(io, UInt128(x.uuid))
149 +
    Base.write(io, x.sha === nothing ? zeros(UInt8, 32) : x.sha)
150 +
end
151 +
152 +
function write_vector(io, x)
153 +
    Base.write(io, length(x))
154 +
    for p in x
155 +
        write(io, p)
156 +
    end
157 +
end
158 +
159 +
function read(io, t = Base.read(io, UInt8))
160 +
    if t === VarRefHeader
161 +
        VarRef(read(io), read(io))
162 +
    elseif t === NothingHeader
163 +
        nothing
164 +
    elseif t === SymbolHeader
165 +
        n = Base.read(io, Int)
166 +
        out = UInt8[]
167 +
        readbytes!(io, out, n)
168 +
        Symbol(String(out))
169 +
    elseif t === StringHeader
170 +
        n = Base.read(io, Int)
171 +
        out = UInt8[]
172 +
        readbytes!(io, out, n)
173 +
        String(out)
174 +
    elseif t === CharHeader
175 +
        Char(Base.read(io, UInt32))
176 +
    elseif t === IntegerHeader
177 +
        Base.read(io, Int)
178 +
    elseif t === FakeTypeNameHeader
179 +
        FakeTypeName(read(io), read_vector(io, Any))
180 +
    elseif t === FakeTypeofBottomHeader
181 +
        FakeTypeofBottom()
182 +
    elseif t === FakeTypeVarHeader
183 +
        FakeTypeVar(read(io), read(io), read(io))
184 +
    elseif t === FakeUnionHeader
185 +
        FakeUnion(read(io), read(io))
186 +
    elseif t === FakeUnionAllHeader
187 +
        FakeUnionAll(read(io), read(io))
188 +
    elseif t === MethodStoreHeader
189 +
        name = read(io)
190 +
        mod = read(io)
191 +
        file = read(io)
192 +
        line = Base.read(io, UInt32)
193 +
        nsig = Base.read(io, Int)
194 +
        sig = Any[]
195 +
        for i = 1:nsig
196 +
            push!(sig, read(io)=>read(io))
197 +
        end
198 +
        kws = read_vector(io, Symbol)
199 +
        rt = read(io)
200 +
        MethodStore(name, mod, file, line, sig, kws, rt)
201 +
    elseif t === FunctionStoreHeader
202 +
        FunctionStore(read(io), read_vector(io, MethodStore), read(io), read(io), read(io))
203 +
    elseif t === DataTypeStoreHeader
204 +
        DataTypeStore(read(io), read(io), read_vector(io, Any), read_vector(io, Any), read_vector(io, Any), read_vector(io, MethodStore), read(io), read(io))
205 +
    elseif t === GenericStoreHeader
206 +
        GenericStore(read(io), read(io), read(io), read(io))
207 +
    elseif t === ModuleStoreHeader
208 +
        name = read(io)
209 +
        n = Base.read(io, Int)
210 +
        vals = Dict{Symbol,Any}()
211 +
        for i = 1:n
212 +
            k = read(io)
213 +
            v = read(io)
214 +
            vals[k] = v
215 +
        end
216 +
        doc = read(io)
217 +
        exported = read(io)
218 +
        exportednames = read_vector(io, Symbol)
219 +
        used_modules = read_vector(io, Symbol)
220 +
        ModuleStore(name, vals, doc, exported, exportednames, used_modules)
221 +
    elseif t === TrueHeader
222 +
        true
223 +
    elseif t === FalseHeader
224 +
        false
225 +
    elseif t === TupleHeader
226 +
        N = Base.read(io, Int)
227 +
        ntuple(i->read(io), N)
228 +
    elseif t === PackageHeader
229 +
        name = read(io)
230 +
        val = read(io)
231 +
        uuid = Base.UUID(Base.read(io, UInt128))
232 +
        sha = Base.read(io, 32)
233 +
        Package(name, val, uuid, all(x == 0x00 for x in sha) ? nothing : sha)
234 +
    else
235 +
        error("HIYA: $t")
236 +
    end
237 +
end
238 +
239 +
function read_vector(io, T)
240 +
    n = Base.read(io, Int)
241 +
    v = T[]
242 +
    for i = 1:n
243 +
        push!(v, read(io))
244 +
    end
245 +
    v
246 +
end
247 +
248 +
function storeunstore(x)
249 +
    io = IOBuffer()
250 +
    write(io, x)
251 +
    bs = take!(io)    
252 +
    read(IOBuffer(bs))
253 +
end
254 +
end

@@ -385,7 +385,6 @@
Loading
385 385
386 386
recursive_copy(p::Package) = Package(p.name,
387 387
                                     recursive_copy(p.val),
388 -
                                     recursive_copy(p.ver),
389 388
                                     p.uuid,
390 389
                                     recursive_copy(p.sha))
391 390
@@ -416,3 +415,20 @@
Loading
416 415
                                                recursive_copy(gs.typ),
417 416
                                                gs.doc,
418 417
                                                gs.exported)
418 +
419 +
function todo_path(x::ModuleStore)
420 +
    for (_, a) in x.vals
421 +
        todo_path(a)
422 +
    end
423 +
end
424 +
425 +
function todo_path(x::Union{DataTypeStore,FunctionStore})
426 +
    for m in x.methods
427 +
        todo_path(m)
428 +
    end
429 +
end
430 +
function todo_path(x::MethodStore)
431 +
    # @info x.file
432 +
end
433 +
function todo_path(x)
434 +
end

@@ -2,13 +2,15 @@
Loading
2 2
3 3
export SymbolServerInstance, getstore
4 4
5 -
using Serialization, Pkg, SHA
5 +
using Pkg, SHA
6 6
using Base: UUID, Process
7 7
import Sockets, UUIDs
8 8
9 9
include("faketypes.jl")
10 10
include("symbols.jl")
11 11
include("utils.jl")
12 +
include("serialize.jl")
13 +
using .CacheStore
12 14
13 15
mutable struct SymbolServerInstance
14 16
    process::Union{Nothing,Base.Process}
@@ -169,7 +171,7 @@
Loading
169 171
    if isfile(cache_path)
170 172
        try
171 173
            package_data = open(cache_path) do io
172 -
                deserialize(io)
174 +
                CacheStore.read(io)
173 175
            end
174 176
            store[Symbol(pe_name)] = package_data.val
175 177
            for dep in deps(pe)

Learn more Showing 2 files with coverage changes found.

Changes in src/utils.jl
-2
+2
Loading file...
New file src/serialize.jl
New
Loading file...
Files Coverage
src +2.42% 73.60%
Project Totals (6 files) 73.60%
Loading