1 30
module SymbolServer
2
import Sockets
3

4
pipename = length(ARGS) > 1 ? ARGS[2] : nothing
5

6
conn = pipename !== nothing ? Sockets.connect(pipename) : nothing
7

8
start_time = time_ns()
9

10
# Try to lower the priority of this process so that it doesn't block the
11
# user system.
12
@static if Sys.iswindows()
13
    # Get process handle
14 12
    p_handle = ccall(:GetCurrentProcess, stdcall, Ptr{Cvoid}, ())
15

16
    # Set BELOW_NORMAL_PRIORITY_CLASS
17 12
    ret = ccall(:SetPriorityClass, stdcall, Cint, (Ptr{Cvoid}, Culong), p_handle, 0x00004000)
18 12
    ret != 1 && @warn "Something went wrong when setting BELOW_NORMAL_PRIORITY_CLASS."
19
else
20 18
    ret = ccall(:nice, Cint, (Cint,), 1)
21
    # We don't check the return value because it doesn't really matter
22
end
23

24 30
module LoadingBay
25
end
26

27
# Make sure we can load stdlibs
28
!in("@stdlib", LOAD_PATH) && push!(LOAD_PATH, "@stdlib")
29

30
using Pkg, SHA
31
using Base: UUID
32

33
include("faketypes.jl")
34
include("symbols.jl")
35
include("utils.jl")
36
include("serialize.jl")
37
using .CacheStore
38

39
store_path = length(ARGS) > 0 ? ARGS[1] : abspath(joinpath(@__DIR__, "..", "store"))
40

41
ctx = try
42
    Pkg.Types.Context()
43
catch err
44
    @info "Package environment can't be read."
45
    exit()
46
end
47
# Add some methods to check whether a package is part of the standard library and so
48
# won't need recaching.
49
if isdefined(Pkg.Types, :is_stdlib)
50 20
    is_stdlib(uuid::UUID) = Pkg.Types.is_stdlib(uuid)
51
else
52 10
    is_stdlib(uuid::UUID) = uuid in keys(ctx.stdlibs)
53
end
54

55
server = Server(store_path, ctx, Dict{UUID,Package}())
56

57
function load_package(c::Pkg.Types.Context, uuid, conn)
58 30
    isinmanifest(c, uuid isa String ? Base.UUID(uuid) : uuid) || return
59 30
    pe_name = packagename(c, uuid)
60

61 30
    pid = Base.PkgId(uuid isa String ? Base.UUID(uuid) : uuid, pe_name)
62 30
    if pid in keys(Base.loaded_modules)
63 30
        conn !== nothing && println(conn, "PROCESSPKG;$pe_name;$uuid;noversion")
64 30
        LoadingBay.eval(:($(Symbol(pe_name)) = $(Base.loaded_modules[pid])))
65 30
        m = getfield(LoadingBay, Symbol(pe_name))
66
    else
67 30
        m = try
68 30
            conn !== nothing && println(conn, "STARTLOAD;$pe_name;$uuid;noversion")
69 30
            LoadingBay.eval(:(import $(Symbol(pe_name))))
70 30
            conn !== nothing && println(conn, "STOPLOAD;$pe_name")
71 30
            m = getfield(LoadingBay, Symbol(pe_name))
72
        catch e
73 30
            return
74
        end
75
    end
76
end
77

78
function write_cache(name, pkg)
79 30
    open(joinpath(server.storedir, name), "w") do io
80 10
        CacheStore.write(io, pkg)
81
    end
82
end
83

84
function write_depot(server, ctx, written_caches)
85 30
    for (uuid, pkg) in server.depot
86 30
        filename = get_filename_from_name(ctx.env.manifest, uuid)
87 30
        filename === nothing && continue
88 30
        cache_path = joinpath(server.storedir, filename)
89 30
        cache_path in written_caches && continue
90 25
        push!(written_caches, cache_path)
91 25
        @info "Now writing to disc $uuid"
92 30
        write_cache(cache_path, pkg)
93
    end
94
end
95
# List of caches that have already been written
96
written_caches = String[]
97

98
# First get a list of all package UUIds that we want to cache
99
toplevel_pkgs = deps(project(ctx))
100
packages_to_load = []
101
# Next make sure the cache is up-to-date for all of these
102

103
for (pk_name, uuid) in toplevel_pkgs
104
    file_name = get_filename_from_name(ctx.env.manifest, uuid)
105
    # We sometimes have UUIDs in the project file that are not in the
106
    # manifest file. That seems like something that shouldn't happen, but
107
    # in practice is not under our control. For now, we just skip these
108
    # packages
109
    file_name === nothing && continue
110
    cache_path = joinpath(server.storedir, file_name)
111

112
    if isfile(cache_path)
113
        if is_package_deved(ctx.env.manifest, uuid)
114
            cached_version = open(cache_path) do io
115 0
                CacheStore.read(io)
116
            end
117
            if sha_pkg(frommanifest(ctx.env.manifest, uuid)) != cached_version.sha
118
                @info "Outdated sha, will recache package $pk_name ($uuid)"
119
                push!(packages_to_load, uuid)
120
            else
121
                @info "Package $pk_name ($uuid) is cached."
122
            end
123
        else
124
            @info "Package $pk_name ($uuid) is cached."
125
        end
126
    else
127
        @info "Will cache package $pk_name ($uuid)"
128
        push!(packages_to_load, uuid)
129
    end
130
end
131

132
# Load all packages together
133 20
for uuid in packages_to_load
134 30
    load_package(ctx, uuid, conn)
135
end
136

137
# Create image of whole package env. This creates the module structure only.
138
env_symbols = getenvtree()
139

140
# Populate the above with symbols, skipping modules that don't need caching.
141
# symbols (env_symbols)
142
visited = Base.IdSet{Module}([Base, Core]) # don't need to cache these each time...
143 20
for (pid, m) in Base.loaded_modules
144 30
    if pid.uuid !== nothing && is_stdlib(pid.uuid) &&
145
        (file_name = get_filename_from_name(ctx.env.manifest, pid.uuid)) !== nothing &&
146
        isfile(joinpath(server.storedir, file_name))
147 0
        push!(visited, m)
148 5
        delete!(env_symbols, Symbol(pid.name))
149
    end
150
end
151

152
symbols(env_symbols, nothing, getallns(), visited)
153

154
# Wrap the `ModuleStore`s as `Package`s.
155 20
for (pkg_name, cache) in env_symbols
156 30
    pkg_name = String(pkg_name)
157 30
    !isinmanifest(ctx, pkg_name) && continue
158 30
    uuid = packageuuid(ctx, String(pkg_name))
159 30
    pe = frommanifest(ctx, uuid)
160 30
    server.depot[uuid] = Package(String(pkg_name), cache, uuid, sha_pkg(pe))
161
end
162

163
# Write to disc
164
write_depot(server, server.context, written_caches)
165
end_time = time_ns()
166

167
elapsed_time_in_s = (end_time - start_time) / 1e9
168
@info "Symbol server indexing took $elapsed_time_in_s seconds."
169

170
end

Read our documentation on viewing source code .

Loading