tkf / BenchmarkCI.jl

Compare 969652c ... +0 ... 5bb0335

Coverage Reach
BenchmarkCI.jl runtimeinfo.jl

No flags found

Use flags to group coverage reports by test type, project and/or folders.
Then setup custom commit statuses and notifications for each flag.

e.g., #unittest #integration

#production #enterprise

#frontend #backend

Learn more about Codecov Flags here.

Showing 1 of 3 files from the diff.
Other files ignored by Codecov
README.md has changed.
test/runtests.jl has changed.

@@ -26,7 +26,7 @@
Loading
26 26
is_in_ci(ENV = ENV) =
27 27
    lowercase(get(ENV, "CI", "false")) == "true" || haskey(ENV, "GITHUB_EVENT_PATH")
28 28
29 -
function generate_script(default_script, project)
29 +
function generate_script(default_script, project, should_resolve)
30 30
    default_script = abspath(default_script)
31 31
    project = abspath(project)
32 32
    """
@@ -35,6 +35,7 @@
Loading
35 35
            "Pkg",
36 36
        ))
37 37
        Pkg.activate($(repr(project)))
38 +
        $(repr(should_resolve)) && Pkg.resolve()
38 39
        Pkg.instantiate()
39 40
    end
40 41
    include($(repr(default_script)))
@@ -52,6 +53,40 @@
Loading
52 53
    end
53 54
end
54 55
56 +
function maybe_with_merged_project(f, project, pkgdir)
57 +
    project = abspath(project)
58 +
    dir = endswith(project, ".toml") ? dirname(project) : project
59 +
    if any(isfile.(joinpath.(dir, ("JuliaManifest.toml", "Manifest.toml"))))
60 +
        @info "Using existing manifest file."
61 +
        return f(project, false)  # should_resolve = false
62 +
    else
63 +
        if isfile(project)
64 +
            file = project
65 +
        else
66 +
            candidates = joinpath.(dir, ("JuliaProject.toml", "Project.toml"))
67 +
            i = findfirst(isfile, candidates)
68 +
            if i === nothing
69 +
                error("One of the following files must exist:\n", join(candidates, "\n"))
70 +
            end
71 +
            file = candidates[i]
72 +
        end
73 +
        return mktempdir(prefix = "BenchmarkCI_jl_") do tmp
74 +
            tmpproject = joinpath(tmp, "Project.toml")
75 +
            cp(file, tmpproject)
76 +
            code = """
77 +
            using Pkg
78 +
            Pkg.develop(Pkg.PackageSpec(path = $(repr(pkgdir))))
79 +
            """
80 +
            run(setenv(
81 +
                `$(Base.julia_cmd()) --startup-file=no --project=$tmpproject -e $code`,
82 +
                "JULIA_LOAD_PATH" => "@:@stdlib",
83 +
            ))
84 +
            @info "Using temporary project `$tmp`."
85 +
            f(tmpproject, true)  # should_resolve = true
86 +
        end
87 +
    end
88 +
end
89 +
55 90
function format_period(seconds::Real)
56 91
    seconds < 60 && return string(floor(Int, seconds), " seconds")
57 92
    minutes = floor(Int, seconds / 60)
@@ -65,8 +100,8 @@
Loading
65 100
    target,
66 101
    baseline = "origin/master";
67 102
    workspace = DEFAULT_WORKSPACE,
68 -
    pkg = pwd(),
69 -
    script = joinpath(pkg, "benchmark", "benchmarks.jl"),
103 +
    pkgdir = pwd(),
104 +
    script = joinpath(pkgdir, "benchmark", "benchmarks.jl"),
70 105
    project = dirname(script),
71 106
    progressoptions = is_in_ci() ? (dt = 60 * 9.0,) : NamedTuple(),
72 107
)
@@ -75,14 +110,29 @@
Loading
75 110
        baseline = @set target.id = baseline
76 111
    end
77 112
78 -
    mkpath(workspace)
79 -
    script_wrapper = abspath(joinpath(workspace, "benchmarks_wrapper.jl"))
80 -
    write(script_wrapper, generate_script(script, project))
81 -
82 113
    # Make sure `origin/master` etc. exists:
83 114
    ensure_origin(target)
84 115
    ensure_origin(baseline)
85 116
117 +
    mkpath(workspace)
118 +
    script_wrapper = abspath(joinpath(workspace, "benchmarks_wrapper.jl"))
119 +
120 +
    maybe_with_merged_project(project, pkgdir) do tmpproject, should_resolve
121 +
        write(script_wrapper, generate_script(script, tmpproject, should_resolve))
122 +
        _judge(;
123 +
            target = target,
124 +
            baseline = baseline,
125 +
            workspace = workspace,
126 +
            pkgdir = pkgdir,
127 +
            benchmarkpkg_kwargs = (
128 +
                progressoptions = progressoptions,
129 +
                script = script_wrapper,
130 +
            ),
131 +
        )
132 +
    end
133 +
end
134 +
135 +
function noisily(f)
86 136
    if is_in_ci()
87 137
        ch = Channel() do ch
88 138
            t0 = time_ns()
@@ -93,25 +143,31 @@
Loading
93 143
                @info "$minutes minutes passed.  Still running `judge`..."
94 144
            end
95 145
        end
146 +
        try
147 +
            f()
148 +
        finally
149 +
            close(ch)
150 +
        end
96 151
    else
97 -
        ch = nothing
152 +
        f()
98 153
    end
154 +
end
155 +
156 +
function _judge(; target, baseline, workspace, pkgdir, benchmarkpkg_kwargs)
99 157
100 -
    try
158 +
    noisily() do
101 159
        time_target = @elapsed group_target = PkgBenchmark.benchmarkpkg(
102 -
            pkg,
103 -
            target,
104 -
            progressoptions = progressoptions,
160 +
            pkgdir,
161 +
            target;
105 162
            resultfile = joinpath(workspace, "result-target.json"),
106 -
            script = script_wrapper,
163 +
            benchmarkpkg_kwargs...,
107 164
        )
108 165
        @debug("`git status`", output = Text(read(`git status`, String)))
109 166
        time_baseline = @elapsed group_baseline = PkgBenchmark.benchmarkpkg(
110 -
            pkg,
111 -
            baseline,
112 -
            progressoptions = progressoptions,
167 +
            pkgdir,
168 +
            baseline;
113 169
            resultfile = joinpath(workspace, "result-baseline.json"),
114 -
            script = script_wrapper,
170 +
            benchmarkpkg_kwargs...,
115 171
        )
116 172
        @info """
117 173
        Finish running benchmarks.
@@ -123,8 +179,6 @@
Loading
123 179
            display(judgement)
124 180
        end
125 181
        return judgement
126 -
    finally
127 -
        ch === nothing || close(ch)
128 182
    end
129 183
end
130 184
@@ -255,6 +309,7 @@
Loading
255 309
    printresultmd(io, CIResult(judgement = judgement))
256 310
    seekstart(io)
257 311
    display(Markdown.parse(io))
312 +
    display(Text("\n"))
258 313
end
259 314
260 315
runall(args...; kwargs...) = postjudge(judge(args...; kwargs...))

Learn more Showing 1 files with coverage changes found.

Changes in src/BenchmarkCI.jl
-1
+1
Loading file...
Files Coverage
src +4.66% 83.12%
Project Totals (2 files) 83.12%
Loading