buildbot / buildbot

@@ -0,0 +1,43 @@
Loading
1 +
# Copyright Buildbot Team Members
2 +
#
3 +
# Permission is hereby granted, free of charge, to any person obtaining
4 +
# a copy of this software and associated documentation files (the
5 +
# "Software"), to deal in the Software without restriction, including
6 +
# without limitation the rights to use, copy, modify, merge, publish,
7 +
# distribute, sublicense, and/or sell copies of the Software, and to
8 +
# permit persons to whom the Software is furnished to do so, subject to
9 +
# the following conditions:
10 +
#
11 +
# The above copyright notice and this permission notice shall be
12 +
# included in all copies or substantial portions of the Software.
13 +
#
14 +
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 +
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 +
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 +
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 +
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 +
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 +
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 +
22 +
23 +
from twisted.internet import defer
24 +
25 +
26 +
@defer.inlineCallbacks
27 +
def async_sort(l, key, max_parallel=10):
28 +
    """perform an asynchronous sort with parallel run of the key algorithm
29 +
30 +
    We use a schwarzian transform and the standard python sort algorithm
31 +
    """
32 +
33 +
    sem = defer.DeferredSemaphore(max_parallel)
34 +
    try:
35 +
        keys = yield defer.gatherResults(
36 +
            [sem.run(key, i) for i in l])
37 +
    except defer.FirstError as e:
38 +
        raise e.subFailure.value
39 +
40 +
    keys = {id(l[i]): v for i, v in enumerate(keys)}
41 +
42 +
    # sort the transformed list synchronously with the first item
43 +
    l.sort(key=lambda x: keys[id(x)])

@@ -15,11 +15,10 @@
Loading
15 15
16 16
17 17
import copy
18 +
import math
18 19
import random
19 20
from datetime import datetime
20 21
21 -
from dateutil.tz import tzutc
22 -
23 22
from twisted.internet import defer
24 23
from twisted.python import log
25 24
from twisted.python.failure import Failure
@@ -30,6 +29,7 @@
Loading
30 29
from buildbot.util import deferwaiter
31 30
from buildbot.util import epoch2datetime
32 31
from buildbot.util import service
32 +
from buildbot.util.async_sort import async_sort
33 33
34 34
35 35
class BuildChooserBase:
@@ -369,44 +369,23 @@
Loading
369 369
    def _defaultSorter(self, master, builders):
370 370
        timer = metrics.Timer("BuildRequestDistributor._defaultSorter()")
371 371
        timer.start()
372 -
        # perform an asynchronous schwarzian transform, transforming None
373 -
        # into sys.maxint so that it sorts to the end
374 -
375 -
        def xform(bldr):
376 -
            d = defer.maybeDeferred(bldr.getOldestRequestTime)
377 -
            d.addCallback(lambda time:
378 -
                          (((time is None) and None or time), bldr))
379 -
            return d
380 -
        xformed = yield defer.gatherResults(
381 -
            [xform(bldr) for bldr in builders])
382 -
383 -
        # sort the transformed list synchronously, comparing None to the end of
384 -
        # the list
385 -
        def xformedKey(a):
386 -
            """
387 -
            Key function can be used to sort a list
388 -
            where each list element is a tuple:
389 -
                (datetime.datetime, Builder)
390 -
391 -
            @return: a tuple of (date, builder name)
392 -
            """
393 -
            (date, builder) = a
394 -
            if date is None:
395 -
                # Choose a really big date, so that any
396 -
                # date set to 'None' will appear at the
397 -
                # end of the list during comparisons.
398 -
                date = datetime.max
399 -
                # Need to set the timezone on the date, in order
400 -
                # to perform comparisons with other dates which
401 -
                # have the time zone set.
402 -
                date = date.replace(tzinfo=tzutc())
403 -
            return (date, builder.name)
404 -
        xformed.sort(key=xformedKey)
405 -
406 -
        # and reverse the transform
407 -
        rv = [xf[1] for xf in xformed]
372 +
373 +
        @defer.inlineCallbacks
374 +
        def key(bldr):
375 +
            # Sort by time of oldest build request
376 +
            time = yield bldr.getOldestRequestTime()
377 +
            if time is None:
378 +
                # for builders that do not have pending buildrequest, we just use large number
379 +
                time = math.inf
380 +
            else:
381 +
                if isinstance(time, datetime):
382 +
                    time = time.timestamp()
383 +
            defer.returnValue((time, bldr.name))
384 +
385 +
        yield async_sort(builders, key)
386 +
408 387
        timer.stop()
409 -
        return rv
388 +
        return builders
410 389
411 390
    @defer.inlineCallbacks
412 391
    def _sortBuilders(self, buildernames):
Files Coverage
master/buildbot 92.21%
worker/buildbot_worker 85.07%
Project Totals (332 files) 91.78%
Untitled
Untitled
Untitled

No yaml found.

Create your codecov.yml to customize your Codecov experience

Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading