buildbot / buildbot

Compare c99fcb4 ... +2 ... 3a50054

Coverage Reach
master/buildbot/steps/source/git.py master/buildbot/steps/source/svn.py master/buildbot/steps/source/repo.py master/buildbot/steps/source/cvs.py master/buildbot/steps/source/mercurial.py master/buildbot/steps/source/mtn.py master/buildbot/steps/source/p4.py master/buildbot/steps/source/bzr.py master/buildbot/steps/source/darcs.py master/buildbot/steps/source/base.py master/buildbot/steps/source/gerrit.py master/buildbot/steps/source/gitlab.py master/buildbot/steps/source/github.py master/buildbot/steps/source/__init__.py master/buildbot/steps/package/deb/pbuilder.py master/buildbot/steps/package/deb/lintian.py master/buildbot/steps/package/rpm/mock.py master/buildbot/steps/package/rpm/rpmbuild.py master/buildbot/steps/package/rpm/rpmlint.py master/buildbot/steps/package/rpm/__init__.py master/buildbot/steps/package/util.py master/buildbot/steps/package/__init__.py master/buildbot/steps/shell.py master/buildbot/steps/vstudio.py master/buildbot/steps/transfer.py master/buildbot/steps/python.py master/buildbot/steps/python_twisted.py master/buildbot/steps/trigger.py master/buildbot/steps/worker.py master/buildbot/steps/master.py master/buildbot/steps/http.py master/buildbot/steps/subunit.py master/buildbot/steps/shellsequence.py master/buildbot/steps/cppcheck.py master/buildbot/steps/mswin.py master/buildbot/steps/gitdiffinfo.py master/buildbot/steps/download_secret_to_worker.py master/buildbot/steps/maxq.py master/buildbot/steps/cmake.py master/buildbot/reporters/words.py master/buildbot/reporters/telegram.py master/buildbot/reporters/generators/utils.py master/buildbot/reporters/generators/buildset.py master/buildbot/reporters/generators/build.py master/buildbot/reporters/generators/buildrequest.py master/buildbot/reporters/generators/worker.py master/buildbot/reporters/irc.py master/buildbot/reporters/gerrit.py master/buildbot/reporters/bitbucketserver.py master/buildbot/reporters/mail.py master/buildbot/reporters/message.py master/buildbot/reporters/github.py master/buildbot/reporters/utils.py master/buildbot/reporters/gitlab.py master/buildbot/reporters/gerrit_verify_status.py master/buildbot/reporters/bitbucket.py master/buildbot/reporters/base.py master/buildbot/reporters/pushover.py master/buildbot/reporters/pushjet.py master/buildbot/reporters/zulip.py master/buildbot/reporters/http.py master/buildbot/process/buildstep.py master/buildbot/process/properties.py master/buildbot/process/build.py master/buildbot/process/metrics.py master/buildbot/process/buildrequestdistributor.py master/buildbot/process/remotecommand.py master/buildbot/process/builder.py master/buildbot/process/buildrequest.py master/buildbot/process/botmaster.py master/buildbot/process/users/manual.py master/buildbot/process/users/users.py master/buildbot/process/users/manager.py master/buildbot/process/factory.py master/buildbot/process/workerforbuilder.py master/buildbot/process/log.py master/buildbot/process/logobserver.py master/buildbot/process/remotetransfer.py master/buildbot/process/results.py master/buildbot/process/debug.py master/buildbot/process/cache.py master/buildbot/process/measured_service.py master/buildbot/process/base.py master/buildbot/process/subunitlogobserver.py master/buildbot/util/__init__.py master/buildbot/util/service.py master/buildbot/util/croniter.py master/buildbot/util/runprocess.py master/buildbot/util/git.py master/buildbot/util/kubeclientservice.py master/buildbot/util/ssfilter.py master/buildbot/util/lru.py master/buildbot/util/httpclientservice.py master/buildbot/util/test_result_submitter.py master/buildbot/util/poll.py master/buildbot/util/queue.py master/buildbot/util/maildir.py master/buildbot/util/raml.py master/buildbot/util/deferwaiter.py master/buildbot/util/debounce.py master/buildbot/util/subscription.py master/buildbot/util/eventual.py master/buildbot/util/pathmatch.py master/buildbot/util/config.py master/buildbot/util/backoff.py master/buildbot/util/lineboundaries.py master/buildbot/util/sautils.py master/buildbot/util/identifiers.py master/buildbot/util/protocol.py master/buildbot/util/netstrings.py master/buildbot/util/bbcollections.py master/buildbot/util/misc.py master/buildbot/util/private_tempdir.py master/buildbot/util/latent.py master/buildbot/util/pullrequest.py master/buildbot/util/giturlparse.py master/buildbot/util/codebase.py master/buildbot/util/ssl.py master/buildbot/util/logger.py master/buildbot/util/_notifier.py master/buildbot/util/state.py master/buildbot/util/async_sort.py master/buildbot/util/tuplematch.py master/buildbot/db/logs.py master/buildbot/db/changes.py master/buildbot/db/model.py master/buildbot/db/test_results.py master/buildbot/db/pool.py master/buildbot/db/workers.py master/buildbot/db/builds.py master/buildbot/db/buildrequests.py master/buildbot/db/users.py master/buildbot/db/buildsets.py master/buildbot/db/enginestrategy.py master/buildbot/db/schedulers.py master/buildbot/db/steps.py master/buildbot/db/state.py master/buildbot/db/sourcestamps.py master/buildbot/db/build_data.py master/buildbot/db/base.py master/buildbot/db/connector.py master/buildbot/db/builders.py master/buildbot/db/test_result_sets.py master/buildbot/db/masters.py master/buildbot/db/dbconfig.py master/buildbot/db/changesources.py master/buildbot/db/migrate_utils.py master/buildbot/db/types/json.py master/buildbot/db/migrations/versions/059_2021-09-07_alembic_initial.py master/buildbot/db/tags.py master/buildbot/db/exceptions.py master/buildbot/db/__init__.py master/buildbot/www/hooks/github.py master/buildbot/www/hooks/bitbucketserver.py master/buildbot/www/hooks/gitlab.py master/buildbot/www/hooks/bitbucketcloud.py master/buildbot/www/hooks/base.py master/buildbot/www/hooks/poller.py master/buildbot/www/hooks/gitorious.py master/buildbot/www/hooks/bitbucket.py master/buildbot/www/rest.py master/buildbot/www/authz/endpointmatchers.py master/buildbot/www/authz/roles.py master/buildbot/www/authz/authz.py master/buildbot/www/authz/__init__.py master/buildbot/www/oauth2.py master/buildbot/www/service.py master/buildbot/www/avatar.py master/buildbot/www/ws.py master/buildbot/www/auth.py master/buildbot/www/ldapuserinfo.py master/buildbot/www/change_hook.py master/buildbot/www/sse.py master/buildbot/www/config.py master/buildbot/www/resource.py master/buildbot/www/graphql.py master/buildbot/www/plugin.py master/buildbot/data/types.py master/buildbot/data/resultspec.py master/buildbot/data/connector.py master/buildbot/data/graphql.py master/buildbot/data/buildrequests.py master/buildbot/data/base.py master/buildbot/data/buildsets.py master/buildbot/data/builds.py master/buildbot/data/changes.py master/buildbot/data/masters.py master/buildbot/data/workers.py master/buildbot/data/steps.py master/buildbot/data/logs.py master/buildbot/data/schedulers.py master/buildbot/data/builders.py master/buildbot/data/logchunks.py master/buildbot/data/test_result_sets.py master/buildbot/data/forceschedulers.py master/buildbot/data/changesources.py master/buildbot/data/build_data.py master/buildbot/data/properties.py master/buildbot/data/test_results.py master/buildbot/data/sourcestamps.py master/buildbot/data/root.py master/buildbot/data/patches.py master/buildbot/data/exceptions.py master/buildbot/worker/base.py master/buildbot/worker/ec2.py master/buildbot/worker/protocols/pb.py master/buildbot/worker/protocols/null.py master/buildbot/worker/protocols/base.py master/buildbot/worker/latent.py master/buildbot/worker/docker.py master/buildbot/worker/openstack.py master/buildbot/worker/libvirt.py master/buildbot/worker/upcloud.py master/buildbot/worker/manager.py master/buildbot/worker/kubernetes.py master/buildbot/worker/marathon.py master/buildbot/worker/local.py master/buildbot/worker/__init__.py master/buildbot/changes/mail.py master/buildbot/changes/gitpoller.py master/buildbot/changes/gerritchangesource.py master/buildbot/changes/svnpoller.py master/buildbot/changes/p4poller.py master/buildbot/changes/hgpoller.py master/buildbot/changes/github.py master/buildbot/changes/changes.py master/buildbot/changes/pb.py master/buildbot/changes/bitbucket.py master/buildbot/changes/base.py master/buildbot/changes/filter.py master/buildbot/changes/manager.py master/buildbot/schedulers/forcesched.py master/buildbot/schedulers/trysched.py master/buildbot/schedulers/timed.py master/buildbot/schedulers/base.py master/buildbot/schedulers/canceller.py master/buildbot/schedulers/basic.py master/buildbot/schedulers/canceller_buildset.py master/buildbot/schedulers/dependent.py master/buildbot/schedulers/triggerable.py master/buildbot/schedulers/manager.py master/buildbot/schedulers/filter.py master/buildbot/scripts/runner.py master/buildbot/scripts/base.py master/buildbot/scripts/logwatcher.py master/buildbot/scripts/upgrade_master.py master/buildbot/scripts/start.py master/buildbot/scripts/create_master.py master/buildbot/scripts/reconfig.py master/buildbot/scripts/stop.py master/buildbot/scripts/cleanupdb.py master/buildbot/scripts/sendchange.py master/buildbot/scripts/checkconfig.py master/buildbot/scripts/user.py master/buildbot/scripts/tryserver.py master/buildbot/scripts/sample.cfg master/buildbot/scripts/restart.py master/buildbot/scripts/trycmd.py master/buildbot/clients/tryclient.py master/buildbot/clients/sendchange.py master/buildbot/clients/usersclient.py master/buildbot/config.py master/buildbot/master.py master/buildbot/statistics/capture.py master/buildbot/statistics/stats_service.py master/buildbot/statistics/storage_backends/influxdb_client.py master/buildbot/statistics/storage_backends/base.py master/buildbot/statistics/__init__.py master/buildbot/machine/generic.py master/buildbot/machine/latent.py master/buildbot/machine/base.py master/buildbot/machine/manager.py master/buildbot/plugins/db.py master/buildbot/plugins/__init__.py master/buildbot/locks.py master/buildbot/mq/wamp.py master/buildbot/mq/simple.py master/buildbot/mq/base.py master/buildbot/mq/connector.py master/buildbot/secrets/providers/vault.py master/buildbot/secrets/providers/file.py master/buildbot/secrets/providers/passwordstore.py master/buildbot/secrets/providers/base.py master/buildbot/secrets/secret.py master/buildbot/secrets/manager.py master/buildbot/manhole.py master/buildbot/buildbot_net_usage_data.py master/buildbot/pbmanager.py master/buildbot/wamp/connector.py master/buildbot/configurators/janitor.py master/buildbot/configurators/__init__.py master/buildbot/monkeypatches/__init__.py master/buildbot/monkeypatches/servicechecks.py master/buildbot/monkeypatches/decorators.py master/buildbot/pbutil.py master/buildbot/interfaces.py master/buildbot/__init__.py master/buildbot/asyncio.py master/buildbot/revlinks.py master/buildbot/scheduler.py master/buildbot/warnings.py master/buildbot/errors.py master/buildbot/buildrequest.py worker/buildbot_worker/runprocess.py worker/buildbot_worker/commands/transfer.py worker/buildbot_worker/commands/fs.py worker/buildbot_worker/commands/base.py worker/buildbot_worker/commands/shell.py worker/buildbot_worker/commands/utils.py worker/buildbot_worker/commands/registry.py worker/buildbot_worker/scripts/runner.py worker/buildbot_worker/scripts/create_worker.py worker/buildbot_worker/scripts/logwatcher.py worker/buildbot_worker/scripts/start.py worker/buildbot_worker/scripts/stop.py worker/buildbot_worker/scripts/restart.py worker/buildbot_worker/scripts/base.py worker/buildbot_worker/base.py worker/buildbot_worker/pb.py worker/buildbot_worker/util/__init__.py worker/buildbot_worker/util/_hangcheck.py worker/buildbot_worker/util/_notifier.py worker/buildbot_worker/__init__.py worker/buildbot_worker/pbutil.py worker/buildbot_worker/monkeypatches/testcase_assert.py worker/buildbot_worker/monkeypatches/__init__.py worker/buildbot_worker/compat.py worker/buildbot_worker/null.py worker/buildbot_worker/interfaces.py worker/buildbot_worker/backports/__init__.py worker/buildbot_worker/bot.py worker/buildbot_worker/exceptions.py

No flags found

Use flags to group coverage reports by test type, project and/or folders.
Then setup custom commit statuses and notifications for each flag.

e.g., #unittest #integration

#production #enterprise

#frontend #backend

Learn more about Codecov Flags here.


@@ -0,0 +1,42 @@
Loading
1 +
# Copyright Buildbot Team Members
2 +
#
3 +
# Permission is hereby granted, free of charge, to any person obtaining
4 +
# a copy of this software and associated documentation files (the
5 +
# "Software"), to deal in the Software without restriction, including
6 +
# without limitation the rights to use, copy, modify, merge, publish,
7 +
# distribute, sublicense, and/or sell copies of the Software, and to
8 +
# permit persons to whom the Software is furnished to do so, subject to
9 +
# the following conditions:
10 +
#
11 +
# The above copyright notice and this permission notice shall be
12 +
# included in all copies or substantial portions of the Software.
13 +
#
14 +
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 +
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 +
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 +
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 +
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 +
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 +
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 +
22 +
23 +
from twisted.internet import defer
24 +
25 +
26 +
@defer.inlineCallbacks
27 +
def async_sort(l, key, max_parallel=10):
28 +
    """perform an asynchronous sort with parallel run of the key algorithm
29 +
    """
30 +
31 +
    sem = defer.DeferredSemaphore(max_parallel)
32 +
    try:
33 +
        keys = yield defer.gatherResults(
34 +
            [sem.run(key, i) for i in l])
35 +
    except defer.FirstError as e:
36 +
        raise e.subFailure.value
37 +
38 +
    # Index the keys by the id of the original item in list
39 +
    keys = {id(l[i]): v for i, v in enumerate(keys)}
40 +
41 +
    # now we can sort the list in place
42 +
    l.sort(key=lambda x: keys[id(x)])

@@ -15,11 +15,10 @@
Loading
15 15
16 16
17 17
import copy
18 +
import math
18 19
import random
19 20
from datetime import datetime
20 21
21 -
from dateutil.tz import tzutc
22 -
23 22
from twisted.internet import defer
24 23
from twisted.python import log
25 24
from twisted.python.failure import Failure
@@ -30,6 +29,7 @@
Loading
30 29
from buildbot.util import deferwaiter
31 30
from buildbot.util import epoch2datetime
32 31
from buildbot.util import service
32 +
from buildbot.util.async_sort import async_sort
33 33
34 34
35 35
class BuildChooserBase:
@@ -369,38 +369,23 @@
Loading
369 369
    def _defaultSorter(self, master, builders):
370 370
        timer = metrics.Timer("BuildRequestDistributor._defaultSorter()")
371 371
        timer.start()
372 -
        # perform an asynchronous schwarzian transform, transforming None
373 -
        # into a really big date, so that any
374 -
        # date set to 'None' will appear at the
375 -
        # end of the list during comparisons.
376 -
        max_time = datetime.max
377 -
        # Need to set the timezone on the date, in order
378 -
        # to perform comparisons with other dates which
379 -
        # have the time zone set.
380 -
        max_time = max_time.replace(tzinfo=tzutc())
381 372
382 373
        @defer.inlineCallbacks
383 -
        def transform(bldr):
374 +
        def key(bldr):
375 +
            # Sort by time of oldest build request
384 376
            time = yield bldr.getOldestRequestTime()
385 377
            if time is None:
386 -
                time = max_time
387 -
            defer.returnValue((time, bldr))
388 -
389 -
        transformed = yield defer.gatherResults(
390 -
            [transform(bldr) for bldr in builders])
378 +
                # for builders that do not have pending buildrequest, we just use large number
379 +
                time = math.inf
380 +
            else:
381 +
                if isinstance(time, datetime):
382 +
                    time = time.timestamp()
383 +
            defer.returnValue((time, bldr.name))
391 384
392 -
        # sort the transformed list synchronously, comparing None to the end of
393 -
        # the list
394 -
        def transformedKey(a):
395 -
            (date, builder) = a
396 -
            return (date, builder.name)
385 +
        yield async_sort(builders, key)
397 386
398 -
        transformed.sort(key=transformedKey)
399 -
400 -
        # and reverse the transform
401 -
        rv = [xf[1] for xf in transformed]
402 387
        timer.stop()
403 -
        return rv
388 +
        return builders
404 389
405 390
    @defer.inlineCallbacks
406 391
    def _sortBuilders(self, buildernames):

Learn more Showing 2 files with coverage changes found.

New file master/buildbot/util/async_sort.py
New
Loading file...
Changes in master/buildbot/util/queue.py
-5
+5
Loading file...
Files Coverage
master/buildbot 0.02% 92.30%
worker/buildbot_worker 85.52%
Project Totals (332 files) 91.90%
Loading