Showing 98 of 461 files from the diff.
Other files ignored by Codecov
.travis.yml was deleted.
Makefile has changed.
smokes/yarn.lock has changed.
.pyup.yml has changed.
worker/setup.py has changed.
master/Dockerfile has changed.
RELEASING.rst has changed.
appveyor.yml has changed.
master/setup.py has changed.
.bbtravis.yml has changed.

@@ -14,20 +14,17 @@
Loading
14 14
# Copyright Buildbot Team Members
15 15
16 16
import inspect
17 -
import re
18 17
import sys
19 -
from io import StringIO
20 18
21 19
from twisted.internet import defer
22 20
from twisted.internet import error
23 -
from twisted.python import components
24 21
from twisted.python import deprecate
25 -
from twisted.python import failure
26 22
from twisted.python import log
27 -
from twisted.python import util as twutil
28 23
from twisted.python import versions
24 +
from twisted.python.deprecate import deprecatedModuleAttribute
29 25
from twisted.python.failure import Failure
30 26
from twisted.python.reflect import accumulateClassList
27 +
from twisted.python.versions import Version
31 28
from twisted.web.util import formatFailure
32 29
from zope.interface import implementer
33 30
@@ -52,7 +49,6 @@
Loading
52 49
from buildbot.process.results import SUCCESS
53 50
from buildbot.process.results import WARNINGS
54 51
from buildbot.process.results import Results
55 -
from buildbot.process.results import worst_status
56 52
from buildbot.util import bytes2unicode
57 53
from buildbot.util import debounce
58 54
from buildbot.util import flatten
@@ -76,14 +72,52 @@
Loading
76 72
77 73
# old import paths for these classes
78 74
RemoteCommand = remotecommand.RemoteCommand
75 +
deprecatedModuleAttribute(
76 +
    Version("buildbot", 2, 10, 1),
77 +
    message="Use buildbot.process.remotecommand.RemoteCommand instead.",
78 +
    moduleName="buildbot.process.buildstep",
79 +
    name="RemoteCommand",
80 +
)
81 +
79 82
LoggedRemoteCommand = remotecommand.LoggedRemoteCommand
83 +
deprecatedModuleAttribute(
84 +
    Version("buildbot", 2, 10, 1),
85 +
    message="Use buildbot.process.remotecommand.LoggedRemoteCommand instead.",
86 +
    moduleName="buildbot.process.buildstep",
87 +
    name="LoggedRemoteCommand",
88 +
)
89 +
80 90
RemoteShellCommand = remotecommand.RemoteShellCommand
91 +
deprecatedModuleAttribute(
92 +
    Version("buildbot", 2, 10, 1),
93 +
    message="Use buildbot.process.remotecommand.RemoteShellCommand instead.",
94 +
    moduleName="buildbot.process.buildstep",
95 +
    name="RemoteShellCommand",
96 +
)
97 +
81 98
LogObserver = logobserver.LogObserver
99 +
deprecatedModuleAttribute(
100 +
    Version("buildbot", 2, 10, 1),
101 +
    message="Use buildbot.process.logobserver.LogObserver instead.",
102 +
    moduleName="buildbot.process.buildstep",
103 +
    name="LogObserver",
104 +
)
105 +
82 106
LogLineObserver = logobserver.LogLineObserver
107 +
deprecatedModuleAttribute(
108 +
    Version("buildbot", 2, 10, 1),
109 +
    message="Use buildbot.util.LogLineObserver instead.",
110 +
    moduleName="buildbot.process.buildstep",
111 +
    name="LogLineObserver",
112 +
)
113 +
83 114
OutputProgressObserver = logobserver.OutputProgressObserver
84 -
_hush_pyflakes = [
85 -
    RemoteCommand, LoggedRemoteCommand, RemoteShellCommand,
86 -
    LogObserver, LogLineObserver, OutputProgressObserver]
115 +
deprecatedModuleAttribute(
116 +
    Version("buildbot", 2, 10, 1),
117 +
    message="Use buildbot.process.logobserver.OutputProgressObserver instead.",
118 +
    moduleName="buildbot.process.buildstep",
119 +
    name="OutputProgressObserver",
120 +
)
87 121
88 122
89 123
@implementer(interfaces.IBuildStepFactory)
@@ -110,150 +144,31 @@
Loading
110 144
            raise
111 145
112 146
113 -
def _maybeUnhandled(fn):
114 -
    def wrap(self, *args, **kwargs):
115 -
        d = fn(self, *args, **kwargs)
116 -
        if self._start_unhandled_deferreds is not None:
117 -
            self._start_unhandled_deferreds.append(d)
118 -
        return d
119 -
    wrap.__wrapped__ = fn
120 -
    twutil.mergeFunctionMetadata(fn, wrap)
121 -
    return wrap
122 -
123 -
124 -
class SyncLogFileWrapper(logobserver.LogObserver):
125 -
126 -
    # A temporary wrapper around process.log.Log to emulate *synchronous*
127 -
    # writes to the logfile by handling the Deferred from each add* operation
128 -
    # as part of the step's _start_unhandled_deferreds.  This has to handle
129 -
    # the tricky case of adding data to a log *before* addLog has returned!
130 -
    # this also adds the read-only methods such as getText
131 -
132 -
    # old constants from the status API
133 -
    HEADER = 0
134 -
    STDERR = 1
135 -
    STDOUT = 2
136 -
137 -
    def __init__(self, step, name, addLogDeferred):
138 -
        self.step = step
139 -
        self.name = name
140 -
        self.delayedOperations = []
141 -
        self.asyncLogfile = None
142 -
        self.chunks = []
143 -
        self.finished = False
144 -
        self.finishDeferreds = []
145 -
146 -
        self.step._sync_addlog_deferreds.append(addLogDeferred)
147 -
148 -
        @addLogDeferred.addCallback
149 -
        def gotAsync(log):
150 -
            self.asyncLogfile = log
151 -
            self._catchup()
152 -
            return log
153 -
154 -
        # run _catchup even if there's an error; it will helpfully generate
155 -
        # a whole bunch more!
156 -
        @addLogDeferred.addErrback
157 -
        def problem(f):
158 -
            self._catchup()
159 -
            return f
160 -
161 -
    def _catchup(self):
162 -
        if not self.asyncLogfile or not self.delayedOperations:
163 -
            return
164 -
        op = self.delayedOperations.pop(0)
165 -
166 -
        try:
167 -
            d = defer.maybeDeferred(op)
168 -
        except Exception:
169 -
            d = defer.fail(failure.Failure())
170 -
171 -
        @d.addBoth
172 -
        def next(x):
173 -
            self._catchup()
174 -
            return x
175 -
        self.step._start_unhandled_deferreds.append(d)
176 -
177 -
    def _delay(self, op):
178 -
        self.delayedOperations.append(op)
179 -
        if len(self.delayedOperations) == 1:
180 -
            self._catchup()
181 -
182 -
    def _maybeFinished(self):
183 -
        if self.finished and self.finishDeferreds:
184 -
            pending = self.finishDeferreds
185 -
            self.finishDeferreds = []
186 -
            for d in pending:
187 -
                d.callback(self)
188 -
189 -
    # write methods
190 -
191 -
    def addStdout(self, data):
192 -
        data = bytes2unicode(data)
193 -
        self.chunks.append((self.STDOUT, data))
194 -
        self._delay(lambda: self.asyncLogfile.addStdout(data))
195 -
196 -
    def addStderr(self, data):
197 -
        data = bytes2unicode(data)
198 -
        self.chunks.append((self.STDERR, data))
199 -
        self._delay(lambda: self.asyncLogfile.addStderr(data))
200 -
201 -
    def addHeader(self, data):
202 -
        data = bytes2unicode(data)
203 -
        self.chunks.append((self.HEADER, data))
204 -
        self._delay(lambda: self.asyncLogfile.addHeader(data))
205 -
206 -
    def finish(self):
207 -
        self.finished = True
208 -
        self._maybeFinished()
209 -
        # pylint: disable=unnecessary-lambda
210 -
        self._delay(lambda: self.asyncLogfile.finish())
211 -
212 -
    def unwrap(self):
213 -
        d = defer.Deferred()
214 -
        self._delay(lambda: d.callback(self.asyncLogfile))
215 -
        return d
216 -
217 -
    # read-only methods
218 -
219 -
    def getName(self):
220 -
        return self.name
221 -
222 -
    def getText(self):
223 -
        return "".join(self.getChunks([self.STDOUT, self.STDERR], onlyText=True))
147 +
class BuildStepStatus:
148 +
    # used only for old-style steps
149 +
    pass
224 150
225 -
    def readlines(self):
226 -
        alltext = "".join(self.getChunks([self.STDOUT], onlyText=True))
227 -
        io = StringIO(alltext)
228 -
        return io.readlines()
229 151
230 -
    def getChunks(self, channels=None, onlyText=False):
231 -
        chunks = self.chunks
232 -
        if channels:
233 -
            channels = set(channels)
234 -
            chunks = ((c, t) for (c, t) in chunks if c in channels)
235 -
        if onlyText:
236 -
            chunks = (t for (c, t) in chunks)
237 -
        return chunks
152 +
def get_factory_from_step_or_factory(step_or_factory):
153 +
    if hasattr(step_or_factory, 'get_step_factory'):
154 +
        factory = step_or_factory.get_step_factory()
155 +
    else:
156 +
        factory = step_or_factory
157 +
    # make sure the returned value actually implements IBuildStepFactory
158 +
    return interfaces.IBuildStepFactory(factory)
238 159
239 -
    def isFinished(self):
240 -
        return self.finished
241 160
242 -
    def waitUntilFinished(self):
243 -
        d = defer.Deferred()
244 -
        self.finishDeferreds.append(d)
245 -
        self._maybeFinished()
246 -
247 -
248 -
class BuildStepStatus:
249 -
    # used only for old-style steps
250 -
    pass
161 +
def create_step_from_step_or_factory(step_or_factory):
162 +
    return get_factory_from_step_or_factory(step_or_factory).buildStep()
251 163
252 164
253 165
@implementer(interfaces.IBuildStep)
254 166
class BuildStep(results.ResultComputingConfigMixin,
255 167
                properties.PropertiesMixin,
256 168
                util.ComparableMixin):
169 +
    # Note that the BuildStep is at the same time a template from which per-build steps are
170 +
    # constructed. This works by creating a new IBuildStepFactory in __new__, retrieving it via
171 +
    # get_step_factory() and then calling buildStep() on that factory.
257 172
258 173
    alwaysRun = False
259 174
    doStepIf = True
@@ -317,9 +232,6 @@
Loading
317 232
    _workdir = None
318 233
    _waitingForLocks = False
319 234
320 -
    def _run_finished_hook(self):
321 -
        return None  # override in tests
322 -
323 235
    def __init__(self, **kwargs):
324 236
        self.worker = None
325 237
@@ -421,11 +333,10 @@
Loading
421 333
    def workdir(self, workdir):
422 334
        self._workdir = workdir
423 335
424 -
    def addFactoryArguments(self, **kwargs):
425 -
        # this is here for backwards compatibility
426 -
        pass
336 +
    def getProperties(self):
337 +
        return self.build.getProperties()
427 338
428 -
    def _getStepFactory(self):
339 +
    def get_step_factory(self):
429 340
        return self._factory
430 341
431 342
    def setupProgress(self):
@@ -498,9 +409,6 @@
Loading
498 409
            buildResult = summary.get('build', None)
499 410
            if buildResult and not isinstance(buildResult, str):
500 411
                raise TypeError("build result string must be unicode")
501 -
    # updateSummary gets patched out for old-style steps, so keep a copy we can
502 -
    # call internally for such steps
503 -
    realUpdateSummary = updateSummary
504 412
505 413
    @defer.inlineCallbacks
506 414
    def addStep(self):
@@ -558,7 +466,7 @@
Loading
558 466
            yield defer.gatherResults(dl)
559 467
            self.rendered = True
560 468
            # we describe ourselves only when renderables are interpolated
561 -
            self.realUpdateSummary()
469 +
            self.updateSummary()
562 470
563 471
            # check doStepIf (after rendering)
564 472
            if isinstance(self.doStepIf, bool):
@@ -615,8 +523,6 @@
Loading
615 523
                self.results = EXCEPTION
616 524
                hidden = False
617 525
618 -
        yield self.master.data.updates.finishStep(self.stepid, self.results,
619 -
                                                  hidden)
620 526
        # perform final clean ups
621 527
        success = yield self._cleanup_logs()
622 528
        if not success:
@@ -624,16 +530,23 @@
Loading
624 530
625 531
        # update the summary one last time, make sure that completes,
626 532
        # and then don't update it any more.
627 -
        self.realUpdateSummary()
628 -
        yield self.realUpdateSummary.stop()
533 +
        self.updateSummary()
534 +
        yield self.updateSummary.stop()
629 535
630 536
        for sub in self._test_result_submitters.values():
631 537
            yield sub.finish()
632 538
633 539
        self.releaseLocks()
634 540
541 +
        yield self.master.data.updates.finishStep(self.stepid, self.results,
542 +
                                                  hidden)
543 +
635 544
        return self.results
636 545
546 +
    def setBuildData(self, name, value, source):
547 +
        # returns a Deferred that yields nothing
548 +
        return self.master.data.updates.setBuildData(self.build.buildid, name, value, source)
549 +
637 550
    @defer.inlineCallbacks
638 551
    def _cleanup_logs(self):
639 552
        all_success = True
@@ -693,92 +606,17 @@
Loading
693 606
        self._waitingForLocks = False
694 607
        return defer.succeed(None)
695 608
696 -
    @defer.inlineCallbacks
697 609
    def run(self):
698 -
        self._start_deferred = defer.Deferred()
699 -
        unhandled = self._start_unhandled_deferreds = []
700 -
        self._sync_addlog_deferreds = []
701 -
        try:
702 -
            # here's where we set things up for backward compatibility for
703 -
            # old-style steps, using monkey patches so that new-style steps
704 -
            # aren't bothered by any of this equipment
705 -
706 -
            # monkey-patch self.step_status.{setText,setText2} back into
707 -
            # existence for old steps, signalling an update to the summary
708 -
            self.step_status = BuildStepStatus()
709 -
            self.step_status.setText = lambda text: self.realUpdateSummary()
710 -
            self.step_status.setText2 = lambda text: self.realUpdateSummary()
711 -
712 -
            # monkey-patch in support for old statistics functions
713 -
            self.step_status.setStatistic = self.setStatistic
714 -
            self.step_status.getStatistic = self.getStatistic
715 -
            self.step_status.hasStatistic = self.hasStatistic
716 -
717 -
            # monkey-patch an addLog that returns an write-only, sync log
718 -
            self.addLog = self.addLog_oldStyle
719 -
            self._logFileWrappers = {}
720 -
721 -
            # and a getLog that returns a read-only, sync log, captured by
722 -
            # LogObservers installed by addLog_oldStyle
723 -
            self.getLog = self.getLog_oldStyle
724 -
725 -
            # old-style steps shouldn't be calling updateSummary
726 -
            def updateSummary():
727 -
                assert 0, 'updateSummary is only valid on new-style steps'
728 -
            self.updateSummary = updateSummary
729 -
730 -
            results = yield self.start()
731 -
            if results is not None:
732 -
                self._start_deferred.callback(results)
733 -
            results = yield self._start_deferred
734 -
        finally:
735 -
            # hook for tests
736 -
            # assert so that it is only run in non optimized mode
737 -
            assert self._run_finished_hook() is None
738 -
            # wait until all the sync logs have been actually created before
739 -
            # finishing
740 -
            yield defer.DeferredList(self._sync_addlog_deferreds,
741 -
                                     consumeErrors=True)
742 -
            self._start_deferred = None
743 -
            unhandled = self._start_unhandled_deferreds
744 -
            self.realUpdateSummary()
745 -
746 -
            # Wait for any possibly-unhandled deferreds.  If any fail, change the
747 -
            # result to EXCEPTION and log.
748 -
            while unhandled:
749 -
                self._start_unhandled_deferreds = []
750 -
                unhandled_results = yield defer.DeferredList(unhandled,
751 -
                                                             consumeErrors=True)
752 -
                for success, res in unhandled_results:
753 -
                    if not success:
754 -
                        log.err(
755 -
                            res, "from an asynchronous method executed in an old-style step")
756 -
                        results = EXCEPTION
757 -
                unhandled = self._start_unhandled_deferreds
758 -
759 -
        return results
760 -
761 -
    def finished(self, results):
762 -
        assert self._start_deferred, \
763 -
            "finished() can only be called from old steps implementing start()"
764 -
        self._start_deferred.callback(results)
765 -
766 -
    def failed(self, why):
767 -
        assert self._start_deferred, \
768 -
            "failed() can only be called from old steps implementing start()"
769 -
        self._start_deferred.errback(why)
610 +
        raise NotImplementedError("A custom build step must implement run()")
770 611
771 612
    def isNewStyle(self):
772 -
        # **temporary** method until new-style steps are the only supported style
773 -
        return self.run.__func__ is not BuildStep.run
774 -
775 -
    def start(self):
776 -
        # New-style classes implement 'run'.
777 -
        # Old-style classes implemented 'start'. Advise them to do 'run'
778 -
        # instead.
779 -
        raise NotImplementedError("your subclass must implement run()")
613 +
        warn_deprecated('3.0.0', 'BuildStep.isNewStyle() always returns True')
614 +
        return True
780 615
616 +
    @defer.inlineCallbacks
781 617
    def interrupt(self, reason):
618 +
        if self.stopped:
619 +
            return
782 620
        self.stopped = True
783 621
        if self._acquiringLocks:
784 622
            for (lock, access, d) in self._acquiringLocks:
@@ -786,14 +624,15 @@
Loading
786 624
            self._acquiringLocks = []
787 625
788 626
        if self._waitingForLocks:
789 -
            self.addCompleteLog(
627 +
            yield self.addCompleteLog(
790 628
                'cancelled while waiting for locks', str(reason))
791 629
        else:
792 -
            self.addCompleteLog('cancelled', str(reason))
630 +
            yield self.addCompleteLog('cancelled', str(reason))
793 631
794 632
        if self.cmd:
795 633
            d = self.cmd.interrupt(reason)
796 634
            d.addErrback(log.err, 'while cancelling command')
635 +
            yield d
797 636
798 637
    def releaseLocks(self):
799 638
        log.msg("releaseLocks({}): {}".format(self, self.locks))
@@ -836,26 +675,10 @@
Loading
836 675
        def newLog(logid):
837 676
            return self._newLog(name, type, logid, logEncoding)
838 677
        return d
839 -
    addLog_newStyle = addLog
840 -
841 -
    def addLog_oldStyle(self, name, type='s', logEncoding=None):
842 -
        # create a logfile instance that acts like old-style status logfiles
843 -
        # begin to create a new-style logfile
844 -
        loog_d = self.addLog_newStyle(name, type, logEncoding)
845 -
        self._start_unhandled_deferreds.append(loog_d)
846 -
        # and wrap the deferred that will eventually fire with that logfile
847 -
        # into a write-only logfile instance
848 -
        wrapper = SyncLogFileWrapper(self, name, loog_d)
849 -
        self._logFileWrappers[name] = wrapper
850 -
        return wrapper
851 678
852 679
    def getLog(self, name):
853 680
        return self.logs[name]
854 681
855 -
    def getLog_oldStyle(self, name):
856 -
        return self._logFileWrappers[name]
857 -
858 -
    @_maybeUnhandled
859 682
    @defer.inlineCallbacks
860 683
    def addCompleteLog(self, name, text):
861 684
        if self.stepid is None:
@@ -866,7 +689,6 @@
Loading
866 689
        yield _log.addContent(text)
867 690
        yield _log.finish()
868 691
869 -
    @_maybeUnhandled
870 692
    @defer.inlineCallbacks
871 693
    def addHTMLLog(self, name, html):
872 694
        if self.stepid is None:
@@ -912,7 +734,6 @@
Loading
912 734
                observer.setLog(self.logs[logname])
913 735
                self._pendingLogObservers.remove((logname, observer))
914 736
915 -
    @_maybeUnhandled
916 737
    @defer.inlineCallbacks
917 738
    def addURL(self, name, url):
918 739
        yield self.master.data.updates.addStepURL(self.stepid, str(name), str(url))
@@ -920,6 +741,9 @@
Loading
920 741
921 742
    @defer.inlineCallbacks
922 743
    def runCommand(self, command):
744 +
        if self.stopped:
745 +
            return CANCELLED
746 +
923 747
        self.cmd = command
924 748
        command.worker = self.worker
925 749
        try:
@@ -940,191 +764,6 @@
Loading
940 764
    def setStatistic(self, name, value):
941 765
        self.statistics[name] = value
942 766
943 -
    def _describe(self, done=False):
944 -
        # old-style steps expect this function to exist
945 -
        assert not self.isNewStyle()
946 -
        return []
947 -
948 -
    def describe(self, done=False):
949 -
        # old-style steps expect this function to exist
950 -
        assert not self.isNewStyle()
951 -
        desc = self._describe(done)
952 -
        if not desc:
953 -
            return []
954 -
        if self.descriptionSuffix:
955 -
            desc += self.descriptionSuffix
956 -
        return desc
957 -
958 -
    def warn_deprecated_if_oldstyle_subclass(self, name):
959 -
        if self.__class__.__name__ != name:
960 -
            warn_deprecated('2.9.0', ('Subclassing old-style step {0} in {1} is deprecated, '
961 -
                                      'please migrate to new-style equivalent {0}NewStyle'
962 -
                                      ).format(name, self.__class__.__name__))
963 -
964 -
965 -
components.registerAdapter(
966 -
    BuildStep._getStepFactory,
967 -
    BuildStep, interfaces.IBuildStepFactory)
968 -
components.registerAdapter(
969 -
    lambda step: interfaces.IProperties(step.build),
970 -
    BuildStep, interfaces.IProperties)
971 -
972 -
973 -
class LoggingBuildStep(BuildStep):
974 -
975 -
    progressMetrics = ('output',)
976 -
    logfiles = {}
977 -
978 -
    parms = BuildStep.parms + ['logfiles', 'lazylogfiles', 'log_eval_func']
979 -
    cmd = None
980 -
981 -
    renderables = ['logfiles', 'lazylogfiles']
982 -
983 -
    def __init__(self, logfiles=None, lazylogfiles=False, log_eval_func=None,
984 -
                 *args, **kwargs):
985 -
        super().__init__(*args, **kwargs)
986 -
987 -
        if logfiles is None:
988 -
            logfiles = {}
989 -
        if logfiles and not isinstance(logfiles, dict):
990 -
            config.error(
991 -
                "the ShellCommand 'logfiles' parameter must be a dictionary")
992 -
993 -
        # merge a class-level 'logfiles' attribute with one passed in as an
994 -
        # argument
995 -
        self.logfiles = self.logfiles.copy()
996 -
        self.logfiles.update(logfiles)
997 -
        self.lazylogfiles = lazylogfiles
998 -
        if log_eval_func and not callable(log_eval_func):
999 -
            config.error(
1000 -
                "the 'log_eval_func' parameter must be a callable")
1001 -
        self.log_eval_func = log_eval_func
1002 -
        self.addLogObserver('stdio', OutputProgressObserver("output"))
1003 -
1004 -
    def isNewStyle(self):
1005 -
        # LoggingBuildStep subclasses are never new-style
1006 -
        return False
1007 -
1008 -
    def addLogFile(self, logname, filename):
1009 -
        self.logfiles[logname] = filename
1010 -
1011 -
    def buildCommandKwargs(self):
1012 -
        kwargs = dict()
1013 -
        kwargs['logfiles'] = self.logfiles
1014 -
        return kwargs
1015 -
1016 -
    def startCommand(self, cmd, errorMessages=None):
1017 -
        if errorMessages is None:
1018 -
            errorMessages = []
1019 -
        log.msg("ShellCommand.startCommand(cmd={})".format(cmd))
1020 -
        log.msg("  cmd.args = %r" % (cmd.args))
1021 -
        self.cmd = cmd  # so we can interrupt it
1022 -
1023 -
        # stdio is the first log
1024 -
        self.stdio_log = stdio_log = self.addLog("stdio")
1025 -
        cmd.useLog(stdio_log, closeWhenFinished=True)
1026 -
        for em in errorMessages:
1027 -
            stdio_log.addHeader(em)
1028 -
            # TODO: consider setting up self.stdio_log earlier, and have the
1029 -
            # code that passes in errorMessages instead call
1030 -
            # self.stdio_log.addHeader() directly.
1031 -
1032 -
        # there might be other logs
1033 -
        self.setupLogfiles(cmd, self.logfiles)
1034 -
1035 -
        d = self.runCommand(cmd)  # might raise ConnectionLost
1036 -
        d.addCallback(lambda res: self.commandComplete(cmd))
1037 -
1038 -
        # TODO: when the status.LogFile object no longer exists, then this
1039 -
        # method will a synthetic logfile for old-style steps, and to be called
1040 -
        # without the `logs` parameter for new-style steps.  Unfortunately,
1041 -
        # lots of createSummary methods exist, but don't look at the log, so
1042 -
        # it's difficult to optimize when the synthetic logfile is needed.
1043 -
        d.addCallback(lambda res: self.createSummary(cmd.logs['stdio']))
1044 -
1045 -
        d.addCallback(lambda res: self.evaluateCommand(cmd))  # returns results
1046 -
1047 -
        @d.addCallback
1048 -
        def _gotResults(results):
1049 -
            self.setStatus(cmd, results)
1050 -
            return results
1051 -
        d.addCallback(self.finished)
1052 -
        d.addErrback(self.failed)
1053 -
1054 -
    def setupLogfiles(self, cmd, logfiles):
1055 -
        for logname, remotefilename in logfiles.items():
1056 -
            if self.lazylogfiles:
1057 -
                # Ask RemoteCommand to watch a logfile, but only add
1058 -
                # it when/if we see any data.
1059 -
                #
1060 -
                # The dummy default argument local_logname is a work-around for
1061 -
                # Python name binding; default values are bound by value, but
1062 -
                # captured variables in the body are bound by name.
1063 -
                def callback(cmd_arg, local_logname=logname):
1064 -
                    return self.addLog(local_logname)
1065 -
                cmd.useLogDelayed(logname, callback, True)
1066 -
            else:
1067 -
                # add a LogFile
1068 -
                newlog = self.addLog(logname)
1069 -
                # and tell the RemoteCommand to feed it
1070 -
                cmd.useLog(newlog, True)
1071 -
1072 -
    def checkDisconnect(self, f):
1073 -
        # this is now handled by self.failed
1074 -
        log.msg("WARNING: step %s uses deprecated checkDisconnect method")
1075 -
        return f
1076 -
1077 -
    def commandComplete(self, cmd):
1078 -
        pass
1079 -
1080 -
    def createSummary(self, stdio):
1081 -
        pass
1082 -
1083 -
    def evaluateCommand(self, cmd):
1084 -
        # NOTE: log_eval_func is undocumented, and will die with
1085 -
        # LoggingBuildStep/ShellCOmmand
1086 -
        if self.log_eval_func:
1087 -
            # self.step_status probably doesn't have the desired behaviors, but
1088 -
            # those were never well-defined..
1089 -
            return self.log_eval_func(cmd, self.step_status)
1090 -
        return cmd.results()
1091 -
1092 -
    # TODO: delete
1093 -
    def getText(self, cmd, results):
1094 -
        if results == SUCCESS:
1095 -
            return self.describe(True)
1096 -
        elif results == WARNINGS:
1097 -
            return self.describe(True) + ["warnings"]
1098 -
        elif results == EXCEPTION:
1099 -
            return self.describe(True) + ["exception"]
1100 -
        elif results == CANCELLED:
1101 -
            return self.describe(True) + ["cancelled"]
1102 -
        return self.describe(True) + ["failed"]
1103 -
1104 -
    # TODO: delete
1105 -
    def getText2(self, cmd, results):
1106 -
        return [self.name]
1107 -
1108 -
    # TODO: delete
1109 -
    def maybeGetText2(self, cmd, results):
1110 -
        if results == SUCCESS:
1111 -
            # successful steps do not add anything to the build's text
1112 -
            pass
1113 -
        elif results == WARNINGS:
1114 -
            if (self.flunkOnWarnings or self.warnOnWarnings):
1115 -
                # we're affecting the overall build, so tell them why
1116 -
                return self.getText2(cmd, results)
1117 -
        else:
1118 -
            if (self.haltOnFailure or self.flunkOnFailure or
1119 -
                    self.warnOnFailure):
1120 -
                # we're affecting the overall build, so tell them why
1121 -
                return self.getText2(cmd, results)
1122 -
        return []
1123 -
1124 -
    def setStatus(self, cmd, results):
1125 -
        self.realUpdateSummary()
1126 -
        return defer.succeed(None)
1127 -
1128 767
1129 768
class CommandMixin:
1130 769
@@ -1199,8 +838,6 @@
Loading
1199 838
    renderables = _shellMixinArgs
1200 839
1201 840
    def setupShellMixin(self, constructorArgs, prohibitArgs=None):
1202 -
        assert self.isNewStyle(
1203 -
        ), "ShellMixin is only compatible with new-style steps"
1204 841
        constructorArgs = constructorArgs.copy()
1205 842
1206 843
        if prohibitArgs is None:
@@ -1307,31 +944,6 @@
Loading
1307 944
            return {'step': summary}
1308 945
        return super().getResultSummary()
1309 946
1310 -
# Parses the logs for a list of regexs. Meant to be invoked like:
1311 -
# regexes = ((re.compile(...), FAILURE), (re.compile(...), WARNINGS))
1312 -
# self.addStep(ShellCommand,
1313 -
#   command=...,
1314 -
#   ...,
1315 -
#   log_eval_func=lambda c,s: regex_log_evaluator(c, s, regexs)
1316 -
# )
1317 -
# NOTE: log_eval_func is undocumented, and will die with
1318 -
# LoggingBuildStep/ShellCOmmand
1319 -
1320 -
1321 -
def regex_log_evaluator(cmd, _, regexes):
1322 -
    worst = cmd.results()
1323 -
    for err, possible_status in regexes:
1324 -
        # worst_status returns the worse of the two status' passed to it.
1325 -
        # we won't be changing "worst" unless possible_status is worse than it,
1326 -
        # so we don't even need to check the log if that's the case
1327 -
        if worst_status(worst, possible_status) == possible_status:
1328 -
            if isinstance(err, str):
1329 -
                err = re.compile(".*{}.*".format(err), re.DOTALL)
1330 -
            for l in cmd.logs.values():
1331 -
                if err.search(l.getText()):
1332 -
                    worst = possible_status
1333 -
    return worst
1334 -
1335 947
1336 948
_hush_pyflakes = [WithProperties]
1337 949
del _hush_pyflakes

@@ -218,6 +218,11 @@
Loading
218 218
        ['quiet', 'q', "Don't display log messages about reconfiguration"],
219 219
    ]
220 220
221 +
    optParameters = [
222 +
        ['progress_timeout', None, None,
223 +
         'The amount of time the script waits for messages in the logs that indicate progress.'],
224 +
    ]
225 +
221 226
    def getSynopsis(self):
222 227
        return "Usage:    buildbot reconfig [<basedir>]"
223 228
@@ -651,6 +656,16 @@
Loading
651 656
        return "Usage:   buildbot dataspec [options]"
652 657
653 658
659 +
class GenGraphQLOption(base.BasedirMixin, base.SubcommandOptions):
660 +
    subcommandFunction = "buildbot.scripts.gengraphql.gengraphql"
661 +
    optParameters = [
662 +
        ['out', 'o', "graphql.schema", "output to specified path"],
663 +
    ]
664 +
665 +
    def getSynopsis(self):
666 +
        return "Usage:   buildbot graphql-schema [options]"
667 +
668 +
654 669
class DevProxyOptions(base.BasedirMixin, base.SubcommandOptions):
655 670
656 671
    """Run a fake web server serving the local ui frontend and a distant rest and websocket api.
@@ -737,6 +752,8 @@
Loading
737 752
        ['dev-proxy', None, DevProxyOptions,
738 753
         "Run a fake web server serving the local ui frontend and a distant rest and websocket api."
739 754
         ],
755 +
        ['graphql-schema', None, GenGraphQLOption,
756 +
         "Output graphql api schema"],
740 757
        ['cleanupdb', None, CleanupDBOptions,
741 758
         "cleanup the database"
742 759
         ]

@@ -1,3 +1,4 @@
Loading
1 +
1 2
# This file is part of Buildbot.  Buildbot is free software: you can
2 3
# redistribute it and/or modify it under the terms of the GNU General Public
3 4
# License as published by the Free Software Foundation, version 2.
@@ -17,7 +18,6 @@
Loading
17 18
from random import randint
18 19
19 20
from twisted.internet import defer
20 -
from twisted.internet import task
21 21
from twisted.python import log
22 22
23 23
_poller_instances = None
@@ -25,69 +25,107 @@
Loading
25 25
26 26
class Poller:
27 27
28 -
    __slots__ = ['fn', 'instance', 'loop', 'started', 'running',
29 -
                 'pending', 'stopDeferreds', '_reactor']
30 -
31 28
    def __init__(self, fn, instance, reactor):
32 29
        self.fn = fn
33 30
        self.instance = instance
34 -
        self.loop = None
35 -
        self.started = False
31 +
36 32
        self.running = False
37 33
        self.pending = False
38 -
        self.stopDeferreds = []
34 +
35 +
        # Invariants:
36 +
        #   - If self._call is not None or self._currently_executing then it is guaranteed that
37 +
        #     self.pending and self._run_complete_deferreds will be handled at some point in the
38 +
        #     future.
39 +
        #   - If self._call is not None then _run will be executed at some point, but it's not being
40 +
        #     executed now.
41 +
        self._currently_executing = False
42 +
        self._call = None
43 +
        self._next_call_time = None  # valid when self._call is not None
44 +
45 +
        self._start_time = 0
46 +
        self._interval = 0
47 +
        self._random_delay_min = 0
48 +
        self._random_delay_max = 0
49 +
        self._run_complete_deferreds = []
50 +
39 51
        self._reactor = reactor
40 52
41 53
    @defer.inlineCallbacks
42 -
    def _run(self, random_delay_min=0, random_delay_max=0):
43 -
        self.running = True
44 -
        if random_delay_max:
45 -
            yield task.deferLater(self._reactor, randint(random_delay_min, random_delay_max),
46 -
                                  lambda: None)
54 +
    def _run(self):
55 +
        self._call = None
56 +
        self._currently_executing = True
57 +
47 58
        try:
48 59
            yield self.fn(self.instance)
49 60
        except Exception as e:
50 -
            log.err(e, 'while running {}'.format(self.fn))
61 +
            log.err(e, 'while executing {}'.format(self.fn))
62 +
        finally:
63 +
            self._currently_executing = False
51 64
52 -
        self.running = False
53 -
        # loop if there's another pending call
54 -
        if self.pending:
55 -
            self.pending = False
56 -
            yield self._run(random_delay_min, random_delay_max)
65 +
        was_pending = self.pending
66 +
        self.pending = False
67 +
68 +
        if self.running:
69 +
            self._schedule(force_now=was_pending)
70 +
71 +
        while self._run_complete_deferreds:
72 +
            self._run_complete_deferreds.pop(0).callback(None)
73 +
74 +
    def _get_wait_time(self, curr_time, force_now=False, force_initial_now=False):
75 +
        if force_now:
76 +
            return 0
77 +
78 +
        extra_wait = randint(self._random_delay_min, self._random_delay_max)
79 +
80 +
        if force_initial_now or self._interval == 0:
81 +
            return extra_wait
82 +
83 +
        # note that differently from twisted.internet.task.LoopingCall, we don't care about
84 +
        # floating-point precision issues as we don't have the withCount feature.
85 +
        running_time = curr_time - self._start_time
86 +
        return self._interval - (running_time % self._interval) + extra_wait
87 +
88 +
    def _schedule(self, force_now=False, force_initial_now=False):
89 +
        curr_time = self._reactor.seconds()
90 +
        wait_time = self._get_wait_time(curr_time, force_now=force_now,
91 +
                                        force_initial_now=force_initial_now)
92 +
        next_call_time = curr_time + wait_time
93 +
94 +
        if self._call is not None:
95 +
            # Note that self._call can ever be moved to earlier time, so we can always cancel it.
96 +
            self._call.cancel()
97 +
98 +
        self._next_call_time = next_call_time
99 +
        self._call = self._reactor.callLater(wait_time, self._run)
57 100
58 101
    def __call__(self):
59 -
        if self.started:
60 -
            if self.running:
61 -
                self.pending = True
62 -
            else:
63 -
                # terrible hack..
64 -
                old_interval = self.loop.interval
65 -
                self.loop.interval = 0
66 -
                self.loop.reset()
67 -
                self.loop.interval = old_interval
102 +
        if not self.running:
103 +
            return
104 +
        if self._currently_executing:
105 +
            self.pending = True
106 +
        else:
107 +
            self._schedule(force_now=True)
68 108
69 109
    def start(self, interval, now=False, random_delay_min=0, random_delay_max=0):
70 -
        assert not self.started
71 -
        if not self.loop:
72 -
            self.loop = task.LoopingCall(self._run, random_delay_min, random_delay_max)
73 -
            self.loop.clock = self._reactor
74 -
        stopDeferred = self.loop.start(interval, now=now)
75 -
76 -
        @stopDeferred.addCallback
77 -
        def inform(_):
78 -
            self.started = False
79 -
            while self.stopDeferreds:
80 -
                self.stopDeferreds.pop().callback(None)
81 -
        self.started = True
110 +
        assert not self.running
111 +
        self._interval = interval
112 +
        self._random_delay_min = random_delay_min
113 +
        self._random_delay_max = random_delay_max
114 +
        self._start_time = self._reactor.seconds()
115 +
116 +
        self.running = True
117 +
        self._schedule(force_initial_now=now)
82 118
119 +
    @defer.inlineCallbacks
83 120
    def stop(self):
84 -
        if self.loop and self.loop.running:
85 -
            self.loop.stop()
86 -
        if self.started:
121 +
        self.running = False
122 +
        if self._call is not None:
123 +
            self._call.cancel()
124 +
            self._call = None
125 +
        if self._currently_executing:
87 126
            d = defer.Deferred()
88 -
            self.stopDeferreds.append(d)
89 -
            return d
90 -
        return defer.succeed(None)
127 +
            self._run_complete_deferreds.append(d)
128 +
            yield d
91 129
92 130
93 131
class _Descriptor:
94 132
imilarity index 54%
95 133
ename from master/buildbot/status/client.py
96 134
ename to master/buildbot/util/pullrequest.py

@@ -16,7 +16,6 @@
Loading
16 16
17 17
import os
18 18
import sys
19 -
import textwrap
20 19
21 20
from twisted.internet import protocol
22 21
from twisted.internet import reactor
@@ -127,38 +126,10 @@
Loading
127 126
            pidfile.write("{0}".format(proc.pid))
128 127
129 128
130 -
def py2Warning(config):
131 -
    if sys.version[0] == '2' and not config['quiet']:
132 -
        print(textwrap.dedent("""\
133 -
        WARNING: You are running Buildbot with Python 2.7.x !
134 -
        -----------------------------------------------------
135 -
136 -
        Python 2 is going unmaintained as soon as 2020: https://pythonclock.org/
137 -
138 -
        To prepare for that transition, we recommend upgrading your buildmaster to run on
139 -
        Python 3.6 now! Buildbot open source project is as well deprecating running buildmaster
140 -
        on Python 2 for better maintainability.
141 -
142 -
        Buildbot 2.0 going to be released in February 2019 will remove support for Python < 3.5
143 -
        https://github.com/buildbot/buildbot/issues/4439
144 -
145 -
        On most installations, switching to Python 3 can be accomplished by running the 2to3 tool
146 -
        over the master.cfg file.
147 -
148 -
        https://docs.python.org/3.7/library/2to3.html
149 -
150 -
        Note that the above applies only for the buildmaster.
151 -
        Workers will still support running under Python 2.7.
152 -
        Additionally, the buildmaster still supports workers using old versions of Buildbot.
153 -
        """))
154 -
155 -
156 129
def start(config):
157 130
    if not base.isBuildmasterDir(config['basedir']):
158 131
        return 1
159 132
160 -
    py2Warning(config)
161 -
162 133
    if config['nodaemon']:
163 134
        launchNoDaemon(config)
164 135
        return 0

@@ -218,36 +218,27 @@
Loading
218 218
            eventually(self._finished, failure)
219 219
        return None
220 220
221 -
    def _unwrap(self, log):
222 -
        from buildbot.process import buildstep
223 -
        if isinstance(log, buildstep.SyncLogFileWrapper):
224 -
            return log.unwrap()
225 -
        return log
226 -
227 221
    @util.deferredLocked('loglock')
228 -
    @defer.inlineCallbacks
229 222
    def addStdout(self, data):
230 223
        if self.collectStdout:
231 224
            self.stdout += data
232 225
        if self.stdioLogName is not None and self.stdioLogName in self.logs:
233 -
            log_ = yield self._unwrap(self.logs[self.stdioLogName])
234 -
            log_.addStdout(data)
226 +
            self.logs[self.stdioLogName].addStdout(data)
227 +
        return defer.succeed(None)
235 228
236 229
    @util.deferredLocked('loglock')
237 -
    @defer.inlineCallbacks
238 230
    def addStderr(self, data):
239 231
        if self.collectStderr:
240 232
            self.stderr += data
241 233
        if self.stdioLogName is not None and self.stdioLogName in self.logs:
242 -
            log_ = yield self._unwrap(self.logs[self.stdioLogName])
243 -
            log_.addStderr(data)
234 +
            self.logs[self.stdioLogName].addStderr(data)
235 +
        return defer.succeed(None)
244 236
245 237
    @util.deferredLocked('loglock')
246 -
    @defer.inlineCallbacks
247 238
    def addHeader(self, data):
248 239
        if self.stdioLogName is not None and self.stdioLogName in self.logs:
249 -
            log_ = yield self._unwrap(self.logs[self.stdioLogName])
250 -
            log_.addHeader(data)
240 +
            self.logs[self.stdioLogName].addHeader(data)
241 +
        return defer.succeed(None)
251 242
252 243
    @util.deferredLocked('loglock')
253 244
    @defer.inlineCallbacks
@@ -257,13 +248,11 @@
Loading
257 248
            (activateCallBack, closeWhenFinished) = self.delayedLogs[logname]
258 249
            del self.delayedLogs[logname]
259 250
            loog = yield activateCallBack(self)
260 -
            loog = yield self._unwrap(loog)
261 251
            self.logs[logname] = loog
262 252
            self._closeWhenFinished[logname] = closeWhenFinished
263 253
264 254
        if logname in self.logs:
265 -
            log_ = yield self._unwrap(self.logs[logname])
266 -
            yield log_.addStdout(data)
255 +
            yield self.logs[logname].addStdout(data)
267 256
        else:
268 257
            log.msg("{}.addToLog: no such log {}".format(self, logname))
269 258

@@ -24,12 +24,13 @@
Loading
24 24
from buildbot.process.properties import Interpolate
25 25
from buildbot.process.properties import Properties
26 26
from buildbot.process.results import SUCCESS
27 -
from buildbot.reporters import http
28 -
from buildbot.reporters import notifier
27 +
from buildbot.reporters.base import ReporterBase
28 +
from buildbot.reporters.generators.build import BuildStartEndStatusGenerator
29 +
from buildbot.reporters.generators.build import BuildStatusGenerator
30 +
from buildbot.reporters.message import MessageFormatterRenderable
29 31
from buildbot.util import bytes2unicode
30 32
from buildbot.util import httpclientservice
31 33
from buildbot.util import unicode2bytes
32 -
from buildbot.warnings import warn_deprecated
33 34
34 35
from .utils import merge_reports_prop
35 36
@@ -44,30 +45,46 @@
Loading
44 45
HTTP_CREATED = 201
45 46
46 47
47 -
class BitbucketServerStatusPush(http.HttpStatusPushBase):
48 +
class BitbucketServerStatusPush(ReporterBase):
48 49
    name = "BitbucketServerStatusPush"
49 50
50 -
    def checkConfig(self, base_url, user, password, key=None, statusName=None,
51 -
                    startDescription=None, endDescription=None, verbose=False,
52 -
                    **kwargs):
53 -
        super().checkConfig(wantProperties=True, _has_old_arg_names={'wantProperties': False},
54 -
                            **kwargs)
51 +
    def checkConfig(self, base_url, user, password, key=None, statusName=None, verbose=False,
52 +
                    debug=None, verify=None, generators=None, **kwargs):
53 +
54 +
        if generators is None:
55 +
            generators = self._create_default_generators()
56 +
57 +
        super().checkConfig(generators=generators, **kwargs)
58 +
        httpclientservice.HTTPClientService.checkAvailable(self.__class__.__name__)
55 59
56 60
    @defer.inlineCallbacks
57 -
    def reconfigService(self, base_url, user, password, key=None,
58 -
                        statusName=None, startDescription=None,
59 -
                        endDescription=None, verbose=False, **kwargs):
61 +
    def reconfigService(self, base_url, user, password, key=None, statusName=None, verbose=False,
62 +
                        debug=None, verify=None, generators=None, **kwargs):
60 63
        user, password = yield self.renderSecrets(user, password)
61 -
        yield super().reconfigService(wantProperties=True, **kwargs)
64 +
        self.debug = debug
65 +
        self.verify = verify
66 +
        self.verbose = verbose
67 +
68 +
        if generators is None:
69 +
            generators = self._create_default_generators()
70 +
71 +
        yield super().reconfigService(generators=generators, **kwargs)
72 +
62 73
        self.key = key or Interpolate('%(prop:buildername)s')
63 74
        self.context = statusName
64 -
        self.endDescription = endDescription or 'Build done.'
65 -
        self.startDescription = startDescription or 'Build started.'
66 -
        self.verbose = verbose
67 75
        self._http = yield httpclientservice.HTTPClientService.getService(
68 76
            self.master, base_url, auth=(user, password),
69 77
            debug=self.debug, verify=self.verify)
70 78
79 +
    def _create_default_generators(self):
80 +
        start_formatter = MessageFormatterRenderable('Build started.')
81 +
        end_formatter = MessageFormatterRenderable('Build done.')
82 +
83 +
        return [
84 +
            BuildStartEndStatusGenerator(start_formatter=start_formatter,
85 +
                                         end_formatter=end_formatter)
86 +
        ]
87 +
71 88
    def createStatus(self, sha, state, url, key, description=None, context=None):
72 89
        payload = {
73 90
            'state': state,
@@ -82,36 +99,23 @@
Loading
82 99
83 100
        return self._http.post(STATUS_API_URL.format(sha=sha), json=payload)
84 101
85 -
    @defer.inlineCallbacks
86 -
    def send(self, build):
87 -
        # the only case when this function is called is when the user derives this class, overrides
88 -
        # send() and calls super().send(build) from there.
89 -
        yield self._send_impl(build)
90 -
91 102
    @defer.inlineCallbacks
92 103
    def sendMessage(self, reports):
104 +
        report = reports[0]
93 105
        build = reports[0]['builds'][0]
94 -
        if self.send.__func__ is not BitbucketServerStatusPush.send:
95 -
            warn_deprecated('2.9.0', 'send() in reporters has been deprecated. Use sendMessage()')
96 -
            yield self.send(build)
97 -
        else:
98 -
            yield self._send_impl(build)
99 106
100 -
    @defer.inlineCallbacks
101 -
    def _send_impl(self, build):
102 107
        props = Properties.fromDict(build['properties'])
103 108
        props.master = self.master
104 109
110 +
        description = report.get('body', None)
111 +
105 112
        results = build['results']
106 113
        if build['complete']:
107 114
            state = SUCCESSFUL if results == SUCCESS else FAILED
108 -
            description = self.endDescription
109 115
        else:
110 116
            state = INPROGRESS
111 -
            description = self.startDescription
112 117
113 118
        key = yield props.render(self.key)
114 -
        description = yield props.render(description) if description else None
115 119
        context = yield props.render(self.context) if self.context else None
116 120
117 121
        sourcestamps = build['buildset']['sourcestamps']
@@ -151,42 +155,51 @@
Loading
151 155
                    ))
152 156
153 157
154 -
class BitbucketServerCoreAPIStatusPush(http.HttpStatusPushBase):
158 +
class BitbucketServerCoreAPIStatusPush(ReporterBase):
155 159
    name = "BitbucketServerCoreAPIStatusPush"
156 160
    secrets = ["token", "auth"]
157 161
158 162
    def checkConfig(self, base_url, token=None, auth=None,
159 -
                    statusName=None, statusSuffix=None, startDescription=None,
160 -
                    endDescription=None, key=None, parentName=None,
163 +
                    statusName=None, statusSuffix=None, key=None, parentName=None,
161 164
                    buildNumber=None, ref=None, duration=None,
162 -
                    testResults=None, verbose=False, debug=None,
163 -
                    verify=None, **kwargs):
165 +
                    testResults=None, verbose=False, debug=None, verify=None, generators=None,
166 +
                    **kwargs):
167 +
168 +
        if generators is None:
169 +
            generators = self._create_default_generators()
170 +
171 +
        super().checkConfig(generators=generators, **kwargs)
172 +
        httpclientservice.HTTPClientService.checkAvailable(self.__class__.__name__)
173 +
164 174
        if not base_url:
165 175
            config.error("Parameter base_url has to be given")
166 176
        if token is not None and auth is not None:
167 177
            config.error("Only one authentication method can be given "
168 178
                         "(token or auth)")
169 -
        super().checkConfig(wantProperties=True, _has_old_arg_names={'wantProperties': False},
170 -
                            **kwargs)
171 179
172 180
    @defer.inlineCallbacks
173 181
    def reconfigService(self, base_url, token=None, auth=None,
174 -
                        statusName=None, statusSuffix=None, startDescription=None,
175 -
                        endDescription=None, key=None, parentName=None,
182 +
                        statusName=None, statusSuffix=None, key=None, parentName=None,
176 183
                        buildNumber=None, ref=None, duration=None,
177 -
                        testResults=None, verbose=False, debug=None,
178 -
                        verify=None, **kwargs):
179 -
        yield super().reconfigService(wantProperties=True, **kwargs)
184 +
                        testResults=None, verbose=False, debug=None, verify=None, generators=None,
185 +
                        **kwargs):
180 186
        self.status_name = statusName
181 187
        self.status_suffix = statusSuffix
182 -
        self.start_description = startDescription or 'Build started.'
183 -
        self.end_description = endDescription or 'Build done.'
184 188
        self.key = key or Interpolate('%(prop:buildername)s')
185 189
        self.parent_name = parentName
186 190
        self.build_number = buildNumber or Interpolate('%(prop:buildnumber)s')
187 191
        self.ref = ref
188 192
        self.duration = duration
189 193
194 +
        self.debug = debug
195 +
        self.verify = verify
196 +
        self.verbose = verbose
197 +
198 +
        if generators is None:
199 +
            generators = self._create_default_generators()
200 +
201 +
        yield super().reconfigService(generators=generators, **kwargs)
202 +
190 203
        if testResults:
191 204
            self.test_results = testResults
192 205
        else:
@@ -204,7 +217,6 @@
Loading
204 217
                return None
205 218
            self.test_results = r_testresults
206 219
207 -
        self.verbose = verbose
208 220
        headers = {}
209 221
        if token:
210 222
            headers["Authorization"] = "Bearer {}".format(token)
@@ -212,6 +224,15 @@
Loading
212 224
            self.master, base_url, auth=auth, headers=headers, debug=debug,
213 225
            verify=verify)
214 226
227 +
    def _create_default_generators(self):
228 +
        start_formatter = MessageFormatterRenderable('Build started.')
229 +
        end_formatter = MessageFormatterRenderable('Build done.')
230 +
231 +
        return [
232 +
            BuildStartEndStatusGenerator(start_formatter=start_formatter,
233 +
                                         end_formatter=end_formatter)
234 +
        ]
235 +
215 236
    def createStatus(self, proj_key, repo_slug, sha, state, url, key, parent,
216 237
                     build_number, ref, description, name, duration,
217 238
                     test_results):
@@ -237,31 +258,20 @@
Loading
237 258
                                          sha=sha)
238 259
        return self._http.post(_url, json=payload)
239 260
240 -
    @defer.inlineCallbacks
241 -
    def send(self, build):
242 -
        # the only case when this function is called is when the user derives this class, overrides
243 -
        # send() and calls super().send(build) from there.
244 -
        yield self._send_impl(build)
245 -
246 261
    @defer.inlineCallbacks
247 262
    def sendMessage(self, reports):
263 +
        report = reports[0]
248 264
        build = reports[0]['builds'][0]
249 -
        if self.send.__func__ is not BitbucketServerCoreAPIStatusPush.send:
250 -
            warn_deprecated('2.9.0', 'send() in reporters has been deprecated. Use sendMessage()')
251 -
            yield self.send(build)
252 -
        else:
253 -
            yield self._send_impl(build)
254 265
255 -
    @defer.inlineCallbacks
256 -
    def _send_impl(self, build):
257 266
        props = Properties.fromDict(build['properties'])
258 267
        props.master = self.master
259 268
269 +
        description = report.get('body', None)
270 +
260 271
        duration = None
261 272
        test_results = None
262 273
        if build['complete']:
263 274
            state = SUCCESSFUL if build['results'] == SUCCESS else FAILED
264 -
            description = yield props.render(self.end_description)
265 275
            if self.duration:
266 276
                duration = yield props.render(self.duration)
267 277
            else:
@@ -271,7 +281,6 @@
Loading
271 281
                test_results = yield props.render(self.test_results)
272 282
        else:
273 283
            state = INPROGRESS
274 -
            description = yield props.render(self.start_description)
275 284
            duration = None
276 285
277 286
        parent_name = (build['parentbuilder'] or {}).get('name')
@@ -369,42 +378,34 @@
Loading
369 378
                    ))
370 379
371 380
372 -
class BitbucketServerPRCommentPush(notifier.NotifierBase):
381 +
class BitbucketServerPRCommentPush(ReporterBase):
373 382
    name = "BitbucketServerPRCommentPush"
374 383
375 384
    @defer.inlineCallbacks
376 -
    def reconfigService(self, base_url, user, password, messageFormatter=None,
377 -
                        verbose=False, debug=None, verify=None, **kwargs):
385 +
    def reconfigService(self, base_url, user, password,
386 +
                        verbose=False, debug=None, verify=None, generators=None, **kwargs):
378 387
        user, password = yield self.renderSecrets(user, password)
379 -
        yield super().reconfigService(
380 -
            messageFormatter=messageFormatter, watchedWorkers=None,
381 -
            messageFormatterMissingWorker=None, subject='', addLogs=False,
382 -
            addPatch=False, **kwargs)
383 388
        self.verbose = verbose
389 +
390 +
        if generators is None:
391 +
            generators = self._create_default_generators()
392 +
393 +
        yield super().reconfigService(generators=generators, **kwargs)
384 394
        self._http = yield httpclientservice.HTTPClientService.getService(
385 395
            self.master, base_url, auth=(user, password),
386 396
            debug=debug, verify=verify)
387 397
388 -
    def checkConfig(self, base_url, user, password, messageFormatter=None,
389 -
                    verbose=False, debug=None, verify=None, **kwargs):
390 -
391 -
        super().checkConfig(messageFormatter=messageFormatter,
392 -
                            watchedWorkers=None,
393 -
                            messageFormatterMissingWorker=None,
394 -
                            subject='',
395 -
                            addLogs=False,
396 -
                            addPatch=False,
397 -
                            _has_old_arg_names={'subject': False},
398 -
                            **kwargs)
399 -
400 -
    def isMessageNeeded(self, build):
401 -
        if 'pullrequesturl' in build['properties']:
402 -
            return super().isMessageNeeded(build)
403 -
        return False
404 -
405 -
    def workerMissing(self, key, worker):
406 -
        # a comment is always associated to a change
407 -
        pass
398 +
    def checkConfig(self, base_url, user, password,
399 +
                    verbose=False, debug=None, verify=None, generators=None, **kwargs):
400 +
401 +
        if generators is None:
402 +
            generators = self._create_default_generators()
403 +
404 +
        super().checkConfig(generators=generators, **kwargs)
405 +
        httpclientservice.HTTPClientService.checkAvailable(self.__class__.__name__)
406 +
407 +
    def _create_default_generators(self):
408 +
        return [BuildStatusGenerator()]
408 409
409 410
    def sendComment(self, pr_url, text):
410 411
        path = urlparse(unicode2bytes(pr_url)).path

@@ -29,7 +29,6 @@
Loading
29 29
30 30
from buildbot import config
31 31
from buildbot.interfaces import LatentWorkerFailedToSubstantiate
32 -
from buildbot.warnings import warn_deprecated
33 32
from buildbot.worker import AbstractLatentWorker
34 33
35 34
try:
@@ -268,42 +267,14 @@
Loading
268 267
            block_device_map) if block_device_map else None
269 268
270 269
    def create_block_device_mapping(self, mapping_definitions):
271 -
        if isinstance(mapping_definitions, list):
272 -
            for mapping_definition in mapping_definitions:
273 -
                ebs = mapping_definition.get('Ebs')
274 -
                if ebs:
275 -
                    ebs.setdefault('DeleteOnTermination', True)
276 -
            return mapping_definitions
277 -
278 -
        warn_deprecated(
279 -
            '0.9.0',
280 -
            "Use of dict value to 'block_device_map' of EC2LatentWorker "
281 -
            "constructor is deprecated. Please use a list matching the AWS API "
282 -
            "https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_BlockDeviceMapping.html"
283 -
        )
284 -
        return self._convert_deprecated_block_device_mapping(mapping_definitions)
285 -
286 -
    def _convert_deprecated_block_device_mapping(self, mapping_definitions):
287 -
        new_mapping_definitions = []
288 -
        for dev_name, dev_config in mapping_definitions.items():
289 -
            new_dev_config = {}
290 -
            new_dev_config['DeviceName'] = dev_name
291 -
            if dev_config:
292 -
                new_dev_config['Ebs'] = {}
293 -
                new_dev_config['Ebs']['DeleteOnTermination'] = dev_config.get(
294 -
                    'delete_on_termination', True)
295 -
                new_dev_config['Ebs'][
296 -
                    'Encrypted'] = dev_config.get('encrypted')
297 -
                new_dev_config['Ebs']['Iops'] = dev_config.get('iops')
298 -
                new_dev_config['Ebs'][
299 -
                    'SnapshotId'] = dev_config.get('snapshot_id')
300 -
                new_dev_config['Ebs']['VolumeSize'] = dev_config.get('size')
301 -
                new_dev_config['Ebs'][
302 -
                    'VolumeType'] = dev_config.get('volume_type')
303 -
                new_dev_config['Ebs'] = self._remove_none_opts(
304 -
                    new_dev_config['Ebs'])
305 -
            new_mapping_definitions.append(new_dev_config)
306 -
        return new_mapping_definitions
270 +
        if not isinstance(mapping_definitions, list):
271 +
            config.error("EC2LatentWorker: 'block_device_map' must be a list")
272 +
273 +
        for mapping_definition in mapping_definitions:
274 +
            ebs = mapping_definition.get('Ebs')
275 +
            if ebs:
276 +
                ebs.setdefault('DeleteOnTermination', True)
277 +
        return mapping_definitions
307 278
308 279
    def get_image(self):
309 280
        # pylint: disable=too-many-nested-blocks

@@ -91,6 +91,11 @@
Loading
91 91
        return results
92 92
93 93
94 +
class UrlEntityType(types.Entity):
95 +
    name = types.String()
96 +
    url = types.String()
97 +
98 +
94 99
class Step(base.ResourceType):
95 100
96 101
    name = "step"
@@ -113,10 +118,7 @@
Loading
113 118
        results = types.NoneOk(types.Integer())
114 119
        state_string = types.String()
115 120
        urls = types.List(
116 -
            of=types.Dict(
117 -
                name=types.String(),
118 -
                url=types.String()
119 -
            ))
121 +
            of=UrlEntityType("Url"))
120 122
        hidden = types.Boolean()
121 123
    entityType = EntityType(name)
122 124

@@ -14,6 +14,7 @@
Loading
14 14
# Copyright Buildbot Team Members
15 15
16 16
17 +
import base64
17 18
import json
18 19
import time
19 20
from datetime import datetime
@@ -27,18 +28,17 @@
Loading
27 28
from buildbot.util import datetime2epoch
28 29
from buildbot.util import deferredLocked
29 30
from buildbot.util import epoch2datetime
30 -
from buildbot.warnings import warn_deprecated
31 +
from buildbot.util.pullrequest import PullRequestMixin
31 32
32 -
_UNSPECIFIED = object()
33 33
34 -
35 -
class BitbucketPullrequestPoller(base.PollingChangeSource):
34 +
class BitbucketPullrequestPoller(base.PollingChangeSource, PullRequestMixin):
36 35
37 36
    compare_attrs = ("owner", "slug", "branch",
38 37
                     "pollInterval", "useTimestamps",
39 38
                     "category", "project", "pollAtLaunch")
40 39
41 40
    db_class_name = 'BitbucketPullrequestPoller'
41 +
    property_basename = "bitbucket"
42 42
43 43
    def __init__(self, owner, slug,
44 44
                 branch=None,
@@ -47,17 +47,17 @@
Loading
47 47
                 category=None,
48 48
                 project='',
49 49
                 pullrequest_filter=True,
50 -
                 encoding=_UNSPECIFIED,
51 -
                 pollAtLaunch=False
50 +
                 pollAtLaunch=False,
51 +
                 auth=None,
52 +
                 bitbucket_property_whitelist=None,
52 53
                 ):
53 -
54 54
        self.owner = owner
55 55
        self.slug = slug
56 56
        self.branch = branch
57 57
        super().__init__(name='/'.join([owner, slug]), pollInterval=pollInterval,
58 58
                         pollAtLaunch=pollAtLaunch)
59 -
        if encoding != _UNSPECIFIED:
60 -
            warn_deprecated('2.6.0', 'encoding of BitbucketPullrequestPoller is deprecated.')
59 +
        if bitbucket_property_whitelist is None:
60 +
            bitbucket_property_whitelist = []
61 61
62 62
        if hasattr(pullrequest_filter, '__call__'):
63 63
            self.pullrequest_filter = pullrequest_filter
@@ -71,6 +71,13 @@
Loading
71 71
            category) else bytes2unicode(category)
72 72
        self.project = bytes2unicode(project)
73 73
        self.initLock = defer.DeferredLock()
74 +
        self.external_property_whitelist = bitbucket_property_whitelist
75 +
76 +
        if auth is not None:
77 +
            encoded_credentials = base64.b64encode(":".join(auth).encode())
78 +
            self.headers = {b"Authorization": b"Basic " + encoded_credentials}
79 +
        else:
80 +
            self.headers = None
74 81
75 82
    def describe(self):
76 83
        return "BitbucketPullrequestPoller watching the "\
@@ -89,7 +96,7 @@
Loading
89 96
                "Bitbucket repository {}/{}, branch: {}".format(self.owner, self.slug, self.branch))
90 97
        url = "https://bitbucket.org/api/2.0/repositories/{}/{}/pullrequests".format(self.owner,
91 98
                                                                                     self.slug)
92 -
        return client.getPage(url, timeout=self.pollInterval)
99 +
        return client.getPage(url, timeout=self.pollInterval, headers=self.headers)
93 100
94 101
    @defer.inlineCallbacks
95 102
    def _processChanges(self, page):
@@ -109,7 +116,8 @@
Loading
109 116
                # compare _short_ hashes to check if the PR has been updated
110 117
                if not current or current[0:12] != revision[0:12]:
111 118
                    # parse pull request api page (required for the filter)
112 -
                    page = yield client.getPage(str(pr['links']['self']['href']))
119 +
                    page = yield client.getPage(str(pr['links']['self']['href']),
120 +
                                                headers=self.headers)
113 121
                    pr_json = json.loads(page)
114 122
115 123
                    # filter pull requests by user function
@@ -131,14 +139,18 @@
Loading
131 139
                    title = pr['title']
132 140
                    # parse commit api page
133 141
                    page = yield client.getPage(
134 -
                            str(pr['source']['commit']['links']['self']['href']))
142 +
                        str(pr['source']['commit']['links']['self']['href']),
143 +
                        headers=self.headers,
144 +
                    )
135 145
                    commit_json = json.loads(page)
136 146
                    # use the full-length hash from now on
137 147
                    revision = commit_json['hash']
138 148
                    revlink = commit_json['links']['html']['href']
139 149
                    # parse repo api page
140 150
                    page = yield client.getPage(
141 -
                            str(pr['source']['repository']['links']['self']['href']))
151 +
                        str(pr['source']['repository']['links']['self']['href']),
152 +
                        headers=self.headers,
153 +
                    )
142 154
                    repo_json = json.loads(page)
143 155
                    repo = repo_json['links']['html']['href']
144 156
@@ -156,6 +168,9 @@
Loading
156 168
                        category=self.category,
157 169
                        project=self.project,
158 170
                        repository=bytes2unicode(repo),
171 +
                        properties={'pullrequesturl': prlink,
172 +
                                    **self.extractProperties(pr),
173 +
                                    },
159 174
                        src='bitbucket',
160 175
                    )
161 176

@@ -14,14 +14,12 @@
Loading
14 14
# Copyright Buildbot Team Members
15 15
16 16
17 -
import os
18 -
19 17
import jinja2
20 18
21 19
from twisted.internet import defer
22 20
23 -
from buildbot import config
24 21
from buildbot import util
22 +
from buildbot.process.properties import Properties
25 23
from buildbot.process.results import CANCELLED
26 24
from buildbot.process.results import EXCEPTION
27 25
from buildbot.process.results import FAILURE
@@ -29,7 +27,6 @@
Loading
29 27
from buildbot.process.results import WARNINGS
30 28
from buildbot.process.results import statusToString
31 29
from buildbot.reporters import utils
32 -
from buildbot.warnings import warn_deprecated
33 30
34 31
35 32
def get_detected_status_text(mode, results, previous_results):
@@ -112,7 +109,7 @@
Loading
112 109
    return ', '.join(list(projects))
113 110
114 111
115 -
def create_context_for_build(mode, buildername, build, master, blamelist):
112 +
def create_context_for_build(mode, build, master, blamelist):
116 113
    buildset = build['buildset']
117 114
    ss_list = buildset['sourcestamps']
118 115
    results = build['results']
@@ -125,7 +122,7 @@
Loading
125 122
    return {
126 123
        'results': build['results'],
127 124
        'mode': mode,
128 -
        'buildername': buildername,
125 +
        'buildername': build['builder']['name'],
129 126
        'workername': build['properties'].get('workername', ["<unknown>"])[0],
130 127
        'buildset': buildset,
131 128
        'build': build,
@@ -167,13 +164,40 @@
Loading
167 164
    def render_message_dict(self, master, context):
168 165
        """Generate a buildbot reporter message and return a dictionary
169 166
           containing the message body, type and subject."""
167 +
168 +
        ''' This is an informal description of what message dictionaries are expected to be
169 +
            produced. It is an internal API and expected to change even within bugfix releases, if
170 +
            needed.
171 +
172 +
            The message dictionary contains the 'body', 'type' and 'subject' keys:
173 +
174 +
              - 'subject' is a string that defines a subject of the message. It's not necessarily
175 +
                used on all reporters. It may be None.
176 +
177 +
              - 'type' must be 'plain', 'html' or 'json'.
178 +
179 +
              - 'body' is the content of the message. It may be None. The type of the data depends
180 +
                on the value of the 'type' parameter:
181 +
182 +
                - 'plain': Must be a string
183 +
184 +
                - 'html': Must be a string
185 +
186 +
                - 'json': Must be a non-encoded jsonnable value. The root element must be either
187 +
                  of dictionary, list or string. This must not change during all invocations of
188 +
                  a particular instance of the formatter.
189 +
190 +
            In case of a report being created for multiple builds (e.g. in the case of a buildset),
191 +
            the values returned by message formatter are concatenated. If this is not possible
192 +
            (e.g. if the body is a dictionary), any subsequent messages are ignored.
193 +
        '''
170 194
        yield self.buildAdditionalContext(master, context)
171 195
        context.update(self.context)
172 196
173 197
        return {
174 -
            'body': self.render_message_body(context),
198 +
            'body': (yield self.render_message_body(context)),
175 199
            'type': self.template_type,
176 -
            'subject': self.render_message_subject(context)
200 +
            'subject': (yield self.render_message_subject(context))
177 201
        }
178 202
179 203
    def render_message_body(self, context):
@@ -182,9 +206,13 @@
Loading
182 206
    def render_message_subject(self, context):
183 207
        return None
184 208
209 +
    def format_message_for_build(self, master, build, **kwargs):
210 +
        # Known kwargs keys: mode, users
211 +
        raise NotImplementedError
212 +
185 213
186 214
class MessageFormatterEmpty(MessageFormatterBase):
187 -
    def format_message_for_build(self, mode, buildername, build, master, blamelist):
215 +
    def format_message_for_build(self, master, build, **kwargs):
188 216
        return {
189 217
            'body': None,
190 218
            'type': 'plain',
@@ -192,63 +220,93 @@
Loading
192 220
        }
193 221
194 222
195 -
class DeprecatedMessageFormatterBuildJson(MessageFormatterBase):
196 -
197 -
    template_type = 'json'
223 +
class MessageFormatterFunction(MessageFormatterBase):
198 224
199 -
    def __init__(self, format_fn, **kwargs):
225 +
    def __init__(self, function, template_type, **kwargs):
200 226
        super().__init__(**kwargs)
201 -
        self.format_fn = format_fn
227 +
        self.template_type = template_type
228 +
        self._function = function
202 229
203 230
    @defer.inlineCallbacks
204 -
    def format_message_for_build(self, mode, buildername, build, master, blamelist):
231 +
    def format_message_for_build(self, master, build, **kwargs):
205 232
        msgdict = yield self.render_message_dict(master, {'build': build})
206 233
        return msgdict
207 234
208 235
    def render_message_body(self, context):
209 -
        return self.format_fn(context['build'])
236 +
        return self._function(context)
210 237
211 238
    def render_message_subject(self, context):
212 239
        return None
213 240
214 241
215 -
class MessageFormatterBaseJinja(MessageFormatterBase):
216 -
    template_filename = 'default_mail.txt'
242 +
class MessageFormatterRenderable(MessageFormatterBase):
217 243
218 -
    compare_attrs = ['body_template', 'subject_template', 'template_type']
244 +
    template_type = 'plain'
219 245
220 -
    def __init__(self, template_dir=None,
221 -
                 template_filename=None, template=None,
222 -
                 subject_filename=None, subject=None,
223 -
                 template_type=None, **kwargs):
224 -
        self.body_template = self.getTemplate(template_filename, template_dir, template)
225 -
        self.subject_template = None
226 -
        if subject_filename or subject:
227 -
            self.subject_template = self.getTemplate(subject_filename, template_dir, subject)
246 +
    def __init__(self, template, subject=None):
247 +
        super().__init__()
248 +
        self.template = template
249 +
        self.subject = subject
228 250
229 -
        if template_type is not None:
230 -
            self.template_type = template_type
251 +
    @defer.inlineCallbacks
252 +
    def format_message_for_build(self, master, build, **kwargs):
253 +
        msgdict = yield self.render_message_dict(master, {'build': build, 'master': master})
254 +
        return msgdict
231 255
232 -
        super().__init__(**kwargs)
256 +
    @defer.inlineCallbacks
257 +
    def render_message_body(self, context):
258 +
        props = Properties.fromDict(context['build']['properties'])
259 +
        props.master = context['master']
260 +
261 +
        body = yield props.render(self.template)
262 +
        return body
263 +
264 +
    @defer.inlineCallbacks
265 +
    def render_message_subject(self, context):
266 +
        props = Properties.fromDict(context['build']['properties'])
267 +
        props.master = context['master']
268 +
269 +
        body = yield props.render(self.subject)
270 +
        return body
271 +
272 +
273 +
default_body_template = '''\
274 +
The Buildbot has detected a {{ status_detected }} on builder {{ buildername }} while building {{ projects }}.
275 +
Full details are available at:
276 +
    {{ build_url }}
233 277
234 -
    def getTemplate(self, filename, dirname, content):
235 -
        if content and (filename or dirname):
236 -
            config.error("Only one of template or template path can be given")
278 +
Buildbot URL: {{ buildbot_url }}
237 279
238 -
        if content:
239 -
            return jinja2.Template(content)
280 +
Worker for this Build: {{ workername }}
240 281
241 -
        if dirname is None:
242 -
            dirname = os.path.join(os.path.dirname(__file__), "templates")
282 +
Build Reason: {{ build['properties'].get('reason', ["<unknown>"])[0] }}
283 +
Blamelist: {{ ", ".join(blamelist) }}
243 284
244 -
        loader = jinja2.FileSystemLoader(dirname)
245 -
        env = jinja2.Environment(
246 -
            loader=loader, undefined=jinja2.StrictUndefined)
285 +
{{ summary }}
247 286
248 -
        if filename is None:
249 -
            filename = self.template_filename
287 +
Sincerely,
288 +
 -The Buildbot
289 +
'''  # noqa pylint: disable=line-too-long
250 290
251 -
        return env.get_template(filename)
291 +
292 +
class MessageFormatterBaseJinja(MessageFormatterBase):
293 +
    compare_attrs = ['body_template', 'subject_template', 'template_type']
294 +
    subject_template = None
295 +
    template_type = 'plain'
296 +
297 +
    def __init__(self, template=None, subject=None, template_type=None, **kwargs):
298 +
        if template is None:
299 +
            template = default_body_template
300 +
301 +
        self.body_template = jinja2.Template(template)
302 +
303 +
        if subject is not None:
304 +
            self.subject_template = jinja2.Template(subject)
305 +
306 +
        if template_type is not None:
307 +
            self.template_type = template_type
308 +
309 +
        super().__init__(**kwargs)
252 310
253 311
    def buildAdditionalContext(self, master, ctx):
254 312
        pass
@@ -263,27 +321,35 @@
Loading
263 321
264 322
265 323
class MessageFormatter(MessageFormatterBaseJinja):
266 -
    template_filename = 'default_mail.txt'
324 +
    @defer.inlineCallbacks
325 +
    def format_message_for_build(self, master, build, users=None, mode=None):
326 +
        ctx = create_context_for_build(mode, build, master, users)
327 +
        msgdict = yield self.render_message_dict(master, ctx)
328 +
        return msgdict
267 329
268 -
    compare_attrs = ['wantProperties', 'wantSteps', 'wantLogs']
269 330
270 -
    def __init__(self, template_name=None, **kwargs):
331 +
default_missing_template = '''\
332 +
The Buildbot working for '{{buildbot_title}}' has noticed that the worker named {{worker.name}} went away.
271 333
272 -
        if template_name is not None:
273 -
            warn_deprecated('0.9.1', "template_name is deprecated, use template_filename")
274 -
            kwargs['template_filename'] = template_name
275 -
        super().__init__(**kwargs)
334 +
It last disconnected at {{worker.last_connection}}.
276 335
277 -
    @defer.inlineCallbacks
278 -
    def format_message_for_build(self, mode, buildername, build, master, blamelist):
279 -
        ctx = create_context_for_build(mode, buildername, build, master, blamelist)
280 -
        msgdict = yield self.render_message_dict(master, ctx)
281 -
        return msgdict
336 +
{% if 'admin' in worker['workerinfo'] %}
337 +
The admin on record (as reported by WORKER:info/admin) was {{worker.workerinfo.admin}}.
338 +
{% endif %}
339 +
340 +
Sincerely,
341 +
 -The Buildbot
342 +
'''  # noqa pylint: disable=line-too-long
282 343
283 344
284 345
class MessageFormatterMissingWorker(MessageFormatterBaseJinja):
285 346
    template_filename = 'missing_mail.txt'
286 347
348 +
    def __init__(self, template=None, **kwargs):
349 +
        if template is None:
350 +
            template = default_missing_template
351 +
        super().__init__(template=template, **kwargs)
352 +
287 353
    @defer.inlineCallbacks
288 354
    def formatMessageForMissingWorker(self, master, worker):
289 355
        ctx = create_context_for_worker(master, worker)

@@ -0,0 +1,103 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
import abc
17 +
18 +
from twisted.internet import defer
19 +
from twisted.python import log
20 +
21 +
from buildbot import config
22 +
from buildbot.reporters import utils
23 +
from buildbot.util import service
24 +
from buildbot.util import tuplematch
25 +
26 +
ENCODING = 'utf-8'
27 +
28 +
29 +
class ReporterBase(service.BuildbotService):
30 +
    name = None
31 +
    __meta__ = abc.ABCMeta
32 +
33 +
    compare_attrs = ['generators']
34 +
35 +
    def __init__(self, *args, **kwargs):
36 +
        super().__init__(*args, **kwargs)
37 +
        self.generators = None
38 +
        self._event_consumers = []
39 +
40 +
    def checkConfig(self, generators):
41 +
        if not isinstance(generators, list):
42 +
            config.error('{}: generators argument must be a list')
43 +
44 +
        for g in generators:
45 +
            g.check()
46 +
47 +
        if self.name is None:
48 +
            self.name = self.__class__.__name__
49 +
            for g in generators:
50 +
                self.name += "_" + g.generate_name()
51 +
52 +
    @defer.inlineCallbacks
53 +
    def reconfigService(self, generators):
54 +
55 +
        for consumer in self._event_consumers:
56 +
            yield consumer.stopConsuming()
57 +
        self._event_consumers = []
58 +
59 +
        self.generators = generators
60 +
61 +
        wanted_event_keys = set()
62 +
        for g in self.generators:
63 +
            wanted_event_keys.update(g.wanted_event_keys)
64 +
65 +
        for key in sorted(list(wanted_event_keys)):
66 +
            consumer = yield self.master.mq.startConsuming(self._got_event, key)
67 +
            self._event_consumers.append(consumer)
68 +
69 +
    @defer.inlineCallbacks
70 +
    def stopService(self):
71 +
        for consumer in self._event_consumers:
72 +
            yield consumer.stopConsuming()
73 +
        self._event_consumers = []
74 +
        yield super().stopService()
75 +
76 +
    def _does_generator_want_key(self, generator, key):
77 +
        for filter in generator.wanted_event_keys:
78 +
            if tuplematch.matchTuple(key, filter):
79 +
                return True
80 +
        return False
81 +
82 +
    @defer.inlineCallbacks
83 +
    def _got_event(self, key, msg):
84 +
        try:
85 +
            reports = []
86 +
            for g in self.generators:
87 +
                if self._does_generator_want_key(g, key):
88 +
                    report = yield g.generate(self.master, self, key, msg)
89 +
                    if report is not None:
90 +
                        reports.append(report)
91 +
92 +
            if reports:
93 +
                yield self.sendMessage(reports)
94 +
        except Exception as e:
95 +
            log.err(e, 'Got exception when handling reporter events')
96 +
97 +
    def getResponsibleUsersForBuild(self, master, buildid):
98 +
        # Use library method but subclassers may want to override that
99 +
        return utils.getResponsibleUsersForBuild(master, buildid)
100 +
101 +
    @abc.abstractmethod
102 +
    def sendMessage(self, reports):
103 +
        pass

@@ -21,7 +21,6 @@
Loading
21 21
from twisted.internet import error
22 22
from twisted.internet import protocol
23 23
from twisted.internet import reactor
24 -
from twisted.protocols.basic import LineOnlyReceiver
25 24
from twisted.python.failure import Failure
26 25
27 26
from buildbot.util import unicode2bytes
@@ -49,10 +48,41 @@
Loading
49 48
        self.lw.dataReceived(data)
50 49
51 50
    def errReceived(self, data):
52 -
        print("ERR: '{}'".format(data))
51 +
        self.lw.print_output("ERR: '{}'".format(data))
52 +
53 +
54 +
class LineOnlyLongLineReceiver(protocol.Protocol):
55 +
    """
56 +
    This is almost the same as Twisted's LineOnlyReceiver except that long lines are handled
57 +
    appropriately.
58 +
    """
59 +
    _buffer = b''
60 +
    delimiter = b'\r\n'
61 +
    MAX_LENGTH = 16384
62 +
63 +
    def dataReceived(self, data):
64 +
        lines = (self._buffer + data).split(self.delimiter)
65 +
        self._buffer = lines.pop(-1)
66 +
        for line in lines:
67 +
            if self.transport.disconnecting:
68 +
                # this is necessary because the transport may be told to lose
69 +
                # the connection by a line within a larger packet, and it is
70 +
                # important to disregard all the lines in that packet following
71 +
                # the one that told it to close.
72 +
                return
73 +
            if len(line) > self.MAX_LENGTH:
74 +
                self.lineLengthExceeded(line)
75 +
            else:
76 +
                self.lineReceived(line)
77 +
78 +
    def lineReceived(self, line):
79 +
        raise NotImplementedError
80 +
81 +
    def lineLengthExceeded(self, line):
82 +
        raise NotImplementedError
53 83
54 84
55 -
class LogWatcher(LineOnlyReceiver):
85 +
class LogWatcher(LineOnlyLongLineReceiver):
56 86
    POLL_INTERVAL = 0.1
57 87
    TIMEOUT_DELAY = 10.0
58 88
    delimiter = unicode2bytes(os.linesep)
@@ -69,8 +99,7 @@
Loading
69 99
70 100
    def start(self):
71 101
        # If the log file doesn't exist, create it now.
72 -
        if not os.path.exists(self.logfile):
73 -
            open(self.logfile, 'a').close()
102 +
        self.create_logfile(self.logfile)
74 103
75 104
        # return a Deferred that fires when the reconfig process has
76 105
        # finished. It errbacks with TimeoutError if the startup has not
@@ -121,6 +150,17 @@
Loading
121 150
        self.in_reconfig = False
122 151
        self.d.callback(results)
123 152
153 +
    def create_logfile(self, path):  # pragma: no cover
154 +
        if not os.path.exists(path):
155 +
            open(path, 'a').close()
156 +
157 +
    def print_output(self, output):  # pragma: no cover
158 +
        print(output)
159 +
160 +
    def lineLengthExceeded(self, line):
161 +
        msg = 'Got an a very long line in the log (length {} bytes), ignoring'.format(len(line))
162 +
        self.print_output(msg)
163 +
124 164
    def lineReceived(self, line):
125 165
        if not self.running:
126 166
            return None
@@ -130,7 +170,7 @@
Loading
130 170
            self.in_reconfig = True
131 171
132 172
        if self.in_reconfig:
133 -
            print(line.decode())
173 +
            self.print_output(line.decode())
134 174
135 175
        # certain lines indicate progress, so we "cancel" the timeout
136 176
        # and it will get re-added when it fires

@@ -33,9 +33,10 @@
Loading
33 33
        return d
34 34
35 35
    def notify(self, result):
36 -
        waiters, self._waiters = self._waiters, []
37 -
        for waiter in waiters:
38 -
            waiter.callback(result)
36 +
        if self._waiters:
37 +
            waiters, self._waiters = self._waiters, []
38 +
            for waiter in waiters:
39 +
                waiter.callback(result)
39 40
40 41
    def __bool__(self):
41 42
        return bool(self._waiters)

@@ -25,8 +25,11 @@
Loading
25 25
from buildbot.process.buildstep import BuildStep
26 26
from buildbot.process.properties import Properties
27 27
from buildbot.process.properties import Property
28 +
from buildbot.process.results import ALL_RESULTS
28 29
from buildbot.process.results import statusToString
29 30
from buildbot.process.results import worst_status
31 +
from buildbot.reporters.utils import getURLForBuild
32 +
from buildbot.reporters.utils import getURLForBuildrequest
30 33
31 34
32 35
class Trigger(BuildStep):
@@ -107,6 +110,7 @@
Loading
107 110
        self.brids = []
108 111
        self.triggeredNames = None
109 112
        self.waitForFinishDeferred = None
113 +
        self._result_list = []
110 114
        super().__init__(**kwargs)
111 115
112 116
    def interrupt(self, reason):
@@ -230,11 +234,25 @@
Loading
230 234
                            builderDict = yield self.master.data.get(("builders", builderid))
231 235
                            builderNames[builderid] = builderDict["name"]
232 236
                        num = build['number']
233 -
                        url = self.master.status.getURLForBuild(builderid, num)
237 +
                        url = getURLForBuild(self.master, builderid, num)
234 238
                        yield self.addURL("{}: {} #{}".format(statusToString(build["results"]),
235 239
                                                              builderNames[builderid], num),
236 240
                                          url)
237 241
242 +
    @defer.inlineCallbacks
243 +
    def _add_results(self, brid):
244 +
        @defer.inlineCallbacks
245 +
        def _is_buildrequest_complete(brid):
246 +
            buildrequest = yield self.master.db.buildrequests.getBuildRequest(brid)
247 +
            return buildrequest['complete']
248 +
249 +
        event = ('buildrequests', str(brid), 'complete')
250 +
        yield self.master.mq.waitUntilEvent(event, lambda: _is_buildrequest_complete(brid))
251 +
        builds = yield self.master.db.builds.getBuilds(buildrequestid=brid)
252 +
        for build in builds:
253 +
            self._result_list.append(build["results"])
254 +
        self.updateSummary()
255 +
238 256
    @defer.inlineCallbacks
239 257
    def run(self):
240 258
        schedulers_and_props = yield self.getSchedulersAndProperties()
@@ -295,8 +313,11 @@
Loading
295 313
            for brid in brids.values():
296 314
                # put the url to the brids, so that we can have the status from
297 315
                # the beginning
298 -
                url = self.master.status.getURLForBuildrequest(brid)
316 +
                url = getURLForBuildrequest(self.master, brid)
299 317
                yield self.addURL("{} #{}".format(sch.name, brid), url)
318 +
                # No yield since we let this happen as the builds complete
319 +
                self._add_results(brid)
320 +
300 321
            dl.append(resultsDeferred)
301 322
            triggeredNames.append(sch.name)
302 323
            if self.ended:
@@ -330,4 +351,11 @@
Loading
330 351
    def getCurrentSummary(self):
331 352
        if not self.triggeredNames:
332 353
            return {'step': 'running'}
333 -
        return {'step': 'triggered {}'.format(', '.join(self.triggeredNames))}
354 +
        summary = ""
355 +
        if self._result_list:
356 +
            for status in ALL_RESULTS:
357 +
                count = self._result_list.count(status)
358 +
                if count:
359 +
                    summary = summary + ", {} {}".format(self._result_list.count(status),
360 +
                                                     statusToString(status, count))
361 +
        return {'step': 'triggered {}{}'.format(', '.join(self.triggeredNames), summary)}

@@ -26,15 +26,16 @@
Loading
26 26
from buildbot.process.results import SKIPPED
27 27
from buildbot.process.results import SUCCESS
28 28
from buildbot.process.results import WARNINGS
29 -
from buildbot.reporters import http
29 +
from buildbot.reporters.base import ReporterBase
30 +
from buildbot.reporters.generators.build import BuildStartEndStatusGenerator
31 +
from buildbot.reporters.message import MessageFormatterRenderable
30 32
from buildbot.util import httpclientservice
31 33
from buildbot.util.logger import Logger
32 -
from buildbot.warnings import warn_deprecated
33 34
34 35
log = Logger()
35 36
36 37
37 -
class GerritVerifyStatusPush(http.HttpStatusPushBase):
38 +
class GerritVerifyStatusPush(ReporterBase):
38 39
    name = "GerritVerifyStatusPush"
39 40
    # overridable constants
40 41
    RESULTS_TABLE = {
@@ -48,29 +49,29 @@
Loading
48 49
    }
49 50
    DEFAULT_RESULT = -1
50 51
51 -
    def checkConfig(self, baseURL, auth, startDescription=None, endDescription=None,
52 -
                    verification_name=None, abstain=False, category=None, reporter=None,
53 -
                    verbose=False, wantProperties=True, **kwargs):
54 -
        super().checkConfig(wantProperties=wantProperties,
55 -
                            _has_old_arg_names={
56 -
                                'wantProperties': wantProperties is not True
57 -
                            }, **kwargs)
52 +
    def checkConfig(self, baseURL, auth, verification_name=None, abstain=False, category=None,
53 +
                    reporter=None, verbose=False, debug=None, verify=None, generators=None,
54 +
                    **kwargs):
55 +
56 +
        if generators is None:
57 +
            generators = self._create_default_generators()
58 +
59 +
        super().checkConfig(generators=generators, **kwargs)
60 +
        httpclientservice.HTTPClientService.checkAvailable(self.__class__.__name__)
58 61
59 62
    @defer.inlineCallbacks
60 -
    def reconfigService(self,
61 -
                        baseURL,
62 -
                        auth,
63 -
                        startDescription=None,
64 -
                        endDescription=None,
65 -
                        verification_name=None,
66 -
                        abstain=False,
67 -
                        category=None,
68 -
                        reporter=None,
69 -
                        verbose=False,
70 -
                        wantProperties=True,
63 +
    def reconfigService(self, baseURL, auth, verification_name=None, abstain=False, category=None,
64 +
                        reporter=None, verbose=False, debug=None, verify=None, generators=None,
71 65
                        **kwargs):
72 66
        auth = yield self.renderSecrets(auth)
73 -
        yield super().reconfigService(wantProperties=wantProperties, **kwargs)
67 +
        self.debug = debug
68 +
        self.verify = verify
69 +
        self.verbose = verbose
70 +
71 +
        if generators is None:
72 +
            generators = self._create_default_generators()
73 +
74 +
        yield super().reconfigService(generators=generators, **kwargs)
74 75
75 76
        if baseURL.endswith('/'):
76 77
            baseURL = baseURL[:-1]
@@ -84,10 +85,17 @@
Loading
84 85
        self._reporter = reporter or "buildbot"
85 86
        self._abstain = abstain
86 87
        self._category = category
87 -
        self._startDescription = startDescription or 'Build started.'
88 -
        self._endDescription = endDescription or 'Build done.'
89 88
        self._verbose = verbose
90 89
90 +
    def _create_default_generators(self):
91 +
        start_formatter = MessageFormatterRenderable('Build started.')
92 +
        end_formatter = MessageFormatterRenderable('Build done.')
93 +
94 +
        return [
95 +
            BuildStartEndStatusGenerator(start_formatter=start_formatter,
96 +
                                         end_formatter=end_formatter)
97 +
        ]
98 +
91 99
    def createStatus(self,
92 100
                     change_id,
93 101
                     revision_id,
@@ -199,33 +207,22 @@
Loading
199 207
            }]
200 208
        return []
201 209
202 -
    @defer.inlineCallbacks
203 -
    def send(self, build):
204 -
        # the only case when this function is called is when the user derives this class, overrides
205 -
        # send() and calls super().send(build) from there.
206 -
        yield self._send_impl(build)
207 -
208 210
    @defer.inlineCallbacks
209 211
    def sendMessage(self, reports):
212 +
        report = reports[0]
210 213
        build = reports[0]['builds'][0]
211 -
        if self.send.__func__ is not GerritVerifyStatusPush.send:
212 -
            warn_deprecated('2.9.0', 'send() in reporters has been deprecated. Use sendMessage()')
213 -
            yield self.send(build)
214 -
        else:
215 -
            yield self._send_impl(build)
216 214
217 -
    @defer.inlineCallbacks
218 -
    def _send_impl(self, build):
219 215
        props = Properties.fromDict(build['properties'])
216 +
        props.master = self.master
217 +
218 +
        comment = report.get('body', None)
219 +
220 220
        if build['complete']:
221 221
            value = self.RESULTS_TABLE.get(build['results'],
222 222
                                           self.DEFAULT_RESULT)
223 -
            comment = yield props.render(self._endDescription)
224 -
            duration = self.formatDuration(build['complete_at'] - build[
225 -
                'started_at'])
223 +
            duration = self.formatDuration(build['complete_at'] - build['started_at'])
226 224
        else:
227 225
            value = 0
228 -
            comment = yield props.render(self._startDescription)
229 226
            duration = 'pending'
230 227
231 228
        name = yield props.render(self._verification_name)

@@ -17,7 +17,7 @@
Loading
17 17
# the door" and trigger a change source to poll.
18 18
19 19
20 -
from buildbot.changes.base import PollingChangeSource
20 +
from buildbot.changes.base import ReconfigurablePollingChangeSource
21 21
from buildbot.util import bytes2unicode
22 22
from buildbot.util import unicode2bytes
23 23
from buildbot.www.hooks.base import BaseHookHandler
@@ -38,7 +38,7 @@
Loading
38 38
        pollers = []
39 39
40 40
        for source in change_svc:
41 -
            if not isinstance(source, PollingChangeSource):
41 +
            if not isinstance(source, ReconfigurablePollingChangeSource):
42 42
                continue
43 43
            if not hasattr(source, "name"):
44 44
                continue

@@ -18,6 +18,8 @@
Loading
18 18
19 19
from buildbot import interfaces
20 20
from buildbot.reporters import utils
21 +
from buildbot.reporters.message import MessageFormatter
22 +
from buildbot.reporters.message import MessageFormatterRenderable
21 23
22 24
from .utils import BuildStatusGeneratorMixin
23 25
@@ -29,17 +31,16 @@
Loading
29 31
        ('builds', None, 'finished'),
30 32
    ]
31 33
34 +
    compare_attrs = ['formatter']
35 +
32 36
    def __init__(self, mode=("failing", "passing", "warnings"),
33 37
                 tags=None, builders=None, schedulers=None, branches=None,
34 38
                 subject="Buildbot %(result)s in %(title)s on %(builder)s",
35 -
                 add_logs=False, add_patch=False, report_new=False, message_formatter=None,
36 -
                 _want_previous_build=None):
37 -
        super().__init__(mode, tags, builders, schedulers, branches, subject, add_logs, add_patch,
38 -
                         message_formatter)
39 -
        self._report_new = report_new
40 -
41 -
        # TODO: private and deprecated, included only to support HttpStatusPushBase
42 -
        self._want_previous_build_override = _want_previous_build
39 +
                 add_logs=False, add_patch=False, report_new=False, message_formatter=None):
40 +
        super().__init__(mode, tags, builders, schedulers, branches, subject, add_logs, add_patch)
41 +
        self.formatter = message_formatter
42 +
        if self.formatter is None:
43 +
            self.formatter = MessageFormatter()
43 44
44 45
        if report_new:
45 46
            self.wanted_event_keys = [
@@ -52,8 +53,6 @@
Loading
52 53
        _, _, event = key
53 54
        is_new = event == 'new'
54 55
        want_previous_build = False if is_new else self._want_previous_build()
55 -
        if self._want_previous_build_override is not None:
56 -
            want_previous_build = self._want_previous_build_override
57 56
58 57
        yield utils.getDetailsForBuild(master, build,
59 58
                                       wantProperties=self.formatter.wantProperties,
@@ -66,12 +65,48 @@
Loading
66 65
        if not is_new and not self.is_message_needed_by_results(build):
67 66
            return None
68 67
69 -
        report = yield self.build_message(master, reporter, build['builder']['name'], [build],
70 -
                                          build['results'])
68 +
        report = yield self.build_message(self.formatter, master, reporter, build)
71 69
        return report
72 70
73 71
    def _want_previous_build(self):
74 72
        return "change" in self.mode or "problem" in self.mode
75 73
76 -
    def _matches_any_tag(self, tags):
77 -
        return self.tags and any(tag for tag in self.tags if tag in tags)
74 +
75 +
@implementer(interfaces.IReportGenerator)
76 +
class BuildStartEndStatusGenerator(BuildStatusGeneratorMixin):
77 +
78 +
    wanted_event_keys = [
79 +
        ('builds', None, 'new'),
80 +
        ('builds', None, 'finished'),
81 +
    ]
82 +
83 +
    compare_attrs = ['start_formatter', 'end_formatter']
84 +
85 +
    def __init__(self, tags=None, builders=None, schedulers=None, branches=None, add_logs=False,
86 +
                 add_patch=False, start_formatter=None, end_formatter=None):
87 +
88 +
        super().__init__('all', tags, builders, schedulers, branches, None, add_logs, add_patch)
89 +
        self.start_formatter = start_formatter
90 +
        if self.start_formatter is None:
91 +
            self.start_formatter = MessageFormatterRenderable('Build started.')
92 +
        self.end_formatter = end_formatter
93 +
        if self.end_formatter is None:
94 +
            self.end_formatter = MessageFormatterRenderable('Build done.')
95 +
96 +
    @defer.inlineCallbacks
97 +
    def generate(self, master, reporter, key, build):
98 +
        _, _, event = key
99 +
        is_new = event == 'new'
100 +
101 +
        formatter = self.start_formatter if is_new else self.end_formatter
102 +
103 +
        yield utils.getDetailsForBuild(master, build,
104 +
                                       wantProperties=formatter.wantProperties,
105 +
                                       wantSteps=formatter.wantSteps,
106 +
                                       wantLogs=formatter.wantLogs)
107 +
108 +
        if not self.is_message_needed_by_props(build):
109 +
            return None
110 +
111 +
        report = yield self.build_message(formatter, master, reporter, build)
112 +
        return report

@@ -87,6 +87,20 @@
Loading
87 87
    return ret
88 88
89 89
90 +
@defer.inlineCallbacks
91 +
def get_details_for_buildrequest(master, buildrequest, build):
92 +
    buildset = yield master.data.get(("buildsets", buildrequest['buildsetid']))
93 +
    builder = yield master.data.get(("builders", buildrequest['builderid']))
94 +
95 +
    build['buildrequest'] = buildrequest
96 +
    build['buildset'] = buildset
97 +
    build['builderid'] = buildrequest['builderid']
98 +
    build['builder'] = builder
99 +
    build['url'] = getURLForBuildrequest(master, buildrequest['buildrequestid'])
100 +
    build['results'] = None
101 +
    build['complete'] = False
102 +
103 +
90 104
@defer.inlineCallbacks
91 105
def getDetailsForBuilds(master, buildset, builds, wantProperties=False, wantSteps=False,
92 106
                        wantPreviousBuild=False, wantLogs=False):
@@ -203,6 +217,11 @@
Loading
203 217
        build_number)
204 218
205 219
220 +
def getURLForBuildrequest(master, buildrequestid):
221 +
    prefix = master.config.buildbotURL
222 +
    return "{}#buildrequests/{}".format(prefix, buildrequestid)
223 +
224 +
206 225
@renderer
207 226
def URLForBuild(props):
208 227
    build = props.getBuild()

@@ -229,6 +229,8 @@
Loading
229 229
230 230
    _msgtypes_re_str = '(?P<errtype>[{}])'.format(''.join(list(_MESSAGES)))
231 231
    _default_line_re = re.compile(r'^{}(\d+)?: *\d+(, *\d+)?:.+'.format(_msgtypes_re_str))
232 +
    _default_2_0_0_line_re = \
233 +
        re.compile(r'^(?P<path>[^:]+):(?P<line>\d+):\d+: *{}(\d+)?:.+'.format(_msgtypes_re_str))
232 234
    _parseable_line_re = re.compile(
233 235
        r'(?P<path>[^:]+):(?P<line>\d+): \[{}(\d+)?(\([a-z-]+\))?[,\]] .+'.format(_msgtypes_re_str))
234 236
@@ -242,6 +244,14 @@
Loading
242 244
243 245
    # returns (message type, path, line) tuple if line has been matched, or None otherwise
244 246
    def _match_line(self, line):
247 +
        m = self._default_2_0_0_line_re.match(line)
248 +
        if m:
249 +
            try:
250 +
                line_int = int(m.group('line'))
251 +
            except ValueError:
252 +
                line_int = None
253 +
            return (m.group('errtype'), m.group('path'), line_int)
254 +
245 255
        m = self._parseable_line_re.match(line)
246 256
        if m:
247 257
            try:

@@ -17,50 +17,40 @@
Loading
17 17
from twisted.internet import defer
18 18
19 19
from buildbot import config
20 -
from buildbot.reporters.http import HttpStatusPushBase
20 +
from buildbot.reporters.base import ReporterBase
21 +
from buildbot.reporters.generators.build import BuildStartEndStatusGenerator
21 22
from buildbot.util import httpclientservice
22 23
from buildbot.util.logger import Logger
23 -
from buildbot.warnings import warn_deprecated
24 24
25 25
log = Logger()
26 26
27 27
28 -
class ZulipStatusPush(HttpStatusPushBase):
28 +
class ZulipStatusPush(ReporterBase):
29 29
    name = "ZulipStatusPush"
30 30
31 -
    def checkConfig(self, endpoint, token, stream=None, **kwargs):
31 +
    def checkConfig(self, endpoint, token, stream=None, debug=None, verify=None):
32 32
        if not isinstance(endpoint, str):
33 33
            config.error("Endpoint must be a string")
34 34
        if not isinstance(token, str):
35 35
            config.error("Token must be a string")
36 -
        super().checkConfig(**kwargs)
36 +
37 +
        super().checkConfig(generators=[BuildStartEndStatusGenerator()])
38 +
        httpclientservice.HTTPClientService.checkAvailable(self.__class__.__name__)
37 39
38 40
    @defer.inlineCallbacks
39 -
    def reconfigService(self, endpoint, token, stream=None, wantProperties=True, **kwargs):
40 -
        super().reconfigService(wantProperties=wantProperties, **kwargs)
41 +
    def reconfigService(self, endpoint, token, stream=None, debug=None, verify=None):
42 +
        self.debug = debug
43 +
        self.verify = verify
44 +
        yield super().reconfigService(generators=[BuildStartEndStatusGenerator()])
41 45
        self._http = yield httpclientservice.HTTPClientService.getService(
42 46
            self.master, endpoint,
43 47
            debug=self.debug, verify=self.verify)
44 48
        self.token = token
45 49
        self.stream = stream
46 50
47 -
    @defer.inlineCallbacks
48 -
    def send(self, build):
49 -
        # the only case when this function is called is when the user derives this class, overrides
50 -
        # send() and calls super().send(build) from there.
51 -
        yield self._send_impl(build)
52 -
53 51
    @defer.inlineCallbacks
54 52
    def sendMessage(self, reports):
55 53
        build = reports[0]['builds'][0]
56 -
        if self.send.__func__ is not ZulipStatusPush.send:
57 -
            warn_deprecated('2.9.0', 'send() in reporters has been deprecated. Use sendMessage()')
58 -
            yield self.send(build)
59 -
        else:
60 -
            yield self._send_impl(build)
61 -
62 -
    @defer.inlineCallbacks
63 -
    def _send_impl(self, build):
64 54
        event = ("new", "finished")[0 if build["complete"] is False else 1]
65 55
        jsondata = dict(event=event, buildid=build["buildid"], buildername=build["builder"]["name"],
66 56
                        url=build["url"], project=build["properties"]["project"][0])

@@ -0,0 +1,88 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
from twisted.internet import defer
17 +
from zope.interface import implementer
18 +
19 +
from buildbot import interfaces
20 +
from buildbot.process.build import Build
21 +
from buildbot.process.buildrequest import BuildRequest
22 +
from buildbot.process.properties import Properties
23 +
from buildbot.reporters import utils
24 +
from buildbot.reporters.message import MessageFormatterRenderable
25 +
26 +
from .utils import BuildStatusGeneratorMixin
27 +
28 +
29 +
@implementer(interfaces.IReportGenerator)
30 +
class BuildRequestGenerator(BuildStatusGeneratorMixin):
31 +
32 +
    wanted_event_keys = [
33 +
        ('buildrequests', None, 'new')
34 +
    ]
35 +
36 +
    compare_attrs = ['formatter']
37 +
38 +
    def __init__(self, tags=None, builders=None, schedulers=None, branches=None,
39 +
                 add_patch=False, formatter=None):
40 +
41 +
        super().__init__('all', tags, builders, schedulers, branches, None, False, add_patch)
42 +
        self.formatter = formatter
43 +
        if self.formatter is None:
44 +
            self.formatter = MessageFormatterRenderable('Build pending.')
45 +
46 +
    @defer.inlineCallbacks
47 +
    def partial_build_dict(self, master, buildrequest):
48 +
        brdict = yield master.db.buildrequests.getBuildRequest(buildrequest['buildrequestid'])
49 +
        bdict = dict()
50 +
51 +
        props = Properties()
52 +
        buildrequest = yield BuildRequest.fromBrdict(master, brdict)
53 +
        builder = yield master.botmaster.getBuilderById(brdict['builderid'])
54 +
55 +
        Build.setupPropertiesKnownBeforeBuildStarts(props, [buildrequest], builder)
56 +
        Build.setupBuildProperties(props, [buildrequest])
57 +
58 +
        bdict['properties'] = props.asDict()
59 +
        yield utils.get_details_for_buildrequest(master, brdict, bdict)
60 +
        return bdict
61 +
62 +
    @defer.inlineCallbacks
63 +
    def generate(self, master, reporter, key, buildrequest):
64 +
        build = yield self.partial_build_dict(master, buildrequest)
65 +
66 +
        if not self.is_message_needed_by_props(build):
67 +
            return None
68 +
69 +
        report = yield self.buildrequest_message(master, build)
70 +
        return report
71 +
72 +
    @defer.inlineCallbacks
73 +
    def buildrequest_message(self, master, build):
74 +
        patches = self._get_patches_for_build(build)
75 +
        users = []
76 +
        buildmsg = yield self.formatter.format_message_for_build(master, build, mode=self.mode,
77 +
                                                                 users=users)
78 +
79 +
        return {
80 +
            'body': buildmsg['body'],
81 +
            'subject': buildmsg['subject'],
82 +
            'type': buildmsg['type'],
83 +
            'results': build['results'],
84 +
            'builds': [build],
85 +
            'users': list(users),
86 +
            'patches': patches,
87 +
            'logs': []
88 +
        }

@@ -14,13 +14,11 @@
Loading
14 14
# Copyright Buildbot Team Members
15 15
16 16
import datetime
17 -
import inspect
18 17
import os
19 18
import re
20 19
import sys
21 20
import traceback
22 21
import warnings
23 -
from types import MethodType
24 22
25 23
from twisted.python import failure
26 24
from twisted.python import log
@@ -39,7 +37,6 @@
Loading
39 37
from buildbot.util import safeTranslate
40 38
from buildbot.util import service as util_service
41 39
from buildbot.warnings import ConfigWarning
42 -
from buildbot.warnings import warn_deprecated
43 40
from buildbot.www import auth
44 41
from buildbot.www import avatar
45 42
from buildbot.www.authz import authz
@@ -232,7 +229,6 @@
Loading
232 229
        "buildbotURL",
233 230
        "buildCacheSize",
234 231
        "builders",
235 -
        "buildHorizon",
236 232
        "caches",
237 233
        "change_source",
238 234
        "codebaseGenerator",
@@ -240,12 +236,10 @@
Loading
240 236
        "changeCacheSize",
241 237
        "changeHorizon",
242 238
        'db',
243 -
        "db_poll_interval",
244 239
        "db_url",
245 240
        "logCompressionLimit",
246 241
        "logCompressionMethod",
247 242
        "logEncoding",
248 -
        "logHorizon",
249 243
        "logMaxSize",
250 244
        "logMaxTailSize",
251 245
        "manhole",
@@ -263,9 +257,6 @@
Loading
263 257
        "schedulers",
264 258
        "secretsProviders",
265 259
        "services",
266 -
        # we had c['status'] = [] for a while in our default master.cfg
267 -
        # so we need to keep it there
268 -
        "status",
269 260
        "title",
270 261
        "titleURL",
271 262
        "user_managers",
@@ -403,19 +394,6 @@
Loading
403 394
                    category=ConfigWarning)
404 395
        copy_str_or_callable_param('buildbotNetUsageData')
405 396
406 -
        for horizon in ('logHorizon', 'buildHorizon', 'eventHorizon'):
407 -
            if horizon in config_dict:
408 -
                warn_deprecated(
409 -
                    '0.9.0',
410 -
                    "NOTE: `{}` is deprecated and ignored "
411 -
                    "They are replaced by util.JanitorConfigurator".format(horizon))
412 -
413 -
        if 'status' in config_dict:
414 -
            warn_deprecated(
415 -
                '0.9.0',
416 -
                "NOTE: `status` targets are deprecated and ignored "
417 -
                "They are replaced by reporters")
418 -
419 397
        copy_int_param('changeHorizon')
420 398
        copy_int_param('logCompressionLimit')
421 399
@@ -516,13 +494,10 @@
Loading
516 494
517 495
        if 'db' in config_dict:
518 496
            db = config_dict['db']
519 -
            if set(db.keys()) - set(['db_url', 'db_poll_interval']) and throwErrors:
497 +
            if set(db.keys()) - set(['db_url']) and throwErrors:
520 498
                error("unrecognized keys in c['db']")
521 -
            config_dict = db
522 499
523 -
        if 'db_poll_interval' in config_dict and throwErrors:
524 -
            warn_deprecated(
525 -
                "0.8.7", "db_poll_interval is deprecated and will be ignored")
500 +
            config_dict = db
526 501
527 502
        # we don't attempt to parse db URLs here - the engine strategy will do
528 503
        # so.
@@ -916,7 +891,7 @@
Loading
916 891
917 892
    def __init__(self, name=None, workername=None, workernames=None,
918 893
                 builddir=None, workerbuilddir=None, factory=None,
919 -
                 tags=None, category=None,
894 +
                 tags=None,
920 895
                 nextWorker=None, nextBuild=None, locks=None, env=None,
921 896
                 properties=None, collapseRequests=None, description=None,
922 897
                 canStartBuild=None, defaultProperties=None
@@ -972,17 +947,6 @@
Loading
972 947
        self.workerbuilddir = workerbuilddir
973 948
974 949
        # remainder are optional
975 -
976 -
        if category and tags:
977 -
            error(("builder '{}': builder categories are deprecated and "
978 -
                   "replaced by tags; you should only specify tags").format(name))
979 -
        if category:
980 -
            warn_deprecated("0.9", ("builder '{}': builder categories are "
981 -
                                    "deprecated and should be replaced with "
982 -
                                    "'tags=[cat]'").format(name))
983 -
            if not isinstance(category, str):
984 -
                error("builder '{}': category must be a string".format(name))
985 -
            tags = [category]
986 950
        if tags:
987 951
            if not isinstance(tags, list):
988 952
                error("builder '{}': tags must be a list".format(name))
@@ -1003,16 +967,6 @@
Loading
1003 967
        self.nextWorker = nextWorker
1004 968
        if nextWorker and not callable(nextWorker):
1005 969