Showing 68 of 181 files from the diff.
Other files ignored by Codecov
Makefile has changed.
worker/Makefile has changed.
master/setup.py has changed.
.bbtravis.yml has changed.
common/pylintrc has changed.

@@ -14,12 +14,21 @@
Loading
14 14
# Copyright Buildbot Team Members
15 15
16 16
from twisted.internet import defer
17 -
from twisted.internet import reactor
18 17
19 18
from buildbot import config
20 19
from buildbot.process.buildstep import FAILURE
21 20
from buildbot.process.buildstep import SUCCESS
22 21
from buildbot.process.buildstep import BuildStep
22 +
from buildbot.steps.http_oldstyle import DELETE
23 +
from buildbot.steps.http_oldstyle import GET
24 +
from buildbot.steps.http_oldstyle import HEAD
25 +
from buildbot.steps.http_oldstyle import OPTIONS
26 +
from buildbot.steps.http_oldstyle import POST
27 +
from buildbot.steps.http_oldstyle import PUT
28 +
from buildbot.steps.http_oldstyle import HTTPStep
29 +
from buildbot.steps.http_oldstyle import closeSession
30 +
from buildbot.steps.http_oldstyle import getSession
31 +
from buildbot.steps.http_oldstyle import setSession
23 32
24 33
# use the 'requests' lib: https://requests.readthedocs.io/en/master/
25 34
try:
@@ -28,36 +37,24 @@
Loading
28 37
except ImportError:
29 38
    txrequests = None
30 39
40 +
_hush_pyflakes = [
41 +
    DELETE,
42 +
    GET,
43 +
    HEAD,
44 +
    HTTPStep,
45 +
    OPTIONS,
46 +
    POST,
47 +
    PUT,
48 +
    closeSession,
49 +
    getSession,
50 +
    setSession,
51 +
]
52 +
del _hush_pyflakes
31 53
32 -
# This step uses a global Session object, which encapsulates a thread pool as
33 -
# well as state such as cookies and authentication.  This state may pose
34 -
# problems for users, where one step may get a cookie that is subsequently used
35 -
# by another step in a different build.
54 +
# TODO: move session singleton handling back to this module from http_oldstyle
36 55
37 -
_session = None
38 56
39 -
40 -
def getSession():
41 -
    global _session
42 -
    if _session is None:
43 -
        _session = txrequests.Session()
44 -
        reactor.addSystemEventTrigger("before", "shutdown", closeSession)
45 -
    return _session
46 -
47 -
48 -
def setSession(session):
49 -
    global _session
50 -
    _session = session
51 -
52 -
53 -
def closeSession():
54 -
    global _session
55 -
    if _session is not None:
56 -
        _session.close()
57 -
        _session = None
58 -
59 -
60 -
class HTTPStep(BuildStep):
57 +
class HTTPStepNewStyle(BuildStep):
61 58
62 59
    name = 'HTTPStep'
63 60
    description = 'Requesting'
@@ -80,7 +77,7 @@
Loading
80 77
        self.method = method
81 78
        self.url = url
82 79
83 -
        for param in HTTPStep.requestsParams:
80 +
        for param in self.requestsParams:
84 81
            setattr(self, param, kwargs.pop(param, None))
85 82
86 83
        super().__init__(**kwargs)
@@ -167,37 +164,37 @@
Loading
167 164
        yield content_log.addStdout(response.text)
168 165
169 166
170 -
class POST(HTTPStep):
167 +
class POSTNewStyle(HTTPStepNewStyle):
171 168
172 169
    def __init__(self, url, **kwargs):
173 170
        super().__init__(url, method='POST', **kwargs)
174 171
175 172
176 -
class GET(HTTPStep):
173 +
class GETNewStyle(HTTPStepNewStyle):
177 174
178 175
    def __init__(self, url, **kwargs):
179 176
        super().__init__(url, method='GET', **kwargs)
180 177
181 178
182 -
class PUT(HTTPStep):
179 +
class PUTNewStyle(HTTPStepNewStyle):
183 180
184 181
    def __init__(self, url, **kwargs):
185 182
        super().__init__(url, method='PUT', **kwargs)
186 183
187 184
188 -
class DELETE(HTTPStep):
185 +
class DELETENewStyle(HTTPStepNewStyle):
189 186
190 187
    def __init__(self, url, **kwargs):
191 188
        super().__init__(url, method='DELETE', **kwargs)
192 189
193 190
194 -
class HEAD(HTTPStep):
191 +
class HEADNewStyle(HTTPStepNewStyle):
195 192
196 193
    def __init__(self, url, **kwargs):
197 194
        super().__init__(url, method='HEAD', **kwargs)
198 195
199 196
200 -
class OPTIONS(HTTPStep):
197 +
class OPTIONSNewStyle(HTTPStepNewStyle):
201 198
202 199
    def __init__(self, url, **kwargs):
203 200
        super().__init__(url, method='OPTIONS', **kwargs)

@@ -56,13 +56,13 @@
Loading
56 56
                    priorities=None, otherParams=None,
57 57
                    watchedWorkers=None, messageFormatterMissingWorker=None,
58 58
                    generators=None):
59 -
        super(PushoverNotifier, self).checkConfig(mode, tags, builders,
60 -
                                                  buildSetSummary, messageFormatter,
61 -
                                                  subject, False, False,
62 -
                                                  schedulers,
63 -
                                                  branches, watchedWorkers,
64 -
                                                  messageFormatterMissingWorker,
65 -
                                                  generators=generators)
59 +
        super().checkConfig(mode, tags, builders,
60 +
                            buildSetSummary, messageFormatter,
61 +
                            subject, False, False,
62 +
                            schedulers,
63 +
                            branches, watchedWorkers,
64 +
                            messageFormatterMissingWorker,
65 +
                            generators=generators)
66 66
67 67
        httpclientservice.HTTPClientService.checkAvailable(self.__class__.__name__)
68 68

@@ -39,9 +39,9 @@
Loading
39 39
        try:
40 40
            from subunit import TestProtocolServer, PROGRESS_CUR, PROGRESS_SET
41 41
            from subunit import PROGRESS_PUSH, PROGRESS_POP
42 -
        except ImportError:
42 +
        except ImportError as e:
43 43
            raise ImportError("subunit is not importable, but is required for "
44 -
                              "SubunitLogObserver support.")
44 +
                              "SubunitLogObserver support.") from e
45 45
        self.PROGRESS_CUR = PROGRESS_CUR
46 46
        self.PROGRESS_SET = PROGRESS_SET
47 47
        self.PROGRESS_PUSH = PROGRESS_PUSH

@@ -62,6 +62,18 @@
Loading
62 62
        self._buildCompleteConsumer.stopConsuming()
63 63
        self._buildStartedConsumer.stopConsuming()
64 64
65 +
    @defer.inlineCallbacks
66 +
    def _got_event(self, key, msg):
67 +
        # This function is used only from tests
68 +
        if key[0] != 'builds':  # pragma: no cover
69 +
            raise Exception('Invalid key for _got_event: {}'.format(key))
70 +
        if key[2] == 'new':
71 +
            yield self.buildStarted(key, msg)
72 +
        elif key[2] == 'finished':
73 +
            yield self.buildFinished(key, msg)
74 +
        else:  # pragma: no cover
75 +
            raise Exception('Invalid key for _got_event: {}'.format(key))
76 +
65 77
    def buildStarted(self, key, build):
66 78
        return self.getMoreInfoAndSend(build)
67 79

@@ -109,9 +109,9 @@
Loading
109 109
    def getEndpoint(self, path):
110 110
        try:
111 111
            return self.matcher[path]
112 -
        except KeyError:
112 +
        except KeyError as e:
113 113
            raise exceptions.InvalidPathError(
114 -
                "Invalid path: " + "/".join([str(p) for p in path]))
114 +
                "Invalid path: " + "/".join([str(p) for p in path])) from e
115 115
116 116
    def getResourceType(self, name):
117 117
        return getattr(self.rtypes, name)

@@ -16,6 +16,7 @@
Loading
16 16
import inspect
17 17
import re
18 18
import sys
19 +
from io import StringIO
19 20
20 21
from twisted.internet import defer
21 22
from twisted.internet import error
@@ -25,7 +26,6 @@
Loading
25 26
from twisted.python import log
26 27
from twisted.python import util as twutil
27 28
from twisted.python import versions
28 -
from twisted.python.compat import NativeStringIO
29 29
from twisted.python.failure import Failure
30 30
from twisted.python.reflect import accumulateClassList
31 31
from twisted.web.util import formatFailure
@@ -223,7 +223,7 @@
Loading
223 223
224 224
    def readlines(self):
225 225
        alltext = "".join(self.getChunks([self.STDOUT], onlyText=True))
226 -
        io = NativeStringIO(alltext)
226 +
        io = StringIO(alltext)
227 227
        return io.readlines()
228 228
229 229
    def getChunks(self, channels=None, onlyText=False):
@@ -1298,7 +1298,7 @@
Loading
1298 1298
            if self.results != SUCCESS:
1299 1299
                summary += ' ({})'.format(Results[self.results])
1300 1300
            return {'step': summary}
1301 -
        return super(ShellMixin, self).getResultSummary()
1301 +
        return super().getResultSummary()
1302 1302
1303 1303
# Parses the logs for a list of regexs. Meant to be invoked like:
1304 1304
# regexes = ((re.compile(...), FAILURE), (re.compile(...), WARNINGS))

@@ -16,6 +16,7 @@
Loading
16 16
import abc
17 17
18 18
from twisted.internet import defer
19 +
from twisted.python import log
19 20
20 21
from buildbot import config
21 22
from buildbot import util
@@ -165,15 +166,18 @@
Loading
165 166
166 167
    @defer.inlineCallbacks
167 168
    def _got_event(self, key, msg):
168 -
        reports = []
169 -
        for g in self.generators:
170 -
            if self._does_generator_want_key(g, key):
171 -
                report = yield g.generate(self.master, self, key, msg)
172 -
                if report is not None:
173 -
                    reports.append(report)
174 -
175 -
        if reports:
176 -
            yield self.sendMessage(reports)
169 +
        try:
170 +
            reports = []
171 +
            for g in self.generators:
172 +
                if self._does_generator_want_key(g, key):
173 +
                    report = yield g.generate(self.master, self, key, msg)
174 +
                    if report is not None:
175 +
                        reports.append(report)
176 +
177 +
            if reports:
178 +
                yield self.sendMessage(reports)
179 +
        except Exception as e:
180 +
            log.err(e, 'Got exception when handling reporter events')
177 181
178 182
    def getResponsibleUsersForBuild(self, master, buildid):
179 183
        # Use library method but subclassers may want to override that

@@ -100,7 +100,7 @@
Loading
100 100
    except IOError as e:
101 101
        raise ConfigErrors([
102 102
            "unable to open configuration file {}: {}".format(repr(filename), e),
103 -
        ])
103 +
        ]) from e
104 104
105 105
    log.msg("Loading configuration from %r" % (filename,))
106 106

@@ -203,22 +203,23 @@
Loading
203 203
        if self._check_extras:
204 204
            try:
205 205
                entry.require()
206 -
            except Exception as err:
206 +
            except Exception as e:
207 207
                raise PluginDBError(('Requirements are not satisfied '
208 -
                                     'for {}:{}: {}').format(self._group, entry.name, str(err)))
208 +
                                     'for {}:{}: {}').format(
209 +
                                         self._group, entry.name, str(e))) from e
209 210
        try:
210 211
            result = entry.load()
211 -
        except Exception as err:
212 +
        except Exception as e:
212 213
            # log full traceback of the bad entry to help support
213 214
            traceback.print_exc()
214 215
            raise PluginDBError('Unable to load {}:{}: {}'.format(self._group, entry.name,
215 -
                                                                  str(err)))
216 +
                                                                  str(e))) from e
216 217
        if self._interface:
217 218
            try:
218 219
                verifyClass(self._interface, result)
219 -
            except Invalid as err:
220 +
            except Invalid as e:
220 221
                raise PluginDBError('Plugin {}:{} does not implement {}: {}'.format(self._group,
221 -
                        entry.name, self._interface.__name__, str(err)))
222 +
                        entry.name, self._interface.__name__, str(e))) from e
222 223
        return result
223 224
224 225
    @property
@@ -266,8 +267,8 @@
Loading
266 267
    def __getattr__(self, name):
267 268
        try:
268 269
            return getattr(self._tree, name)
269 -
        except PluginDBError as err:
270 -
            raise AttributeError(str(err))
270 +
        except PluginDBError as e:
271 +
            raise AttributeError(str(e)) from e
271 272
272 273
273 274
class _PluginDB:

@@ -0,0 +1,203 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
from twisted.internet import defer
17 +
from twisted.internet import reactor
18 +
19 +
from buildbot import config
20 +
from buildbot.process.buildstep import FAILURE
21 +
from buildbot.process.buildstep import SUCCESS
22 +
from buildbot.process.buildstep import BuildStep
23 +
24 +
# use the 'requests' lib: https://requests.readthedocs.io/en/master/
25 +
try:
26 +
    import txrequests
27 +
    import requests
28 +
except ImportError:
29 +
    txrequests = None
30 +
31 +
32 +
# This step uses a global Session object, which encapsulates a thread pool as
33 +
# well as state such as cookies and authentication.  This state may pose
34 +
# problems for users, where one step may get a cookie that is subsequently used
35 +
# by another step in a different build.
36 +
37 +
_session = None
38 +
39 +
40 +
def getSession():
41 +
    global _session
42 +
    if _session is None:
43 +
        _session = txrequests.Session()
44 +
        reactor.addSystemEventTrigger("before", "shutdown", closeSession)
45 +
    return _session
46 +
47 +
48 +
def setSession(session):
49 +
    global _session
50 +
    _session = session
51 +
52 +
53 +
def closeSession():
54 +
    global _session
55 +
    if _session is not None:
56 +
        _session.close()
57 +
        _session = None
58 +
59 +
60 +
class HTTPStep(BuildStep):
61 +
62 +
    name = 'HTTPStep'
63 +
    description = 'Requesting'
64 +
    descriptionDone = 'Requested'
65 +
    requestsParams = ["params", "data", "json", "headers",
66 +
                      "cookies", "files", "auth",
67 +
                      "timeout", "allow_redirects", "proxies",
68 +
                      "hooks", "stream", "verify", "cert"]
69 +
    renderables = requestsParams + ["method", "url"]
70 +
    session = None
71 +
72 +
    def __init__(self, url, method, **kwargs):
73 +
        if txrequests is None:
74 +
            config.error(
75 +
                "Need to install txrequest to use this step:\n\n pip install txrequests")
76 +
77 +
        if method not in ('POST', 'GET', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'):
78 +
            config.error("Wrong method given: '{}' is not known".format(method))
79 +
80 +
        self.method = method
81 +
        self.url = url
82 +
83 +
        for param in self.requestsParams:
84 +
            setattr(self, param, kwargs.pop(param, None))
85 +
86 +
        super().__init__(**kwargs)
87 +
88 +
    @defer.inlineCallbacks
89 +
    def run(self):
90 +
        # create a new session if it doesn't exist
91 +
        self.session = getSession()
92 +
93 +
        requestkwargs = {
94 +
            'method': self.method,
95 +
            'url': self.url
96 +
        }
97 +
98 +
        for param in self.requestsParams:
99 +
            value = getattr(self, param, None)
100 +
            if value is not None:
101 +
                requestkwargs[param] = value
102 +
103 +
        log = yield self.addLog('log')
104 +
105 +
        # known methods already tested in __init__
106 +
107 +
        yield log.addHeader('Performing {} request to {}\n'.format(self.method, self.url))
108 +
        if self.params:
109 +
            yield log.addHeader('Parameters:\n')
110 +
            params = requestkwargs.get("params", {})
111 +
            if params:
112 +
                params = sorted(params.items(), key=lambda x: x[0])
113 +
                requestkwargs['params'] = params
114 +
            for k, v in params:
115 +
                yield log.addHeader('\t{}: {}\n'.format(k, v))
116 +
        data = requestkwargs.get("data", None)
117 +
        if data:
118 +
            yield log.addHeader('Data:\n')
119 +
            if isinstance(data, dict):
120 +
                for k, v in data.items():
121 +
                    yield log.addHeader('\t{}: {}\n'.format(k, v))
122 +
            else:
123 +
                yield log.addHeader('\t{}\n'.format(data))
124 +
125 +
        try:
126 +
            r = yield self.session.request(**requestkwargs)
127 +
        except requests.exceptions.ConnectionError as e:
128 +
            yield log.addStderr('An exception occurred while performing the request: {}'.format(e))
129 +
            return FAILURE
130 +
131 +
        if r.history:
132 +
            yield log.addStdout('\nRedirected %d times:\n\n' % len(r.history))
133 +
            for rr in r.history:
134 +
                yield self.log_response(log, rr)
135 +
                yield log.addStdout('=' * 60 + '\n')
136 +
137 +
        yield self.log_response(log, r)
138 +
139 +
        yield log.finish()
140 +
141 +
        self.descriptionDone = ["Status code: %d" % r.status_code]
142 +
        if (r.status_code < 400):
143 +
            return SUCCESS
144 +
        else:
145 +
            return FAILURE
146 +
147 +
    @defer.inlineCallbacks
148 +
    def log_response(self, log, response):
149 +
150 +
        yield log.addHeader('Request Header:\n')
151 +
        for k, v in response.request.headers.items():
152 +
            yield log.addHeader('\t{}: {}\n'.format(k, v))
153 +
154 +
        yield log.addStdout('URL: {}\n'.format(response.url))
155 +
156 +
        if response.status_code == requests.codes.ok:
157 +
            yield log.addStdout('Status: {}\n'.format(response.status_code))
158 +
        else:
159 +
            yield log.addStderr('Status: {}\n'.format(response.status_code))
160 +
161 +
        yield log.addHeader('Response Header:\n')
162 +
        for k, v in response.headers.items():
163 +
            yield log.addHeader('\t{}: {}\n'.format(k, v))
164 +
165 +
        yield log.addStdout(' ------ Content ------\n{}'.format(response.text))
166 +
        content_log = yield self.addLog('content')
167 +
        yield content_log.addStdout(response.text)
168 +
169 +
170 +
class POST(HTTPStep):
171 +
172 +
    def __init__(self, url, **kwargs):
173 +
        super().__init__(url, method='POST', **kwargs)
174 +
175 +
176 +
class GET(HTTPStep):
177 +
178 +
    def __init__(self, url, **kwargs):
179 +
        super().__init__(url, method='GET', **kwargs)
180 +
181 +
182 +
class PUT(HTTPStep):
183 +
184 +
    def __init__(self, url, **kwargs):
185 +
        super().__init__(url, method='PUT', **kwargs)
186 +
187 +
188 +
class DELETE(HTTPStep):
189 +
190 +
    def __init__(self, url, **kwargs):
191 +
        super().__init__(url, method='DELETE', **kwargs)
192 +
193 +
194 +
class HEAD(HTTPStep):
195 +
196 +
    def __init__(self, url, **kwargs):
197 +
        super().__init__(url, method='HEAD', **kwargs)
198 +
199 +
200 +
class OPTIONS(HTTPStep):
201 +
202 +
    def __init__(self, url, **kwargs):
203 +
        super().__init__(url, method='OPTIONS', **kwargs)

@@ -280,6 +280,21 @@
Loading
280 280
        self._buildCompleteConsumer.stopConsuming()
281 281
        self._buildStartedConsumer.stopConsuming()
282 282
283 +
    @defer.inlineCallbacks
284 +
    def _got_event(self, key, msg):
285 +
        # This function is used only from tests
286 +
        if key[0] == 'builds':
287 +
            if key[2] == 'new':
288 +
                yield self.buildStarted(key, msg)
289 +
                return
290 +
            elif key[2] == 'finished':
291 +
                yield self.buildComplete(key, msg)
292 +
                return
293 +
        if key[0] == 'buildsets' and key[2] == 'complete':  # pragma: no cover
294 +
            yield self.buildsetComplete(key, msg)
295 +
            return
296 +
        raise Exception('Invalid key for _got_event: {}'.format(key))  # pragma: no cover
297 +
283 298
    @defer.inlineCallbacks
284 299
    def buildStarted(self, key, build):
285 300
        if self.startCB is None:

@@ -39,6 +39,10 @@
Loading
39 39
class BitbucketStatusPush(http.HttpStatusPushBase):
40 40
    name = "BitbucketStatusPush"
41 41
42 +
    def checkConfig(self, oauth_key, oauth_secret, base_url=_BASE_URL, oauth_url=_OAUTH_URL,
43 +
                    **kwargs):
44 +
        super().checkConfig(**kwargs)
45 +
42 46
    @defer.inlineCallbacks
43 47
    def reconfigService(self, oauth_key, oauth_secret,
44 48
                        base_url=_BASE_URL,

@@ -51,8 +51,8 @@
Loading
51 51
    try:
52 52
        hostname, port = master.split(":")
53 53
        port = int(port)
54 -
    except (TypeError, ValueError):
55 -
        raise usage.UsageError("master must have the form 'hostname:port'")
54 +
    except (TypeError, ValueError) as e:
55 +
        raise usage.UsageError("master must have the form 'hostname:port'") from e
56 56
57 57
58 58
class UpgradeMasterOptions(base.BasedirMixin, base.SubcommandOptions):
@@ -155,16 +155,16 @@
Loading
155 155
        else:
156 156
            try:
157 157
                self['log-count'] = int(self['log-count'])
158 -
            except ValueError:
158 +
            except ValueError as e:
159 159
                raise usage.UsageError(
160 -
                    "log-count parameter needs to be an int or None")
160 +
                    "log-count parameter needs to be an int or None") from e
161 161
162 162
        # validate 'db' parameter
163 163
        try:
164 164
            # check if sqlalchemy will be able to parse specified URL
165 165
            sa.engine.url.make_url(self['db'])
166 -
        except sa.exc.ArgumentError:
167 -
            raise usage.UsageError("could not parse database URL '{}'".format(self['db']))
166 +
        except sa.exc.ArgumentError as e:
167 +
            raise usage.UsageError("could not parse database URL '{}'".format(self['db'])) from e
168 168
169 169
170 170
class StopOptions(base.BasedirMixin, base.SubcommandOptions):
@@ -286,8 +286,8 @@
Loading
286 286
        if self.get('when'):
287 287
            try:
288 288
                self['when'] = float(self['when'])
289 -
            except (TypeError, ValueError):
290 -
                raise usage.UsageError('invalid "when" value {}'.format(self['when']))
289 +
            except (TypeError, ValueError) as e:
290 +
                raise usage.UsageError('invalid "when" value {}'.format(self['when'])) from e
291 291
        else:
292 292
            self['when'] = None
293 293

@@ -0,0 +1,176 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
import os
17 +
import re
18 +
19 +
from twisted.internet import defer
20 +
from twisted.internet import error
21 +
from twisted.internet import reactor
22 +
from twisted.internet.protocol import ProcessProtocol
23 +
from twisted.python import runtime
24 +
25 +
from buildbot.process.buildstep import FAILURE
26 +
from buildbot.process.buildstep import SUCCESS
27 +
from buildbot.process.buildstep import BuildStep
28 +
from buildbot.util import deferwaiter
29 +
30 +
31 +
class MasterShellCommand(BuildStep):
32 +
33 +
    """
34 +
    Run a shell command locally - on the buildmaster.  The shell command
35 +
    COMMAND is specified just as for a RemoteShellCommand.  Note that extra
36 +
    logfiles are not supported.
37 +
    """
38 +
    name = 'MasterShellCommand'
39 +
    description = 'Running'
40 +
    descriptionDone = 'Ran'
41 +
    descriptionSuffix = None
42 +
    renderables = ['command', 'env']
43 +
    haltOnFailure = True
44 +
    flunkOnFailure = True
45 +
46 +
    def __init__(self, command, **kwargs):
47 +
        self.env = kwargs.pop('env', None)
48 +
        self.usePTY = kwargs.pop('usePTY', 0)
49 +
        self.interruptSignal = kwargs.pop('interruptSignal', 'KILL')
50 +
        self.logEnviron = kwargs.pop('logEnviron', True)
51 +
52 +
        super().__init__(**kwargs)
53 +
54 +
        self.command = command
55 +
        self.masterWorkdir = self.workdir
56 +
        self._deferwaiter = deferwaiter.DeferWaiter()
57 +
        self._status_object = None
58 +
59 +
    class LocalPP(ProcessProtocol):
60 +
61 +
        def __init__(self, step):
62 +
            self.step = step
63 +
            self._finish_d = defer.Deferred()
64 +
            self.step._deferwaiter.add(self._finish_d)
65 +
66 +
        def outReceived(self, data):
67 +
            self.step._deferwaiter.add(self.step.stdio_log.addStdout(data))
68 +
69 +
        def errReceived(self, data):
70 +
            self.step._deferwaiter.add(self.step.stdio_log.addStderr(data))
71 +
72 +
        def processEnded(self, status_object):
73 +
            if status_object.value.exitCode is not None:
74 +
                msg = "exit status {}\n".format(status_object.value.exitCode)
75 +
                self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg))
76 +
77 +
            if status_object.value.signal is not None:
78 +
                msg = "signal {}\n".format(status_object.value.signal)
79 +
                self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg))
80 +
81 +
            self.step._status_object = status_object
82 +
            self._finish_d.callback(None)
83 +
84 +
    @defer.inlineCallbacks
85 +
    def run(self):
86 +
        # render properties
87 +
        command = self.command
88 +
        # set up argv
89 +
        if isinstance(command, (str, bytes)):
90 +
            if runtime.platformType == 'win32':
91 +
                # allow %COMSPEC% to have args
92 +
                argv = os.environ['COMSPEC'].split()
93 +
                if '/c' not in argv:
94 +
                    argv += ['/c']
95 +
                argv += [command]
96 +
            else:
97 +
                # for posix, use /bin/sh. for other non-posix, well, doesn't
98 +
                # hurt to try
99 +
                argv = ['/bin/sh', '-c', command]
100 +
        else:
101 +
            if runtime.platformType == 'win32':
102 +
                # allow %COMSPEC% to have args
103 +
                argv = os.environ['COMSPEC'].split()
104 +
                if '/c' not in argv:
105 +
                    argv += ['/c']
106 +
                argv += list(command)
107 +
            else:
108 +
                argv = command
109 +
110 +
        self.stdio_log = yield self.addLog("stdio")
111 +
112 +
        if isinstance(command, (str, bytes)):
113 +
            yield self.stdio_log.addHeader(command.strip() + "\n\n")
114 +
        else:
115 +
            yield self.stdio_log.addHeader(" ".join(command) + "\n\n")
116 +
        yield self.stdio_log.addHeader("** RUNNING ON BUILDMASTER **\n")
117 +
        yield self.stdio_log.addHeader(" in dir {}\n".format(os.getcwd()))
118 +
        yield self.stdio_log.addHeader(" argv: {}\n".format(argv))
119 +
120 +
        if self.env is None:
121 +
            env = os.environ
122 +
        else:
123 +
            assert isinstance(self.env, dict)
124 +
            env = self.env
125 +
            for key, v in self.env.items():
126 +
                if isinstance(v, list):
127 +
                    # Need to do os.pathsep translation.  We could either do that
128 +
                    # by replacing all incoming ':'s with os.pathsep, or by
129 +
                    # accepting lists.  I like lists better.
130 +
                    # If it's not a string, treat it as a sequence to be
131 +
                    # turned in to a string.
132 +
                    self.env[key] = os.pathsep.join(self.env[key])
133 +
134 +
            # do substitution on variable values matching pattern: ${name}
135 +
            p = re.compile(r'\${([0-9a-zA-Z_]*)}')
136 +
137 +
            def subst(match):
138 +
                return os.environ.get(match.group(1), "")
139 +
            newenv = {}
140 +
            for key, v in env.items():
141 +
                if v is not None:
142 +
                    if not isinstance(v, (str, bytes)):
143 +
                        raise RuntimeError(("'env' values must be strings or "
144 +
                                            "lists; key '{}' is incorrect").format(key))
145 +
                    newenv[key] = p.sub(subst, env[key])
146 +
            env = newenv
147 +
148 +
        if self.logEnviron:
149 +
            yield self.stdio_log.addHeader(" env: %r\n" % (env,))
150 +
151 +
        # TODO add a timeout?
152 +
        self.process = reactor.spawnProcess(self.LocalPP(self), argv[0], argv,
153 +
                                            path=self.masterWorkdir, usePTY=self.usePTY, env=env)
154 +
155 +
        # self._deferwaiter will yield only after LocalPP finishes
156 +
157 +
        yield self._deferwaiter.wait()
158 +
159 +
        status_value = self._status_object.value
160 +
        if status_value.signal is not None:
161 +
            self.descriptionDone = ["killed ({})".format(status_value.signal)]
162 +
            return FAILURE
163 +
        elif status_value.exitCode != 0:
164 +
            self.descriptionDone = ["failed ({})".format(status_value.exitCode)]
165 +
            return FAILURE
166 +
        else:
167 +
            return SUCCESS
168 +
169 +
    def interrupt(self, reason):
170 +
        try:
171 +
            self.process.signalProcess(self.interruptSignal)
172 +
        except KeyError:  # Process not started yet
173 +
            pass
174 +
        except error.ProcessExitedAlready:
175 +
            pass
176 +
        super().interrupt(reason)

@@ -56,7 +56,7 @@
Loading
56 56
57 57
        self.cmake = cmake
58 58
        kwargs = self.setupShellMixin(kwargs, prohibitArgs=['command'])
59 -
        super(CMake, self).__init__(**kwargs)
59 +
        super().__init__(**kwargs)
60 60
61 61
    @defer.inlineCallbacks
62 62
    def run(self):

@@ -104,9 +104,9 @@
Loading
104 104
        if self._secret and signature:
105 105
            try:
106 106
                hash_type, hexdigest = signature.split('=')
107 -
            except ValueError:
107 +
            except ValueError as e:
108 108
                raise ValueError(
109 -
                    'Wrong signature format: {}'.format(signature))
109 +
                    'Wrong signature format: {}'.format(signature)) from e
110 110
111 111
            if hash_type != 'sha1':
112 112
                raise ValueError('Unknown hash type: {}'.format(hash_type))

@@ -58,11 +58,12 @@
Loading
58 58
        return d
59 59
60 60
61 -
class ChangesEndpoint(FixerMixin, base.Endpoint):
61 +
class ChangesEndpoint(FixerMixin, base.BuildNestingMixin, base.Endpoint):
62 62
63 63
    isCollection = True
64 64
    pathPatterns = """
65 65
        /changes
66 +
        /builders/n:builderid/builds/n:build_number/changes
66 67
        /builds/n:buildid/changes
67 68
        /sourcestamps/n:ssid/changes
68 69
    """
@@ -71,6 +72,8 @@
Loading
71 72
    @defer.inlineCallbacks
72 73
    def get(self, resultSpec, kwargs):
73 74
        buildid = kwargs.get('buildid')
75 +
        if 'build_number' in kwargs:
76 +
            buildid = yield self.getBuildid(kwargs)
74 77
        ssid = kwargs.get('ssid')
75 78
        if buildid is not None:
76 79
            changes = yield self.master.db.changes.getChangesForBuild(buildid)

@@ -50,9 +50,8 @@
Loading
50 50
51 51
        msg = yield self.formatter.formatMessageForMissingWorker(master, worker)
52 52
        body = msg['body'].encode(ENCODING)
53 -
        if 'subject' in msg:
54 -
            subject = msg['subject']
55 -
        else:
53 +
        subject = msg['subject']
54 +
        if subject is None:
56 55
            subject = "Buildbot worker {name} missing".format(**worker)
57 56
        assert msg['type'] in ('plain', 'html'), \
58 57
            "'{}' message type must be 'plain' or 'html'.".format(msg['type'])

@@ -29,7 +29,7 @@
Loading
29 29
    renderables = ['secret_to_be_populated']
30 30
31 31
    def __init__(self, populated_secret_list, **kwargs):
32 -
        super(DownloadSecretsToWorker, self).__init__(**kwargs)
32 +
        super().__init__(**kwargs)
33 33
        self.secret_to_be_populated = populated_secret_list
34 34
35 35
    @defer.inlineCallbacks
@@ -58,7 +58,7 @@
Loading
58 58
        for path, secret in populated_secret_list:
59 59
            self.paths.append(path)
60 60
        self.logEnviron = logEnviron
61 -
        super(RemoveWorkerFileSecret, self).__init__(**kwargs)
61 +
        super().__init__(**kwargs)
62 62
63 63
    @defer.inlineCallbacks
64 64
    def runRemoveWorkerFileSecret(self):

@@ -67,9 +67,9 @@
Loading
67 67
68 68
    def patched_init(self, *args):
69 69
        if isinstance(args[0], int):
70 -
            super(Warning, self).__init__("{} {}".format(args[0], args[1]))
70 +
            super().__init__("{} {}".format(args[0], args[1]))
71 71
        else:
72 -
            super(Warning, self).__init__(*args)
72 +
            super().__init__(*args)
73 73
    Warning.__init__ = patched_init
74 74
75 75

@@ -44,7 +44,7 @@
Loading
44 44
45 45
    name = 'p4'
46 46
47 -
    renderables = ['mode', 'p4base', 'p4client', 'p4viewspec', 'p4branch']
47 +
    renderables = ['mode', 'p4base', 'p4client', 'p4viewspec', 'p4branch', 'p4passwd']
48 48
    possible_modes = ('incremental', 'full')
49 49
50 50
    def __init__(self, mode='incremental',
@@ -376,11 +376,11 @@
Loading
376 376
        revision = stdout.split()[1]
377 377
        try:
378 378
            int(revision)
379 -
        except ValueError:
379 +
        except ValueError as e:
380 380
            msg = (("p4.parseGotRevision unable to parse output "
381 381
                    "of 'p4 changes -m1 \"#have\"': '{}'").format(stdout))
382 382
            log.msg(msg)
383 -
            raise buildstep.BuildStepFailed()
383 +
            raise buildstep.BuildStepFailed() from e
384 384
385 385
        if self.debug:
386 386
            log.msg("Got p4 revision {}".format(revision))
@@ -399,7 +399,7 @@
Loading
399 399
400 400
    @defer.inlineCallbacks
401 401
    def checkP4(self):
402 -
        cmd = buildstep.RemoteShellCommand(self.workdir, ['p4', '-V'],
402 +
        cmd = buildstep.RemoteShellCommand(self.workdir, [self.p4bin, '-V'],
403 403
                                           env=self.env,
404 404
                                           logEnviron=self.logEnviron)
405 405
        cmd.useLog(self.stdio_log, False)

@@ -22,7 +22,7 @@
Loading
22 22
from buildbot.process.results import FAILURE
23 23
from buildbot.process.results import SUCCESS
24 24
from buildbot.process.results import WARNINGS
25 -
from buildbot.process.results import Results
25 +
from buildbot.process.results import statusToString
26 26
from buildbot.reporters.message import MessageFormatter as DefaultMessageFormatter
27 27
28 28
@@ -55,6 +55,15 @@
Loading
55 55
        if '\n' in self.subject:
56 56
            config.error('Newlines are not allowed in message subjects')
57 57
58 +
        list_or_none_params = [
59 +
            ('tags', self.tags),
60 +
            ('builders', self.builders),
61 +
            ('schedulers', self.schedulers),
62 +
            ('branches', self.branches),
63 +
        ]
64 +
        for name, param in list_or_none_params:
65 +
            self._verify_list_or_none_param(name, param)
66 +
58 67
        # you should either limit on builders or tags, not both
59 68
        if self.builders is not None and self.tags is not None:
60 69
            config.error("Please specify only builders or tags to include - not both.")
@@ -85,22 +94,24 @@
Loading
85 94
86 95
        return False
87 96
88 -
    def is_message_needed(self, build):
97 +
    def is_message_needed_by_props(self, build):
89 98
        # here is where we actually do something.
90 99
        builder = build['builder']
91 100
        scheduler = build['properties'].get('scheduler', [None])[0]
92 101
        branch = build['properties'].get('branch', [None])[0]
93 -
        results = build['results']
94 102
95 103
        if self.builders is not None and builder['name'] not in self.builders:
96 -
            return False  # ignore this build
104 +
            return False
97 105
        if self.schedulers is not None and scheduler not in self.schedulers:
98 -
            return False  # ignore this build
106 +
            return False
99 107
        if self.branches is not None and branch not in self.branches:
100 -
            return False  # ignore this build
108 +
            return False
101 109
        if self.tags is not None and not self._matches_any_tag(builder['tags']):
102 -
            return False  # ignore this build
110 +
            return False
111 +
        return True
103 112
113 +
    def is_message_needed_by_results(self, build):
114 +
        results = build['results']
104 115
        if "change" in self.mode:
105 116
            prev = build['prev_build']
106 117
            if prev and prev['results'] != results:
@@ -124,9 +135,10 @@
Loading
124 135
125 136
    @defer.inlineCallbacks
126 137
    def build_message(self, master, reporter, name, builds, results):
138 +
        # The given builds must refer to builds from a single buildset
127 139
        patches = []
128 140
        logs = []
129 -
        body = ""
141 +
        body = None
130 142
        subject = None
131 143
        msgtype = None
132 144
        users = set()
@@ -144,21 +156,22 @@
Loading
144 156
                    build_logs = [log for log in build_logs if self._should_attach_log(log)]
145 157
                logs.extend(build_logs)
146 158
147 -
            if 'prev_build' in build and build['prev_build'] is not None:
148 -
                previous_results = build['prev_build']['results']
149 -
            else:
150 -
                previous_results = None
151 159
            blamelist = yield reporter.getResponsibleUsersForBuild(master, build['buildid'])
152 -
            buildmsg = yield self.formatter.formatMessageForBuildResults(
153 -
                self.mode, name, build['buildset'], build, master, previous_results, blamelist)
160 +
            buildmsg = yield self.formatter.format_message_for_build(self.mode, name, build,
161 +
                                                                     master, blamelist)
154 162
            users.update(set(blamelist))
155 163
            msgtype = buildmsg['type']
156 -
            body += buildmsg['body']
157 -
            if 'subject' in buildmsg:
164 +
165 +
            if body is None:
166 +
                body = buildmsg['body']
167 +
            elif buildmsg['body'] is not None:
168 +
                body = body + buildmsg['body']
169 +
170 +
            if buildmsg['subject'] is not None:
158 171
                subject = buildmsg['subject']
159 172
160 173
        if subject is None:
161 -
            subject = self.subject % {'result': Results[results],
174 +
            subject = self.subject % {'result': statusToString(results),
162 175
                                      'projectName': master.config.title,
163 176
                                      'title': master.config.title,
164 177
                                      'builder': name}
@@ -196,6 +209,10 @@
Loading
196 209
                else:
197 210
                    config.error("mode {} is not a valid mode".format(m))
198 211
212 +
    def _verify_list_or_none_param(self, name, param):
213 +
        if param is not None and not isinstance(param, list):
214 +
            config.error("{} must be a list or None".format(name))
215 +
199 216
    def _compute_shortcut_modes(self, mode):
200 217
        if isinstance(mode, str):
201 218
            if mode == "all":

@@ -96,11 +96,11 @@
Loading
96 96
            for pn in filtered_prop_names:
97 97
                try:
98 98
                    ret_val = self._callback(properties, pn)
99 -
                except KeyError:
99 +
                except KeyError as e:
100 100
                    raise CaptureCallbackError(("CaptureProperty failed."
101 101
                                                " The property {} not found for build number {} on"
102 102
                                                " builder {}.").format(pn, msg['number'],
103 -
                                                                       builder_info['name']))
103 +
                                                                       builder_info['name'])) from e
104 104
                context = self._defaultContext(msg, builder_info['name'])
105 105
                series_name = "{}-{}".format(builder_info['name'], pn)
106 106
                post_data = {
@@ -170,7 +170,7 @@
Loading
170 170
                # it
171 171
                raise CaptureCallbackError("{} Exception raised: {} with message: {}".format(
172 172
                                                    self._err_msg(msg, builder_info['name']),
173 -
                                                    type(e).__name__, str(e)))
173 +
                                                    type(e).__name__, str(e))) from e
174 174
175 175
            context = self._defaultContext(msg, builder_info['name'])
176 176
            post_data = {
@@ -351,7 +351,7 @@
Loading
351 351
                raise CaptureCallbackError(("CaptureData failed for build {} of builder {}."
352 352
                                            " Exception generated: {} with message {}"
353 353
                                            ).format(build_data['number'], builder_info['name'],
354 -
                                              type(e).__name__, str(e)))
354 +
                                              type(e).__name__, str(e))) from e
355 355
            post_data = ret_val
356 356
            series_name = '{}-{}'.format(builder_info['name'], self._data_name)
357 357
            context = self._defaultContext(build_data, builder_info['name'])

@@ -56,8 +56,8 @@
Loading
56 56
        type = str(type)
57 57
        try:
58 58
            subcls = cls._byType[type]
59 -
        except KeyError:
60 -
            raise RuntimeError("Invalid log type %r" % (type,))
59 +
        except KeyError as e:
60 +
            raise RuntimeError("Invalid log type %r" % (type,)) from e
61 61
        decoder = Log._decoderFromString(logEncoding)
62 62
        return subcls(master, name, type, logid, decoder)
63 63
@@ -124,7 +124,7 @@
Loading
124 124
class PlainLog(Log):
125 125
126 126
    def __init__(self, master, name, type, logid, decoder):
127 -
        super(PlainLog, self).__init__(master, name, type, logid, decoder)
127 +
        super().__init__(master, name, type, logid, decoder)
128 128
129 129
        def wholeLines(lines):
130 130
            self.subPoint.deliver(None, lines)
@@ -140,7 +140,7 @@
Loading
140 140
    @defer.inlineCallbacks
141 141
    def finish(self):
142 142
        yield self.lbf.flush()
143 -
        yield super(PlainLog, self).finish()
143 +
        yield super().finish()
144 144
145 145
146 146
class TextLog(PlainLog):
@@ -164,7 +164,7 @@
Loading
164 164
    pat = re.compile('^', re.M)
165 165
166 166
    def __init__(self, step, name, type, logid, decoder):
167 -
        super(StreamLog, self).__init__(step, name, type, logid, decoder)
167 +
        super().__init__(step, name, type, logid, decoder)
168 168
        self.lbfs = {}
169 169
170 170
    def _getLbf(self, stream):
@@ -200,7 +200,7 @@
Loading
200 200
    def finish(self):
201 201
        for lbf in self.lbfs.values():
202 202
            yield lbf.flush()
203 -
        yield super(StreamLog, self).finish()
203 +
        yield super().finish()
204 204
205 205
206 206
Log._byType['s'] = StreamLog

@@ -177,7 +177,7 @@
Loading
177 177
            content = request.content.read()
178 178
            payload = json.loads(bytes2unicode(content))
179 179
        except Exception as e:
180 -
            raise ValueError("Error loading JSON: " + str(e))
180 +
            raise ValueError("Error loading JSON: " + str(e)) from e
181 181
        event_type = request.getHeader(_HEADER_EVENT)
182 182
        event_type = bytes2unicode(event_type)
183 183
        # newer version of gitlab have a object_kind parameter,

@@ -41,7 +41,7 @@
Loading
41 41
        This is not supported. Please start from a clean database
42 42
        http://docs.buildbot.net/latest/manual/installation/nine-upgrade.html"""
43 43
        # Call the base class constructor with the parameters it needs
44 -
        super(EightUpgradeError, self).__init__(message)
44 +
        super().__init__(message)
45 45
46 46
47 47
class Model(base.DBConnectorComponent):

@@ -51,8 +51,8 @@
Loading
51 51
        try:
52 52
            with open(pidfile) as f:
53 53
                pid = int(f.read())
54 -
        except ValueError:
55 -
            raise ValueError('Pidfile {} contains non-numeric value'.format(pidfile))
54 +
        except ValueError as e:
55 +
            raise ValueError('Pidfile {} contains non-numeric value'.format(pidfile)) from e
56 56
        try:
57 57
            os.kill(pid, 0)
58 58
        except OSError as why:
@@ -62,7 +62,7 @@
Loading
62 62
                os.remove(pidfile)
63 63
            else:
64 64
                raise OSError("Can't check status of PID {} from pidfile {}: {}".format(
65 -
                    pid, pidfile, why))
65 +
                    pid, pidfile, why)) from why
66 66
        else:
67 67
            raise BusyError("'{}' exists - is this master still running?".format(pidfile))
68 68

@@ -13,11 +13,14 @@
Loading
13 13
#
14 14
# Copyright Buildbot Team Members
15 15
16 +
import re
16 17
from urllib.parse import urlparse
17 18
18 19
from twisted.internet import defer
19 20
from twisted.python import log
20 21
22 +
from buildbot import config
23 +
from buildbot.plugins import util
21 24
from buildbot.process.properties import Interpolate
22 25
from buildbot.process.properties import Properties
23 26
from buildbot.process.results import SUCCESS
@@ -34,6 +37,7 @@
Loading
34 37
SUCCESSFUL = 'SUCCESSFUL'
35 38
FAILED = 'FAILED'
36 39
STATUS_API_URL = '/rest/build-status/1.0/commits/{sha}'
40 +
STATUS_CORE_API_URL = '/rest/api/1.0/projects/{proj_key}/repos/{repo_slug}/commits/{sha}/builds'
37 41
COMMENT_API_URL = '/rest/api/1.0{path}/comments'
38 42
HTTP_PROCESSED = 204
39 43
HTTP_CREATED = 201
@@ -42,6 +46,11 @@
Loading
42 46
class BitbucketServerStatusPush(http.HttpStatusPushBase):
43 47
    name = "BitbucketServerStatusPush"
44 48
49 +
    def checkConfig(self, base_url, user, password, key=None, statusName=None,
50 +
                    startDescription=None, endDescription=None, verbose=False,
51 +
                    **kwargs):
52 +
        super().checkConfig(wantProperties=True, **kwargs)
53 +
45 54
    @defer.inlineCallbacks
46 55
    def reconfigService(self, base_url, user, password, key=None,
47 56
                        statusName=None, startDescription=None,
@@ -125,6 +134,208 @@
Loading
125 134
                    ))
126 135
127 136
137 +
class BitbucketServerCoreAPIStatusPush(http.HttpStatusPushBase):
138 +
    name = "BitbucketServerCoreAPIStatusPush"
139 +
    secrets = ["token", "auth"]
140 +
141 +
    def checkConfig(self, base_url, token=None, auth=None,
142 +
                    statusName=None, statusSuffix=None, startDescription=None,
143 +
                    endDescription=None, key=None, parentName=None,
144 +
                    buildNumber=None, ref=None, duration=None,
145 +
                    testResults=None, verbose=False, debug=None,
146 +
                    verify=None, **kwargs):
147 +
        if not base_url:
148 +
            config.error("Parameter base_url has to be given")
149 +
        if token is not None and auth is not None:
150 +
            config.error("Only one authentication method can be given "
151 +
                         "(token or auth)")
152 +
        super().checkConfig(wantProperties=True, **kwargs)
153 +
154 +
    @defer.inlineCallbacks
155 +
    def reconfigService(self, base_url, token=None, auth=None,
156 +
                        statusName=None, statusSuffix=None, startDescription=None,
157 +
                        endDescription=None, key=None, parentName=None,
158 +
                        buildNumber=None, ref=None, duration=None,
159 +
                        testResults=None, verbose=False, debug=None,
160 +
                        verify=None, **kwargs):
161 +
        yield super().reconfigService(wantProperties=True, **kwargs)
162 +
        self.status_name = statusName
163 +
        self.status_suffix = statusSuffix
164 +
        self.start_description = startDescription or 'Build started.'
165 +
        self.end_description = endDescription or 'Build done.'
166 +
        self.key = key or Interpolate('%(prop:buildername)s')
167 +
        self.parent_name = parentName
168 +
        self.build_number = buildNumber or Interpolate('%(prop:buildnumber)s')
169 +
        self.ref = ref
170 +
        self.duration = duration
171 +
172 +
        if testResults:
173 +
            self.test_results = testResults
174 +
        else:
175 +
            @util.renderer
176 +
            def r_testresults(props):
177 +
                failed = props.getProperty("tests_failed", 0)
178 +
                skipped = props.getProperty("tests_skipped", 0)
179 +
                successful = props.getProperty("tests_successful", 0)
180 +
                if any([failed, skipped, successful]):
181 +
                    return {
182 +
                        "failed": failed,
183 +
                        "skipped": skipped,
184 +
                        "successful": successful
185 +
                    }
186 +
                return None
187 +
            self.test_results = r_testresults
188 +
189 +
        self.verbose = verbose
190 +
        headers = {}
191 +
        if token:
192 +
            headers["Authorization"] = "Bearer {}".format(token)
193 +
        self._http = yield httpclientservice.HTTPClientService.getService(
194 +
            self.master, base_url, auth=auth, headers=headers, debug=debug,
195 +
            verify=verify)
196 +
197 +
    def createStatus(self, proj_key, repo_slug, sha, state, url, key, parent,
198 +
                     build_number, ref, description, name, duration,
199 +
                     test_results):
200 +
        payload = {
201 +
            'state': state,
202 +
            'url': url,
203 +
            'key': key,
204 +
            'parent': parent,
205 +
            'ref': ref,
206 +
            'buildNumber': build_number,
207 +
            'description': description,
208 +
            'name': name,
209 +
            'duration': duration,
210 +
            'testResults': test_results
211 +
        }
212 +
213 +
        if self.verbose:
214 +
            log.msg("Sending payload: '{}' for {}/{} {}.".format(
215 +
                payload, proj_key, repo_slug, sha
216 +
            ))
217 +
218 +
        _url = STATUS_CORE_API_URL.format(proj_key=proj_key, repo_slug=repo_slug,
219 +
                                          sha=sha)
220 +
        return self._http.post(_url, json=payload)
221 +
222 +
    @defer.inlineCallbacks
223 +
    def send(self, build):
224 +
        props = Properties.fromDict(build['properties'])
225 +
        props.master = self.master
226 +
227 +
        duration = None
228 +
        test_results = None
229 +
        if build['complete']:
230 +
            state = SUCCESSFUL if build['results'] == SUCCESS else FAILED
231 +
            description = yield props.render(self.end_description)
232 +
            if self.duration:
233 +
                duration = yield props.render(self.duration)
234 +
            else:
235 +
                td = build['complete_at'] - build['started_at']
236 +
                duration = int(td.seconds * 1000)
237 +
            if self.test_results:
238 +
                test_results = yield props.render(self.test_results)
239 +
        else:
240 +
            state = INPROGRESS
241 +
            description = yield props.render(self.start_description)
242 +
            duration = None
243 +
244 +
        parent_name = (build['parentbuilder'] or {}).get('name')
245 +
        if self.parent_name:
246 +
            parent = yield props.render(self.parent_name)
247 +
        elif parent_name:
248 +
            parent = parent_name
249 +
        else:
250 +
            parent = build['builder']['name']
251 +
252 +
        if self.status_name:
253 +
            status_name = yield props.render(self.status_name)
254 +
        else:
255 +
            status_name = "{} #{}".format(props.getProperty("buildername"),
256 +
                                   props.getProperty("buildnumber"))
257 +
            if parent_name:
258 +
                status_name = "{} #{} \u00BB {}".format(
259 +
                    parent_name, build['parentbuild']['number'], status_name
260 +
                )
261 +
        if self.status_suffix:
262 +
            status_name = status_name + (yield props.render(self.status_suffix))
263 +
264 +
        key = yield props.render(self.key)
265 +
        build_number = yield props.render(self.build_number)
266 +
        url = build['url']
267 +
268 +
        sourcestamps = build['buildset']['sourcestamps']
269 +
270 +
        for sourcestamp in sourcestamps:
271 +
            try:
272 +
                ssid = sourcestamp.get('ssid')
273 +
                sha = sourcestamp.get('revision')
274 +
                branch = sourcestamp.get('branch')
275 +
                repo = sourcestamp.get('repository')
276 +
277 +
                if not sha:
278 +
                    log.msg("Unable to get the commit hash for SSID: "
279 +
                            "{}".format(ssid))
280 +
                    continue
281 +
282 +
                ref = None
283 +
                if self.ref is None:
284 +
                    if branch is not None:
285 +
                        if branch.startswith("refs/"):
286 +
                            ref = branch
287 +
                        else:
288 +
                            ref = "refs/heads/{}".format(branch)
289 +
                else:
290 +
                    ref = yield props.render(self.ref)
291 +
292 +
                if not ref:
293 +
                    log.msg("WARNING: Unable to resolve ref for SSID: {}. "
294 +
                            "Build status will not be visible on Builds or "
295 +
                            "PullRequest pages only for commits".format(ssid))
296 +
297 +
                r = re.search(r"^.*?/([^/]+)/([^/]+?)(?:\.git)?$", repo or "")
298 +
                if r:
299 +
                    proj_key = r.group(1)
300 +
                    repo_slug = r.group(2)
301 +
                else:
302 +
                    log.msg("Unable to parse repository info from '{}' for "
303 +
                            "SSID: {}".format(repo, ssid))
304 +
                    continue
305 +
306 +
                res = yield self.createStatus(
307 +
                    proj_key=proj_key,
308 +
                    repo_slug=repo_slug,
309 +
                    sha=sha,
310 +
                    state=state,
311 +
                    url=url,
312 +
                    key=key,
313 +
                    parent=parent,
314 +
                    build_number=build_number,
315 +
                    ref=ref,
316 +
                    description=description,
317 +
                    name=status_name,
318 +
                    duration=duration,
319 +
                    test_results=test_results
320 +
                )
321 +
322 +
                if res.code not in (HTTP_PROCESSED,):
323 +
                    content = yield res.content()
324 +
                    log.msg("{}: Unable to send Bitbucket Server status for "
325 +
                            "{}/{} {}: {}".format(res.code, proj_key, repo_slug,
326 +
                                                  sha, content))
327 +
                elif self.verbose:
328 +
                    log.msg('Status "{}" sent for {}/{} {}'.format(
329 +
                        state, proj_key, repo_slug, sha
330 +
                    ))
331 +
            except Exception as e:
332 +
                log.err(
333 +
                    e,
334 +
                    'Failed to send status "{}" for {}/{} {}'.format(
335 +
                        state, proj_key, repo_slug, sha
336 +
                    ))
337 +
338 +
128 339
class BitbucketServerPRCommentPush(notifier.NotifierBase):
129 340
    name = "BitbucketServerPRCommentPush"
130 341

@@ -154,8 +154,8 @@
Loading
154 154
        f.seek(0, 0)
155 155
        try:
156 156
            p.feed(f.read())
157 -
        except basic.NetstringParseError:
158 -
            raise BadJobfile("unable to parse netstrings")
157 +
        except basic.NetstringParseError as e:
158 +
            raise BadJobfile("unable to parse netstrings") from e
159 159
        if not p.strings:
160 160
            raise BadJobfile("could not find any complete netstrings")
161 161
        ver = bytes2unicode(p.strings.pop(0))
@@ -192,8 +192,8 @@
Loading
192 192
            try:
193 193
                data = bytes2unicode(p.strings[0])
194 194
                parsed_job = json.loads(data)
195 -
            except ValueError:
196 -
                raise BadJobfile("unable to parse JSON")
195 +
            except ValueError as e:
196 +
                raise BadJobfile("unable to parse JSON") from e
197 197
            postprocess_parsed_job()
198 198
        else:
199 199
            raise BadJobfile("unknown version '{}'".format(ver))

@@ -32,17 +32,166 @@
Loading
32 32
from buildbot.warnings import warn_deprecated
33 33
34 34
35 +
def get_detected_status_text(mode, results, previous_results):
36 +
    if results == FAILURE:
37 +
        if ('change' in mode or 'problem' in mode) and previous_results is not None \
38 +
                and previous_results != FAILURE:
39 +
            text = "new failure"
40 +
        else:
41 +
            text = "failed build"
42 +
    elif results == WARNINGS:
43 +
        text = "problem in the build"
44 +
    elif results == SUCCESS:
45 +
        if "change" in mode and previous_results is not None and previous_results != results:
46 +
            text = "restored build"
47 +
        else:
48 +
            text = "passing build"
49 +
    elif results == EXCEPTION:
50 +
        text = "build exception"
51 +
    else:
52 +
        text = "{} build".format(statusToString(results))
53 +
54 +
    return text
55 +
56 +
57 +
def get_message_summary_text(build, results):
58 +
    t = build['state_string']
59 +
    if t:
60 +
        t = ": " + t
61 +
    else:
62 +
        t = ""
63 +
64 +
    if results == SUCCESS:
65 +
        text = "Build succeeded!"
66 +
    elif results == WARNINGS:
67 +
        text = "Build Had Warnings{}".format(t)
68 +
    elif results == CANCELLED:
69 +
        text = "Build was cancelled"
70 +
    else:
71 +
        text = "BUILD FAILED{}".format(t)
72 +
73 +
    return text
74 +
75 +
76 +
def get_message_source_stamp_text(source_stamps):
77 +
    text = ""
78 +
79 +
    for ss in source_stamps:
80 +
        source = ""
81 +
82 +
        if ss['branch']:
83 +
            source += "[branch {}] ".format(ss['branch'])
84 +
85 +
        if ss['revision']:
86 +
            source += str(ss['revision'])
87 +
        else:
88 +
            source += "HEAD"
89 +
90 +
        if ss['patch'] is not None:
91 +
            source += " (plus patch)"
92 +
93 +
        discriminator = ""
94 +
        if ss['codebase']:
95 +
            discriminator = " '{}'".format(ss['codebase'])
96 +
97 +
        text += "Build Source Stamp{}: {}\n".format(discriminator, source)
98 +
99 +
    return text
100 +
101 +
102 +
def get_projects_text(source_stamps, master):
103 +
    projects = set()
104 +
105 +
    for ss in source_stamps:
106 +
        if ss['project']:
107 +
            projects.add(ss['project'])
108 +
109 +
    if not projects:
110 +
        projects = [master.config.title]
111 +
112 +
    return ', '.join(list(projects))
113 +
114 +
115 +
def create_context_for_build(mode, buildername, build, master, blamelist):
116 +
    buildset = build['buildset']
117 +
    ss_list = buildset['sourcestamps']
118 +
    results = build['results']
119 +
120 +
    if 'prev_build' in build and build['prev_build'] is not None:
121 +
        previous_results = build['prev_build']['results']
122 +
    else:
123 +
        previous_results = None
124 +
125 +
    return {
126 +
        'results': build['results'],
127 +
        'mode': mode,
128 +
        'buildername': buildername,
129 +
        'workername': build['properties'].get('workername', ["<unknown>"])[0],
130 +
        'buildset': buildset,
131 +
        'build': build,
132 +
        'projects': get_projects_text(ss_list, master),
133 +
        'previous_results': previous_results,
134 +
        'status_detected': get_detected_status_text(mode, results, previous_results),
135 +
        'build_url': utils.getURLForBuild(master, build['builder']['builderid'], build['number']),
136 +
        'buildbot_url': master.config.buildbotURL,
137 +
        'blamelist': blamelist,
138 +
        'summary': get_message_summary_text(build, results),
139 +
        'sourcestamps': get_message_source_stamp_text(ss_list)
140 +
    }
141 +
142 +
143 +
def create_context_for_worker(master, worker):
144 +
    return {
145 +
        'buildbot_title': master.config.title,
146 +
        'buildbot_url': master.config.buildbotURL,
147 +
        'worker': worker,
148 +
    }
149 +
150 +
35 151
class MessageFormatterBase(util.ComparableMixin):
36 -
    template_filename = 'default_mail.txt'
152 +
37 153
    template_type = 'plain'
38 154
39 -
    compare_attrs = ['body_template', 'subject_teblate', 'template_type']
155 +
    def __init__(self, ctx=None, wantProperties=True, wantSteps=False, wantLogs=False):
156 +
        if ctx is None:
157 +
            ctx = {}
158 +
        self.context = ctx
159 +
        self.wantProperties = wantProperties
160 +
        self.wantSteps = wantSteps
161 +
        self.wantLogs = wantLogs
162 +
163 +
    def buildAdditionalContext(self, master, ctx):
164 +
        pass
165 +
166 +
    @defer.inlineCallbacks
167 +
    def render_message_dict(self, master, context):
168 +
        """Generate a buildbot reporter message and return a dictionary
169 +
           containing the message body, type and subject."""
170 +
        yield self.buildAdditionalContext(master, context)
171 +
        context.update(self.context)
172 +
173 +
        return {
174 +
            'body': self.render_message_body(context),
175 +
            'type': self.template_type,
176 +
            'subject': self.render_message_subject(context)
177 +
        }
178 +
179 +
    def render_message_body(self, context):
180 +
        return None
181 +
182 +
    def render_message_subject(self, context):
183 +
        return None
184 +
185 +
186 +
class MessageFormatterBaseJinja(MessageFormatterBase):
187 +
    template_filename = 'default_mail.txt'
188 +
189 +
    compare_attrs = ['body_template', 'subject_template', 'template_type']
40 190
41 191
    def __init__(self, template_dir=None,
42 192
                 template_filename=None, template=None,
43 193
                 subject_filename=None, subject=None,
44 -
                 template_type=None, ctx=None,
45 -
                 ):
194 +
                 template_type=None, **kwargs):
46 195
        self.body_template = self.getTemplate(template_filename, template_dir, template)
47 196
        self.subject_template = None
48 197
        if subject_filename or subject:
@@ -51,10 +200,7 @@
Loading
51 200
        if template_type is not None:
52 201
            self.template_type = template_type
53 202
54 -
        if ctx is None:
55 -
            ctx = {}
56 -
57 -
        self.ctx = ctx
203 +
        super().__init__(**kwargs)
58 204
59 205
    def getTemplate(self, filename, dirname, content):
60 206
        if content and (filename or dirname):
@@ -76,158 +222,41 @@
Loading
76 222
        return env.get_template(filename)
77 223
78 224
    def buildAdditionalContext(self, master, ctx):
79 -
        ctx.update(self.ctx)
225 +
        pass
80 226
81 -
    def renderMessage(self, ctx):
82 -
        body = self.body_template.render(ctx)
83 -
        msgdict = {'body': body, 'type': self.template_type}
84 -
        if self.subject_template is not None:
85 -
            msgdict['subject'] = self.subject_template.render(ctx)
86 -
        return msgdict
227 +
    def render_message_body(self, context):
228 +
        return self.body_template.render(context)
87 229
230 +
    def render_message_subject(self, context):
231 +
        if self.subject_template is None:
232 +
            return None
233 +
        return self.subject_template.render(context)
88 234
89 -
class MessageFormatter(MessageFormatterBase):
235 +
236 +
class MessageFormatter(MessageFormatterBaseJinja):
90 237
    template_filename = 'default_mail.txt'
91 -
    template_type = 'plain'
92 238
93 239
    compare_attrs = ['wantProperties', 'wantSteps', 'wantLogs']
94 240
95 -
    def __init__(self, template_dir=None,
96 -
                 template_filename=None, template=None, template_name=None,
97 -
                 subject_filename=None, subject=None,
98 -
                 template_type=None, ctx=None,
99 -
                 wantProperties=True, wantSteps=False, wantLogs=False):
241 +
    def __init__(self, template_name=None, **kwargs):
100 242
101 243
        if template_name is not None:
102 244
            warn_deprecated('0.9.1', "template_name is deprecated, use template_filename")
103 -
            template_filename = template_name
104 -
        super().__init__(template_dir=template_dir,
105 -
                         template_filename=template_filename,
106 -
                         template=template,
107 -
                         subject_filename=subject_filename,
108 -
                         subject=subject,
109 -
                         template_type=template_type, ctx=ctx)
110 -
        self.wantProperties = wantProperties
111 -
        self.wantSteps = wantSteps
112 -
        self.wantLogs = wantLogs
113 -
114 -
    def getDetectedStatus(self, mode, results, previous_results):
115 -
116 -
        if results == FAILURE:
117 -
            if "change" in mode and previous_results is not None and previous_results != results:
118 -
                text = "new failure"
119 -
            elif "problem" in mode and previous_results and previous_results != FAILURE:
120 -
                text = "new failure"
121 -
            else:
122 -
                text = "failed build"
123 -
        elif results == WARNINGS:
124 -
            text = "problem in the build"
125 -
        elif results == SUCCESS:
126 -
            if "change" in mode and previous_results is not None and previous_results != results:
127 -
                text = "restored build"
128 -
            else:
129 -
                text = "passing build"
130 -
        elif results == EXCEPTION:
131 -
            text = "build exception"
132 -
        else:
133 -
            text = "{} build".format(statusToString(results))
134 -
135 -
        return text
136 -
137 -
    def getProjects(self, source_stamps, master):
138 -
        projects = set()
139 -
140 -
        for ss in source_stamps:
141 -
            if ss['project']:
142 -
                projects.add(ss['project'])
143 -
144 -
        if not projects:
145 -
            projects = [master.config.title]
146 -
147 -
        return ', '.join(list(projects))
148 -
149 -
    def messageSourceStamps(self, source_stamps):
150 -
        text = ""
151 -
152 -
        for ss in source_stamps:
153 -
            source = ""
154 -
155 -
            if ss['branch']:
156 -
                source += "[branch {}] ".format(ss['branch'])
157 -
158 -
            if ss['revision']:
159 -
                source += str(ss['revision'])
160 -
            else:
161 -
                source += "HEAD"
162 -
163 -
            if ss['patch'] is not None:
164 -
                source += " (plus patch)"
165 -
166 -
            discriminator = ""
167 -
            if ss['codebase']:
168 -
                discriminator = " '{}'".format(ss['codebase'])
169 -
170 -
            text += "Build Source Stamp{}: {}\n".format(discriminator, source)
171 -
172 -
        return text
173 -
174 -
    def messageSummary(self, build, results):
175 -
        t = build['state_string']
176 -
        if t:
177 -
            t = ": " + t
178 -
        else:
179 -
            t = ""
180 -
181 -
        if results == SUCCESS:
182 -
            text = "Build succeeded!"
183 -
        elif results == WARNINGS:
184 -
            text = "Build Had Warnings{}".format(t)
185 -
        elif results == CANCELLED:
186 -
            text = "Build was cancelled"
187 -
        else:
188 -
            text = "BUILD FAILED{}".format(t)
189 -
190 -
        return text
245 +
            kwargs['template_filename'] = template_name
246 +
        super().__init__(**kwargs)
191 247
192 248
    @defer.inlineCallbacks
193 -
    def formatMessageForBuildResults(self, mode, buildername, buildset, build, master,
194 -
                                     previous_results, blamelist):
195 -
        """Generate a buildbot mail message and return a dictionary
196 -
           containing the message body, type and subject."""
197 -
        ss_list = buildset['sourcestamps']
198 -
        results = build['results']
199 -
200 -
        ctx = dict(results=build['results'],
201 -
                   mode=mode,
202 -
                   buildername=buildername,
203 -
                   workername=build['properties'].get(
204 -
                       'workername', ["<unknown>"])[0],
205 -
                   buildset=buildset,
206 -
                   build=build,
207 -
                   projects=self.getProjects(ss_list, master),
208 -
                   previous_results=previous_results,
209 -
                   status_detected=self.getDetectedStatus(
210 -
                       mode, results, previous_results),
211 -
                   build_url=utils.getURLForBuild(
212 -
                       master, build['builder']['builderid'], build['number']),
213 -
                   buildbot_url=master.config.buildbotURL,
214 -
                   blamelist=blamelist,
215 -
                   summary=self.messageSummary(build, results),
216 -
                   sourcestamps=self.messageSourceStamps(ss_list)
217 -
                   )
218 -
        yield self.buildAdditionalContext(master, ctx)
219 -
        msgdict = self.renderMessage(ctx)
249 +
    def format_message_for_build(self, mode, buildername, build, master, blamelist):
250 +
        ctx = create_context_for_build(mode, buildername, build, master, blamelist)
251 +
        msgdict = yield self.render_message_dict(master, ctx)
220 252
        return msgdict
221 253
222 254
223 -
class MessageFormatterMissingWorker(MessageFormatterBase):
255 +
class MessageFormatterMissingWorker(MessageFormatterBaseJinja):
224 256
    template_filename = 'missing_mail.txt'
225 257
226 258
    @defer.inlineCallbacks
227 259
    def formatMessageForMissingWorker(self, master, worker):
228 -
        ctx = dict(buildbot_title=master.config.title,
229 -
                   buildbot_url=master.config.buildbotURL,
230 -
                   worker=worker)
231 -
        yield self.buildAdditionalContext(master, ctx)
232 -
        msgdict = self.renderMessage(ctx)
260 +
        ctx = create_context_for_worker(master, worker)
261 +
        msgdict = yield self.render_message_dict(master, ctx)
233 262
        return msgdict

@@ -109,12 +109,13 @@
Loading
109 109
        try:
110 110
            yield self._kube.createPod(self.namespace, pod_spec)
111 111
        except kubeclientservice.KubeError as e:
112 -
            raise LatentWorkerFailedToSubstantiate(str(e))
112 +
            raise LatentWorkerFailedToSubstantiate(str(e)) from e
113 113
        defer.returnValue(True)
114 114
115 115
    @defer.inlineCallbacks
116 116
    def stop_instance(self, fast=False, reportFailure=True):
117 117
        self.current_pod_spec = None
118 +
        self.resetWorkerPropsOnStop()
118 119
        try:
119 120
            yield self._kube.deletePod(self.namespace, self.getContainerName())
120 121
        except kubeclientservice.KubeError as e:

@@ -96,8 +96,8 @@
Loading
96 96
                    if 'bbget' in _is:
97 97
                        try:
98 98
                            v['eptype'] = _is['bbget']['bbtype']
99 -
                        except TypeError:
100 -
                            raise Exception('Unexpected "is" target {}'.format(_is['bbget']))
99 +
                        except TypeError as e:
100 +
                            raise Exception('Unexpected "is" target {}'.format(_is['bbget'])) from e
101 101
102 102
                        self.endpoints_by_type.setdefault(v['eptype'], {})
103 103
                        self.endpoints_by_type[v['eptype']][base] = api

@@ -495,7 +495,7 @@
Loading
495 495
        try:
496 496
            return shlex.split(args)
497 497
        except ValueError as e:
498 -
            raise UsageError(e)
498 +
            raise UsageError(e) from e
499 499
500 500
    def command_HELLO(self, args, **kwargs):
501 501
        """say hello"""

@@ -143,7 +143,7 @@
Loading
143 143
                q = sch_mst_tbl.insert()
144 144
                conn.execute(q,
145 145
                             dict(schedulerid=schedulerid, masterid=masterid)).close()
146 -
            except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
146 +
            except (sa.exc.IntegrityError, sa.exc.ProgrammingError) as e:
147 147
                # someone already owns this scheduler, but who?
148 148
                join = self.db.model.masters.outerjoin(
149 149
                    sch_mst_tbl,
@@ -157,7 +157,7 @@
Loading
157 157
                if row['masterid'] == masterid:
158 158
                    return None
159 159
                raise SchedulerAlreadyClaimedError(
160 -
                    "already claimed by {}".format(row['name']))
160 +
                    "already claimed by {}".format(row['name'])) from e
161 161
            return None
162 162
163 163
        return self.db.pool.do(thd)

@@ -103,7 +103,7 @@
Loading
103 103
            config.error("twisted-mail is not installed - cannot "
104 104
                         "send mail")
105 105
106 -
        super(MailNotifier, self).checkConfig(
106 +
        super().checkConfig(
107 107
            mode=mode, tags=tags, builders=builders,
108 108
            buildSetSummary=buildSetSummary, messageFormatter=messageFormatter,
109 109
            subject=subject, addLogs=addLogs, addPatch=addPatch,

@@ -49,7 +49,9 @@
Loading
49 49
        buildset = res['buildset']
50 50
51 51
        # only include builds for which isMessageNeeded returns true
52 -
        builds = [build for build in builds if self.is_message_needed(build)]
52 +
        builds = [build for build in builds
53 +
                  if self.is_message_needed_by_props(build) and
54 +
                  self.is_message_needed_by_results(build)]
53 55
        if not builds:
54 56
            return None
55 57

@@ -150,9 +150,9 @@
Loading
150 150
                    dict(brid=id, masterid=self.db.master.masterid,
151 151
                         claimed_at=claimed_at)
152 152
                    for id in brids])
153 -
            except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
153 +
            except (sa.exc.IntegrityError, sa.exc.ProgrammingError) as e:
154 154
                transaction.rollback()
155 -
                raise AlreadyClaimedError()
155 +
                raise AlreadyClaimedError() from e
156 156
157 157
            transaction.commit()
158 158

@@ -16,15 +16,12 @@
Loading
16 16
import re
17 17
18 18
from twisted.internet import defer
19 -
from twisted.python import failure
20 -
from twisted.python import log
21 19
from twisted.python.deprecate import deprecatedModuleAttribute
22 20
from twisted.python.versions import Version
23 21
24 22
from buildbot import config
25 23
from buildbot.process import buildstep
26 24
from buildbot.process import logobserver
27 -
from buildbot.process import remotecommand
28 25
# for existing configurations that import WithProperties from here.  We like
29 26
# to move this class around just to keep our readers guessing.
30 27
from buildbot.process.properties import WithProperties
@@ -33,254 +30,27 @@
Loading
33 30
from buildbot.process.results import WARNINGS
34 31
from buildbot.process.results import Results
35 32
from buildbot.process.results import worst_status
33 +
from buildbot.steps.shell_oldstyle import Compile
34 +
from buildbot.steps.shell_oldstyle import Configure
35 +
from buildbot.steps.shell_oldstyle import SetPropertyFromCommand
36 +
from buildbot.steps.shell_oldstyle import ShellCommand
37 +
from buildbot.steps.shell_oldstyle import Test
38 +
from buildbot.steps.shell_oldstyle import WarningCountingShellCommand
36 39
from buildbot.steps.worker import CompositeStepMixin
37 -
from buildbot.util import command_to_string
38 -
from buildbot.util import flatten
39 40
from buildbot.util import join_list
40 41
41 -
_hush_pyflakes = [WithProperties]
42 +
_hush_pyflakes = [
43 +
    WithProperties,
44 +
    Configure,
45 +
    Compile,
46 +
    ShellCommand,
47 +
    SetPropertyFromCommand,
48 +
    Test,
49 +
    WarningCountingShellCommand,
50 +
]
42 51
del _hush_pyflakes
43 52
44 53
45 -
class ShellCommand(buildstep.LoggingBuildStep):
46 -
47 -
    """I run a single shell command on the worker. I return FAILURE if
48 -
    the exit code of that command is non-zero, SUCCESS otherwise. To change
49 -
    this behavior, override my .evaluateCommand method, or customize
50 -
    decodeRC argument
51 -
52 -
    By default, a failure of this step will mark the whole build as FAILURE.
53 -
    To override this, give me an argument of flunkOnFailure=False .
54 -
55 -
    I create a single Log named 'log' which contains the output of the
56 -
    command. To create additional summary Logs, override my .createSummary
57 -
    method.
58 -
59 -
    The shell command I run (a list of argv strings) can be provided in
60 -
    several ways:
61 -
      - a class-level .command attribute
62 -
      - a command= parameter to my constructor (overrides .command)
63 -
      - set explicitly with my .setCommand() method (overrides both)
64 -
65 -
    @ivar command: a list of renderable objects (typically strings or
66 -
                   WithProperties instances). This will be used by start()
67 -
                   to create a RemoteShellCommand instance.
68 -
69 -
    @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
70 -
                    of their corresponding logfiles. The contents of the file
71 -
                    named FILENAME will be put into a LogFile named NAME, ina
72 -
                    something approximating real-time. (note that logfiles=
73 -
                    is actually handled by our parent class LoggingBuildStep)
74 -
75 -
    @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
76 -
                        `lazily', meaning they will only be added when and if
77 -
                        they are written to. Empty or nonexistent logfiles
78 -
                        will be omitted. (Also handled by class
79 -
                        LoggingBuildStep.)
80 -
    """
81 -
82 -
    name = "shell"
83 -
    renderables = [
84 -
        'command',
85 -
        'flunkOnFailure',
86 -
        'haltOnFailure',
87 -
        'remote_kwargs',
88 -
        'workerEnvironment'
89 -
    ]
90 -
91 -
    command = None  # set this to a command, or set in kwargs
92 -
    # logfiles={} # you can also set 'logfiles' to a dictionary, and it
93 -
    #               will be merged with any logfiles= argument passed in
94 -
    #               to __init__
95 -
96 -
    # override this on a specific ShellCommand if you want to let it fail
97 -
    # without dooming the entire build to a status of FAILURE
98 -
    flunkOnFailure = True
99 -
100 -
    def __init__(self, workdir=None,
101 -
                 command=None,
102 -
                 usePTY=None,
103 -
                 **kwargs):
104 -
        # most of our arguments get passed through to the RemoteShellCommand
105 -
        # that we create, but first strip out the ones that we pass to
106 -
        # BuildStep (like haltOnFailure and friends), and a couple that we
107 -
        # consume ourselves.
108 -
109 -
        if command:
110 -
            self.setCommand(command)
111 -
112 -
        if self.__class__ is ShellCommand and not command:
113 -
            # ShellCommand class is directly instantiated.
114 -
            # Explicitly check that command is set to prevent runtime error
115 -
            # later.
116 -
            config.error("ShellCommand's `command' argument is not specified")
117 -
118 -
        # pull out the ones that LoggingBuildStep wants, then upcall
119 -
        buildstep_kwargs = {}
120 -
        # workdir is here first positional argument, but it belongs to
121 -
        # BuildStep parent
122 -
        kwargs['workdir'] = workdir
123 -
        for k in list(kwargs):
124 -
            if k in self.__class__.parms:
125 -
                buildstep_kwargs[k] = kwargs[k]
126 -
                del kwargs[k]
127 -
        super().__init__(**buildstep_kwargs)
128 -
129 -
        # check validity of arguments being passed to RemoteShellCommand
130 -
        invalid_args = []
131 -
        valid_rsc_args = [
132 -
            'env',
133 -
            'want_stdout',
134 -
            'want_stderr',
135 -
            'timeout',
136 -
            'maxTime',
137 -
            'sigtermTime',
138 -
            'logfiles',
139 -
            'usePTY',
140 -
            'logEnviron',
141 -
            'collectStdout',
142 -
            'collectStderr',
143 -
            'interruptSignal',
144 -
            'initialStdin',
145 -
            'decodeRC',
146 -
            'stdioLogName',
147 -
        ]
148 -
        for arg in kwargs:
149 -
            if arg not in valid_rsc_args:
150 -
                invalid_args.append(arg)
151 -
        # Raise Configuration error in case invalid arguments are present
152 -
        if invalid_args:
153 -
            config.error("Invalid argument(s) passed to RemoteShellCommand: " +
154 -
                         ', '.join(invalid_args))
155 -
156 -
        # everything left over goes to the RemoteShellCommand
157 -
        kwargs['usePTY'] = usePTY
158 -
        self.remote_kwargs = kwargs
159 -
        self.remote_kwargs['workdir'] = workdir
160 -
161 -
    def setBuild(self, build):
162 -
        super().setBuild(build)
163 -
        # Set this here, so it gets rendered when we start the step
164 -
        self.workerEnvironment = self.build.workerEnvironment
165 -
166 -
    def setCommand(self, command):
167 -
        self.command = command
168 -
169 -
    def _describe(self, done=False):
170 -
        return None
171 -
172 -
    def describe(self, done=False):
173 -
        if self.stopped and not self.rendered:
174 -
            return "stopped early"
175 -
        assert(self.rendered)
176 -
        desc = self._describe(done)
177 -
        if not desc:
178 -
            return None
179 -
        if self.descriptionSuffix:
180 -
            desc = desc + ' ' + join_list(self.descriptionSuffix)
181 -
        return desc
182 -
183 -
    def getCurrentSummary(self):
184 -
        cmdsummary = self._getLegacySummary(False)
185 -
        if cmdsummary:
186 -
            return {'step': cmdsummary}
187 -
        return super(ShellCommand, self).getCurrentSummary()
188 -
189 -
    def getResultSummary(self):
190 -
        cmdsummary = self._getLegacySummary(True)
191 -
192 -
        if cmdsummary:
193 -
            if self.results != SUCCESS:
194 -
                cmdsummary += ' ({})'.format(Results[self.results])
195 -
            return {'step': cmdsummary}
196 -
197 -
        return super(ShellCommand, self).getResultSummary()
198 -
199 -
    def _getLegacySummary(self, done):
200 -
        # defer to the describe method, if set
201 -
        description = self.describe(done)
202 -
        if description:
203 -
            return join_list(description)
204 -
205 -
        # defer to descriptions, if they're set
206 -
        if (not done and self.description) or (done and self.descriptionDone):
207 -
            return None
208 -
209 -
        try:
210 -
            # if self.cmd is set, then use the RemoteCommand's info
211 -
            if self.cmd:
212 -
                command = self.cmd.remote_command
213 -
            # otherwise, if we were configured with a command, use that
214 -
            elif self.command:
215 -
                command = self.command
216 -
            else:
217 -
                return None
218 -
219 -
            rv = command_to_string(command)
220 -
221 -
            # add the descriptionSuffix, if one was given
222 -
            if self.descriptionSuffix:
223 -
                rv = rv + ' ' + join_list(self.descriptionSuffix)
224 -
225 -
            return rv
226 -
227 -
        except Exception:
228 -
            log.err(failure.Failure(), "Error describing step")
229 -
            return None
230 -
231 -
    def setupEnvironment(self, cmd):
232 -
        # merge in anything from workerEnvironment (which comes from the builder
233 -
        # config) Environment variables passed in by a BuildStep override those
234 -
        # passed in at the Builder level, so if we have any from the builder,
235 -
        # apply those and then update with the args from the buildstep
236 -
        # (cmd.args)
237 -
        workerEnv = self.workerEnvironment
238 -
        if workerEnv:
239 -
            if cmd.args['env'] is None:
240 -
                cmd.args['env'] = {}
241 -
            fullWorkerEnv = workerEnv.copy()
242 -
            fullWorkerEnv.update(cmd.args['env'])
243 -
            cmd.args['env'] = fullWorkerEnv
244 -
            # note that each RemoteShellCommand gets its own copy of the
245 -
            # dictionary, so we shouldn't be affecting anyone but ourselves.
246 -
247 -
    def buildCommandKwargs(self, warnings):
248 -
        kwargs = super().buildCommandKwargs()
249 -
        kwargs.update(self.remote_kwargs)
250 -
        kwargs['workdir'] = self.workdir
251 -
252 -
        kwargs['command'] = flatten(self.command, (list, tuple))
253 -
254 -
        # check for the usePTY flag
255 -
        if 'usePTY' in kwargs and kwargs['usePTY'] is not None:
256 -
            if self.workerVersionIsOlderThan("shell", "2.7"):
257 -
                warnings.append(
258 -
                    "NOTE: worker does not allow master to override usePTY\n")
259 -
                del kwargs['usePTY']
260 -
261 -
        # check for the interruptSignal flag
262 -
        if "interruptSignal" in kwargs and self.workerVersionIsOlderThan("shell", "2.15"):
263 -
            warnings.append(
264 -
                "NOTE: worker does not allow master to specify interruptSignal\n")
265 -
            del kwargs['interruptSignal']
266 -
267 -
        return kwargs
268 -
269 -
    def start(self):
270 -
        # this block is specific to ShellCommands. subclasses that don't need
271 -
        # to set up an argv array, an environment, or extra logfiles= (like
272 -
        # the Source subclasses) can just skip straight to startCommand()
273 -
274 -
        warnings = []
275 -
276 -
        # create the actual RemoteShellCommand instance now
277 -
        kwargs = self.buildCommandKwargs(warnings)
278 -
        cmd = remotecommand.RemoteShellCommand(**kwargs)
279 -
        self.setupEnvironment(cmd)
280 -
281 -
        self.startCommand(cmd, warnings)
282 -
283 -
284 54
class TreeSize(buildstep.ShellMixin, buildstep.BuildStep):
285 55
    name = "treesize"
286 56
    command = ["du", "-s", "-k", "."]
@@ -320,7 +90,7 @@
Loading
320 90
        return SUCCESS
321 91
322 92
323 -
class SetPropertyFromCommand(buildstep.ShellMixin, buildstep.BuildStep):
93 +
class SetPropertyFromCommandNewStyle(buildstep.ShellMixin, buildstep.BuildStep):
324 94
    name = "setproperty"
325 95
    renderables = ['property']
326 96
@@ -399,6 +169,41 @@
Loading
399 169
class ShellCommandNewStyle(buildstep.ShellMixin, buildstep.BuildStep):
400 170
    # This is a temporary class until old ShellCommand is retired
401 171
    def __init__(self, **kwargs):
172 +
173 +
        if self.__class__ is ShellCommandNewStyle:
174 +
            if 'command' not in kwargs:
175 +
                config.error("ShellCommandNewStyle's `command' argument is not specified")
176 +
177 +
            # check validity of arguments being passed to RemoteShellCommand
178 +
            valid_rsc_args = [
179 +
                'command',
180 +
                'env',
181 +
                'want_stdout',
182 +
                'want_stderr',
183 +
                'timeout',
184 +
                'maxTime',
185 +
                'sigtermTime',
186 +
                'logfiles',
187 +
                'usePTY',
188 +
                'logEnviron',
189 +
                'collectStdout',
190 +
                'collectStderr',
191 +
                'interruptSignal',
192 +
                'initialStdin',
193 +
                'decodeRC',
194 +
                'stdioLogName',
195 +
                'workdir',
196 +
            ] + buildstep.BuildStep.parms
197 +
198 +
            invalid_args = []
199 +
            for arg in kwargs:
200 +
                if arg not in valid_rsc_args:
201 +
                    invalid_args.append(arg)
202 +
203 +
            if invalid_args:
204 +
                config.error("Invalid argument(s) passed to ShellCommandNewStyle: " +
205 +
                             ', '.join(invalid_args))
206 +
402 207
        kwargs = self.setupShellMixin(kwargs)
403 208
        super().__init__(**kwargs)
404 209
@@ -409,7 +214,7 @@
Loading
409 214
        return cmd.results()
410 215
411 216
412 -
class Configure(ShellCommandNewStyle):
217 +
class ConfigureNewStyle(ShellCommandNewStyle):
413 218
    name = "configure"
414 219
    haltOnFailure = 1
415 220
    flunkOnFailure = 1
@@ -418,7 +223,8 @@
Loading
418 223
    command = ["./configure"]
419 224
420 225
421 -
class WarningCountingShellCommand(buildstep.ShellMixin, CompositeStepMixin, buildstep.BuildStep):
226 +
class WarningCountingShellCommandNewStyle(buildstep.ShellMixin, CompositeStepMixin,
227 +
                                          buildstep.BuildStep):
422 228
    renderables = [
423 229
        'suppressionFile',
424 230
        'suppressionList',
@@ -463,12 +269,12 @@
Loading
463 269
            self.warningExtractor = WarningCountingShellCommand.warnExtractWholeLine
464 270
        self.maxWarnCount = maxWarnCount
465 271
466 -
        if self.__class__ is WarningCountingShellCommand and \
272 +
        if self.__class__ is WarningCountingShellCommandNewStyle and \
467 273
                not kwargs.get('command'):
468 -
            # WarningCountingShellCommand class is directly instantiated.
274 +
            # WarningCountingShellCommandNewStyle class is directly instantiated.
469 275
            # Explicitly check that command is set to prevent runtime error
470 276
            # later.
471 -
            config.error("WarningCountingShellCommand's `command' argument "
277 +
            config.error("WarningCountingShellCommandNewStyle's `command' argument "
472 278
                         "is not specified")
473 279
474 280
        kwargs = self.setupShellMixin(kwargs)
@@ -662,7 +468,7 @@
Loading
662 468
        return result
663 469
664 470
665 -
class Compile(WarningCountingShellCommand):
471 +
class CompileNewStyle(WarningCountingShellCommandNewStyle):
666 472
667 473
    name = "compile"
668 474
    haltOnFailure = 1
@@ -672,7 +478,7 @@
Loading
672 478
    command = ["make", "all"]
673 479
674 480
675 -
class Test(WarningCountingShellCommand):
481 +
class TestNewStyle(WarningCountingShellCommandNewStyle):
676 482
677 483
    name = "test"
678 484
    warnOnFailure = 1
@@ -770,7 +576,7 @@
Loading
770 576
                self.total = int(mo.group(1))
771 577
772 578
773 -
class PerlModuleTest(Test):
579 +
class PerlModuleTest(TestNewStyle):
774 580
    command = ["prove", "--lib", "lib", "-r", "t"]
775 581
    total = 0
776 582

@@ -37,6 +37,10 @@
Loading
37 37
class GitLabStatusPush(http.HttpStatusPushBase):
38 38
    name = "GitLabStatusPush"
39 39
40 +
    def checkConfig(token, startDescription=None, endDescription=None,
41 +
                    context=None, baseURL=None, verbose=False, wantProperties=True, **kwargs):
42 +
        super().checkConfig(wantProperties=wantProperties, **kwargs)
43 +
40 44
    @defer.inlineCallbacks
41 45
    def reconfigService(self, token,
42 46
                        startDescription=None, endDescription=None,

@@ -40,6 +40,9 @@
Loading
40 40
        self._actual_build_props = copy.deepcopy(props)
41 41
        defer.returnValue(props)
42 42
43 +
    def resetWorkerPropsOnStop(self):
44 +
        self._actual_build_props = None
45 +
43 46
    @defer.inlineCallbacks
44 47
    def isCompatibleWithBuild(self, build):
45 48
        if self._actual_build_props is None:

@@ -247,9 +247,9 @@
Loading
247 247
        revision = stdout.strip("'")
248 248
        try:
249 249
            int(revision)
250 -
        except ValueError:
250 +
        except ValueError as e:
251 251
            log.msg("Invalid revision number")
252 -
            raise buildstep.BuildStepFailed()
252 +
            raise buildstep.BuildStepFailed() from e
253 253
254 254
        log.msg("Got Git revision {}".format(revision))
255 255
        self.updateSourceProperty('got_revision', revision)

@@ -65,7 +65,7 @@
Loading
65 65
                res = yield sched.force(**args)
66 66
                return res
67 67
            except forcesched.CollectedValidationError as e:
68 -
                raise BadJsonRpc2(e.errors, JSONRPC_CODES["invalid_params"])
68 +
                raise BadJsonRpc2(e.errors, JSONRPC_CODES["invalid_params"]) from e
69 69
        return None
70 70
71 71

@@ -19,10 +19,10 @@
Loading
19 19
20 20
21 21
from buildbot.steps.package import util as pkgutil
22 -
from buildbot.steps.shell import Test
22 +
from buildbot.steps.shell import TestNewStyle
23 23
24 24
25 -
class RpmLint(Test):
25 +
class RpmLint(TestNewStyle):
26 26
27 27
    """
28 28
    Rpmlint build step.

@@ -0,0 +1,673 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
import re
17 +
18 +
from twisted.python import failure
19 +
from twisted.python import log
20 +
21 +
from buildbot import config
22 +
from buildbot.process import buildstep
23 +
from buildbot.process import logobserver
24 +
from buildbot.process import remotecommand
25 +
# for existing configurations that import WithProperties from here.  We like
26 +
# to move this class around just to keep our readers guessing.
27 +
from buildbot.process.properties import WithProperties
28 +
from buildbot.process.results import FAILURE
29 +
from buildbot.process.results import SUCCESS
30 +
from buildbot.process.results import WARNINGS
31 +
from buildbot.process.results import Results
32 +
from buildbot.process.results import worst_status
33 +
from buildbot.steps.worker import CompositeStepMixin
34 +
from buildbot.util import command_to_string
35 +
from buildbot.util import flatten
36 +
from buildbot.util import join_list
37 +
38 +
_hush_pyflakes = [WithProperties]
39 +
del _hush_pyflakes
40 +
41 +
42 +
class ShellCommand(buildstep.LoggingBuildStep):
43 +
44 +
    """I run a single shell command on the worker. I return FAILURE if
45 +
    the exit code of that command is non-zero, SUCCESS otherwise. To change
46 +
    this behavior, override my .evaluateCommand method, or customize
47 +
    decodeRC argument
48 +
49 +
    By default, a failure of this step will mark the whole build as FAILURE.
50 +
    To override this, give me an argument of flunkOnFailure=False .
51 +
52 +
    I create a single Log named 'log' which contains the output of the
53 +
    command. To create additional summary Logs, override my .createSummary
54 +
    method.
55 +
56 +
    The shell command I run (a list of argv strings) can be provided in
57 +
    several ways:
58 +
      - a class-level .command attribute
59 +
      - a command= parameter to my constructor (overrides .command)
60 +
      - set explicitly with my .setCommand() method (overrides both)
61 +
62 +
    @ivar command: a list of renderable objects (typically strings or
63 +
                   WithProperties instances). This will be used by start()
64 +
                   to create a RemoteShellCommand instance.
65 +
66 +
    @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
67 +
                    of their corresponding logfiles. The contents of the file
68 +
                    named FILENAME will be put into a LogFile named NAME, ina
69 +
                    something approximating real-time. (note that logfiles=
70 +
                    is actually handled by our parent class LoggingBuildStep)
71 +
72 +
    @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
73 +
                        `lazily', meaning they will only be added when and if
74 +
                        they are written to. Empty or nonexistent logfiles
75 +
                        will be omitted. (Also handled by class
76 +
                        LoggingBuildStep.)
77 +
    """
78 +
79 +
    name = "shell"
80 +
    renderables = [
81 +
        'command',
82 +
        'flunkOnFailure',
83 +
        'haltOnFailure',
84 +
        'remote_kwargs',
85 +
        'workerEnvironment'
86 +
    ]
87 +
88 +
    command = None  # set this to a command, or set in kwargs
89 +
    # logfiles={} # you can also set 'logfiles' to a dictionary, and it
90 +
    #               will be merged with any logfiles= argument passed in
91 +
    #               to __init__
92 +
93 +
    # override this on a specific ShellCommand if you want to let it fail
94 +
    # without dooming the entire build to a status of FAILURE
95 +
    flunkOnFailure = True
96 +
97 +
    def __init__(self, workdir=None,
98 +
                 command=None,
99 +
                 usePTY=None,
100 +
                 **kwargs):
101 +
        # most of our arguments get passed through to the RemoteShellCommand
102 +
        # that we create, but first strip out the ones that we pass to
103 +
        # BuildStep (like haltOnFailure and friends), and a couple that we
104 +
        # consume ourselves.
105 +
106 +
        if command:
107 +
            self.setCommand(command)
108 +
109 +
        if self.__class__ is ShellCommand and not command:
110 +
            # ShellCommand class is directly instantiated.
111 +
            # Explicitly check that command is set to prevent runtime error
112 +
            # later.
113 +
            config.error("ShellCommand's `command' argument is not specified")
114 +
115 +
        # pull out the ones that LoggingBuildStep wants, then upcall
116 +
        buildstep_kwargs = {}
117 +
        # workdir is here first positional argument, but it belongs to
118 +
        # BuildStep parent
119 +
        kwargs['workdir'] = workdir
120 +
        for k in list(kwargs):
121 +
            if k in self.__class__.parms:
122 +
                buildstep_kwargs[k] = kwargs[k]
123 +
                del kwargs[k]
124 +
        super().__init__(**buildstep_kwargs)
125 +
126 +
        # check validity of arguments being passed to RemoteShellCommand
127 +
        invalid_args = []
128 +
        valid_rsc_args = [
129 +
            'env',
130 +
            'want_stdout',
131 +
            'want_stderr',
132 +
            'timeout',
133 +
            'maxTime',
134 +
            'sigtermTime',
135 +
            'logfiles',
136 +
            'usePTY',
137 +
            'logEnviron',
138 +
            'collectStdout',
139 +
            'collectStderr',
140 +
            'interruptSignal',
141 +
            'initialStdin',
142 +
            'decodeRC',
143 +
            'stdioLogName',
144 +
        ]
145 +
        for arg in kwargs:
146 +
            if arg not in valid_rsc_args:
147 +
                invalid_args.append(arg)
148 +
        # Raise Configuration error in case invalid arguments are present
149 +
        if invalid_args:
150 +
            config.error("Invalid argument(s) passed to RemoteShellCommand: " +
151 +
                         ', '.join(invalid_args))
152 +
153 +
        # everything left over goes to the RemoteShellCommand
154 +
        kwargs['usePTY'] = usePTY
155 +
        self.remote_kwargs = kwargs
156 +
        self.remote_kwargs['workdir'] = workdir
157 +
158 +
    def setBuild(self, build):
159 +
        super().setBuild(build)
160 +
        # Set this here, so it gets rendered when we start the step
161 +
        self.workerEnvironment = self.build.workerEnvironment
162 +
163 +
    def setCommand(self, command):
164 +
        self.command = command
165 +
166 +
    def _describe(self, done=False):
167 +
        return None
168 +
169 +
    def describe(self, done=False):
170 +
        if self.stopped and not self.rendered:
171 +
            return "stopped early"
172 +
        assert(self.rendered)
173 +
        desc = self._describe(done)
174 +
        if not desc:
175 +
            return None
176 +
        if self.descriptionSuffix:
177 +
            desc = desc + ' ' + join_list(self.descriptionSuffix)
178 +
        return desc
179 +
180 +
    def getCurrentSummary(self):
181 +
        cmdsummary = self._getLegacySummary(False)
182 +
        if cmdsummary:
183 +
            return {'step': cmdsummary}
184 +
        return super().getCurrentSummary()
185 +
186 +
    def getResultSummary(self):
187 +
        cmdsummary = self._getLegacySummary(True)
188 +
189 +
        if cmdsummary:
190 +
            if self.results != SUCCESS:
191 +
                cmdsummary += ' ({})'.format(Results[self.results])
192 +
            return {'step': cmdsummary}
193 +
194 +
        return super().getResultSummary()
195 +
196 +
    def _getLegacySummary(self, done):
197 +
        # defer to the describe method, if set
198 +
        description = self.describe(done)
199 +
        if description:
200 +
            return join_list(description)
201 +
202 +
        # defer to descriptions, if they're set
203 +
        if (not done and self.description) or (done and self.descriptionDone):
204 +
            return None
205 +
206 +
        try:
207 +
            # if self.cmd is set, then use the RemoteCommand's info
208 +
            if self.cmd:
209 +
                command = self.cmd.remote_command
210 +
            # otherwise, if we were configured with a command, use that
211 +
            elif self.command:
212 +
                command = self.command
213 +
            else:
214 +
                return None
215 +
216 +
            rv = command_to_string(command)
217 +
218 +
            # add the descriptionSuffix, if one was given
219 +
            if self.descriptionSuffix:
220 +
                rv = rv + ' ' + join_list(self.descriptionSuffix)
221 +
222 +
            return rv
223 +
224 +
        except Exception:
225 +
            log.err(failure.Failure(), "Error describing step")
226 +
            return None
227 +
228 +
    def setupEnvironment(self, cmd):
229 +
        # merge in anything from workerEnvironment (which comes from the builder
230 +
        # config) Environment variables passed in by a BuildStep override those
231 +
        # passed in at the Builder level, so if we have any from the builder,
232 +
        # apply those and then update with the args from the buildstep
233 +
        # (cmd.args)
234 +
        workerEnv = self.workerEnvironment
235 +
        if workerEnv:
236 +
            if cmd.args['env'] is None:
237 +
                cmd.args['env'] = {}
238 +
            fullWorkerEnv = workerEnv.copy()
239 +
            fullWorkerEnv.update(cmd.args['env'])
240 +
            cmd.args['env'] = fullWorkerEnv
241 +
            # note that each RemoteShellCommand gets its own copy of the
242 +
            # dictionary, so we shouldn't be affecting anyone but ourselves.
243 +
244 +
    def buildCommandKwargs(self, warnings):
245 +
        kwargs = super().buildCommandKwargs()
246 +
        kwargs.update(self.remote_kwargs)
247 +
        kwargs['workdir'] = self.workdir
248 +
249 +
        kwargs['command'] = flatten(self.command, (list, tuple))
250 +
251 +
        # check for the usePTY flag
252 +
        if 'usePTY' in kwargs and kwargs['usePTY'] is not None:
253 +
            if self.workerVersionIsOlderThan("shell", "2.7"):
254 +
                warnings.append(
255 +
                    "NOTE: worker does not allow master to override usePTY\n")
256 +
                del kwargs['usePTY']
257 +
258 +
        # check for the interruptSignal flag
259 +
        if "interruptSignal" in kwargs and self.workerVersionIsOlderThan("shell", "2.15"):
260 +
            warnings.append(
261 +
                "NOTE: worker does not allow master to specify interruptSignal\n")
262 +
            del kwargs['interruptSignal']
263 +
264 +
        return kwargs
265 +
266 +
    def start(self):
267 +
        # this block is specific to ShellCommands. subclasses that don't need
268 +
        # to set up an argv array, an environment, or extra logfiles= (like
269 +
        # the Source subclasses) can just skip straight to startCommand()
270 +
271 +
        warnings = []
272 +
273 +
        # create the actual RemoteShellCommand instance now
274 +
        kwargs = self.buildCommandKwargs(warnings)
275 +
        cmd = remotecommand.RemoteShellCommand(**kwargs)
276 +
        self.setupEnvironment(cmd)
277 +
278 +
        self.startCommand(cmd, warnings)
279 +
280 +
281 +
class TreeSize(ShellCommand):
282 +
    name = "treesize"
283 +
    command = ["du", "-s", "-k", "."]
284 +
    description = "measuring tree size"
285 +
    kib = None
286 +
287 +
    def __init__(self, **kwargs):
288 +
        super().__init__(**kwargs)
289 +
        self.observer = logobserver.BufferLogObserver(wantStdout=True,
290 +
                                                      wantStderr=True)
291 +
        self.addLogObserver('stdio', self.observer)
292 +
293 +
    def commandComplete(self, cmd):
294 +
        out = self.observer.getStdout()
295 +
        m = re.search(r'^(\d+)', out)
296 +
        if m:
297 +
            self.kib = int(m.group(1))
298 +
            self.setProperty("tree-size-KiB", self.kib, "treesize")
299 +
300 +
    def evaluateCommand(self, cmd):
301 +
        if cmd.didFail():
302 +
            return FAILURE
303 +
        if self.kib is None:
304 +
            return WARNINGS  # not sure how 'du' could fail, but whatever
305 +
        return SUCCESS
306 +
307 +
    def _describe(self, done=False):
308 +
        if self.kib is not None:
309 +
            return ["treesize", "%d KiB" % self.kib]
310 +
        return ["treesize", "unknown"]
311 +
312 +
313 +
class SetPropertyFromCommand(ShellCommand):
314 +
    name = "setproperty"
315 +
    renderables = ['property']
316 +
317 +
    def __init__(self, property=None, extract_fn=None, strip=True,
318 +
                 includeStdout=True, includeStderr=False, **kwargs):
319 +
        self.property = property
320 +
        self.extract_fn = extract_fn
321 +
        self.strip = strip
322 +
        self.includeStdout = includeStdout
323 +
        self.includeStderr = includeStderr
324 +
325 +
        if not ((property is not None) ^ (extract_fn is not None)):
326 +
            config.error(
327 +
                "Exactly one of property and extract_fn must be set")
328 +
329 +
        super().__init__(**kwargs)
330 +
331 +
        if self.extract_fn:
332 +
            self.includeStderr = True
333 +
334 +
        self.observer = logobserver.BufferLogObserver(
335 +
            wantStdout=self.includeStdout,
336 +
            wantStderr=self.includeStderr)
337 +
        self.addLogObserver('stdio', self.observer)
338 +
339 +
        self.property_changes = {}
340 +
341 +
    def commandComplete(self, cmd):
342 +
        if self.property:
343 +
            if cmd.didFail():
344 +
                return
345 +
            result = self.observer.getStdout()
346 +
            if self.strip:
347 +
                result = result.strip()
348 +
            propname = self.property
349 +
            self.setProperty(propname, result, "SetPropertyFromCommand Step")
350 +
            self.property_changes[propname] = result
351 +
        else:
352 +
            new_props = self.extract_fn(cmd.rc,
353 +
                                        self.observer.getStdout(),
354 +
                                        self.observer.getStderr())
355 +
            for k, v in new_props.items():
356 +
                self.setProperty(k, v, "SetPropertyFromCommand Step")
357 +
            self.property_changes = new_props
358 +
359 +
    def createSummary(self, log):
360 +
        if self.property_changes:
361 +
            props_set = ["{}: {}".format(k, repr(v))
362 +
                         for k, v in sorted(self.property_changes.items())]
363 +
            self.addCompleteLog('property changes', "\n".join(props_set))
364 +
365 +
    def describe(self, done=False):
366 +
        if len(self.property_changes) > 1:
367 +
            return ["%d properties set" % len(self.property_changes)]
368 +
        elif len(self.property_changes) == 1:
369 +
            return ["property '{}' set".format(list(self.property_changes)[0])]
370 +
        # else:
371 +
        # let ShellCommand describe
372 +
        return super().describe(done)
373 +
374 +
375 +
class Configure(ShellCommand):
376 +
377 +
    name = "configure"
378 +
    haltOnFailure = 1
379 +
    flunkOnFailure = 1
380 +
    description = ["configuring"]
381 +
    descriptionDone = ["configure"]
382 +
    command = ["./configure"]
383 +
384 +
385 +
class WarningCountingShellCommand(ShellCommand, CompositeStepMixin):
386 +
    renderables = [
387 +
                    'suppressionFile',
388 +
                    'suppressionList',
389 +
                    'warningPattern',
390 +
                    'directoryEnterPattern',
391 +
                    'directoryLeavePattern',
392 +
                    'maxWarnCount',
393 +
    ]
394 +
395 +
    warnCount = 0
396 +
    warningPattern = '(?i).*warning[: ].*'
397 +
    # The defaults work for GNU Make.
398 +
    directoryEnterPattern = ("make.*: Entering directory "
399 +
                             "[\u2019\"`'](.*)[\u2019'`\"]")
400 +
    directoryLeavePattern = "make.*: Leaving directory"
401 +
    suppressionFile = None
402 +
403 +
    commentEmptyLineRe = re.compile(r"^\s*(#.*)?$")
404 +
    suppressionLineRe = re.compile(
405 +
        r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$")
406 +
407 +
    def __init__(self,
408 +
                 warningPattern=None, warningExtractor=None, maxWarnCount=None,
409 +
                 directoryEnterPattern=None, directoryLeavePattern=None,
410 +
                 suppressionFile=None, suppressionList=None, **kwargs):
411 +
        # See if we've been given a regular expression to use to match
412 +
        # warnings. If not, use a default that assumes any line with "warning"
413 +
        # present is a warning. This may lead to false positives in some cases.
414 +
        if warningPattern:
415 +
            self.warningPattern = warningPattern
416 +
        if directoryEnterPattern:
417 +
            self.directoryEnterPattern = directoryEnterPattern
418 +
        if directoryLeavePattern:
419 +
            self.directoryLeavePattern = directoryLeavePattern
420 +
        if suppressionFile:
421 +
            self.suppressionFile = suppressionFile
422 +
        # self.suppressions is already taken, so use something else
423 +
        self.suppressionList = suppressionList
424 +
        if warningExtractor:
425 +
            self.warningExtractor = warningExtractor
426 +
        else:
427 +
            self.warningExtractor = WarningCountingShellCommand.warnExtractWholeLine
428 +
        self.maxWarnCount = maxWarnCount
429 +
430 +
        # And upcall to let the base class do its work
431 +
        super().__init__(**kwargs)
432 +
433 +
        if self.__class__ is WarningCountingShellCommand and \
434 +
                not kwargs.get('command'):
435 +
            # WarningCountingShellCommand class is directly instantiated.
436 +
            # Explicitly check that command is set to prevent runtime error
437 +
            # later.
438 +
            config.error("WarningCountingShellCommand's `command' argument "
439 +
                         "is not specified")
440 +
441 +
        self.suppressions = []
442 +
        self.directoryStack = []
443 +
444 +
        self.warnCount = 0
445 +
        self.loggedWarnings = []
446 +
447 +
        self.addLogObserver(
448 +
            'stdio',
449 +
            logobserver.LineConsumerLogObserver(self.warningLogConsumer))
450 +
451 +
    def addSuppression(self, suppressionList):
452 +
        """
453 +
        This method can be used to add patters of warnings that should
454 +
        not be counted.
455 +
456 +
        It takes a single argument, a list of patterns.
457 +
458 +
        Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END).
459 +
460 +
        FILE-RE is a regular expression (string or compiled regexp), or None.
461 +
        If None, the pattern matches all files, else only files matching the
462 +
        regexp. If directoryEnterPattern is specified in the class constructor,
463 +
        matching is against the full path name, eg. src/main.c.
464 +
465 +
        WARN-RE is similarly a regular expression matched against the
466 +
        text of the warning, or None to match all warnings.
467 +
468 +
        START and END form an inclusive line number range to match against. If
469 +
        START is None, there is no lower bound, similarly if END is none there
470 +
        is no upper bound."""
471 +
472 +
        for fileRe, warnRe, start, end in suppressionList:
473 +
            if fileRe is not None and isinstance(fileRe, str):
474 +
                fileRe = re.compile(fileRe)
475 +
            if warnRe is not None and isinstance(warnRe, str):
476 +
                warnRe = re.compile(warnRe)
477 +
            self.suppressions.append((fileRe, warnRe, start, end))
478 +
479 +
    def warnExtractWholeLine(self, line, match):
480 +
        """
481 +
        Extract warning text as the whole line.
482 +
        No file names or line numbers."""
483 +
        return (None, None, line)
484 +
485 +
    def warnExtractFromRegexpGroups(self, line, match):
486 +
        """
487 +
        Extract file name, line number, and warning text as groups (1,2,3)
488 +
        of warningPattern match."""
489 +
        file = match.group(1)
490 +
        lineNo = match.group(2)
491 +
        if lineNo is not None:
492 +
            lineNo = int(lineNo)
493 +
        text = match.group(3)
494 +
        return (file, lineNo, text)
495 +
496 +
    def warningLogConsumer(self):
497 +
        # Now compile a regular expression from whichever warning pattern we're
498 +
        # using
499 +
        wre = self.warningPattern
500 +
        if isinstance(wre, str):
501 +
            wre = re.compile(wre)
502 +
503 +
        directoryEnterRe = self.directoryEnterPattern
504 +
        if (directoryEnterRe is not None and
505 +
                isinstance(directoryEnterRe, str)):
506 +
            directoryEnterRe = re.compile(directoryEnterRe)
507 +
508 +
        directoryLeaveRe = self.directoryLeavePattern
509 +
        if (directoryLeaveRe is not None and
510 +
                isinstance(directoryLeaveRe, str)):
511 +
            directoryLeaveRe = re.compile(directoryLeaveRe)
512 +
513 +
        # Check if each line in the output from this command matched our
514 +
        # warnings regular expressions. If did, bump the warnings count and
515 +
        # add the line to the collection of lines with warnings
516 +
        self.loggedWarnings = []
517 +
        while True:
518 +
            stream, line = yield
519 +
            if directoryEnterRe:
520 +
                match = directoryEnterRe.search(line)
521 +
                if match:
522 +
                    self.directoryStack.append(match.group(1))
523 +
                    continue
524 +
            if (directoryLeaveRe and
525 +
                self.directoryStack and
526 +
                    directoryLeaveRe.search(line)):
527 +
                self.directoryStack.pop()
528 +
                continue
529 +
530 +
            match = wre.match(line)
531 +
            if match:
532 +
                self.maybeAddWarning(self.loggedWarnings, line, match)
533 +
534 +
    def maybeAddWarning(self, warnings, line, match):
535 +
        if self.suppressions:
536 +
            (file, lineNo, text) = self.warningExtractor(self, line, match)
537 +
            lineNo = lineNo and int(lineNo)
538 +
539 +
            if file is not None and file != "" and self.directoryStack:
540 +
                currentDirectory = '/'.join(self.directoryStack)
541 +
                if currentDirectory is not None and currentDirectory != "":
542 +
                    file = "{}/{}".format(currentDirectory, file)
543 +
544 +
            # Skip adding the warning if any suppression matches.
545 +
            for fileRe, warnRe, start, end in self.suppressions:
546 +
                if not (file is None or fileRe is None or fileRe.match(file)):
547 +
                    continue
548 +
                if not (warnRe is None or warnRe.search(text)):
549 +
                    continue
550 +
                if ((start is not None and end is not None) and
551 +
                   not (lineNo is not None and start <= lineNo <= end)):
552 +
                    continue
553 +
                return
554 +
555 +
        warnings.append(line)
556 +
        self.warnCount += 1
557 +
558 +
    def start(self):
559 +
        if self.suppressionList is not None:
560 +
            self.addSuppression(self.suppressionList)
561 +
        if self.suppressionFile is None:
562 +
            return super().start()
563 +
        d = self.getFileContentFromWorker(
564 +
            self.suppressionFile, abandonOnFailure=True)
565 +
        d.addCallback(self.uploadDone)
566 +
        d.addErrback(self.failed)
567 +
        return None
568 +
569 +
    def uploadDone(self, data):
570 +
        lines = data.split("\n")
571 +
572 +
        list = []
573 +
        for line in lines:
574 +
            if self.commentEmptyLineRe.match(line):
575 +
                continue
576 +
            match = self.suppressionLineRe.match(line)
577 +
            if (match):
578 +
                file, test, start, end = match.groups()
579 +
                if (end is not None):
580 +
                    end = int(end)
581 +
                if (start is not None):
582 +
                    start = int(start)
583 +
                    if end is None:
584 +
                        end = start
585 +
                list.append((file, test, start, end))
586 +
587 +
        self.addSuppression(list)
588 +
        return super().start()
589 +
590 +
    def createSummary(self, log):
591 +
        """
592 +
        Match log lines against warningPattern.
593 +
594 +
        Warnings are collected into another log for this step, and the
595 +
        build-wide 'warnings-count' is updated."""
596 +
597 +
        # If there were any warnings, make the log if lines with warnings
598 +
        # available
599 +
        if self.warnCount:
600 +
            self.addCompleteLog("warnings (%d)" % self.warnCount,
601 +
                                "\n".join(self.loggedWarnings) + "\n")
602 +
603 +
        warnings_stat = self.getStatistic('warnings', 0)
604 +
        self.setStatistic('warnings', warnings_stat + self.warnCount)
605 +
606 +
        old_count = self.getProperty("warnings-count", 0)
607 +
        self.setProperty(
608 +
            "warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")
609 +
610 +
    def evaluateCommand(self, cmd):
611 +
        result = cmd.results()
612 +
        if (self.maxWarnCount is not None and self.warnCount > self.maxWarnCount):
613 +
            result = worst_status(result, FAILURE)
614 +
        elif self.warnCount:
615 +
            result = worst_status(result, WARNINGS)
616 +
        return result
617 +
618 +
619 +
class Compile(WarningCountingShellCommand):
620 +
621 +
    name = "compile"
622 +
    haltOnFailure = 1
623 +
    flunkOnFailure = 1
624 +
    description = ["compiling"]
625 +
    descriptionDone = ["compile"]
626 +
    command = ["make", "all"]
627 +
628 +
<