@@ -14,12 +14,21 @@
Loading
14 14
# Copyright Buildbot Team Members
15 15
16 16
from twisted.internet import defer
17 -
from twisted.internet import reactor
18 17
19 18
from buildbot import config
20 19
from buildbot.process.buildstep import FAILURE
21 20
from buildbot.process.buildstep import SUCCESS
22 21
from buildbot.process.buildstep import BuildStep
22 +
from buildbot.steps.http_oldstyle import DELETE
23 +
from buildbot.steps.http_oldstyle import GET
24 +
from buildbot.steps.http_oldstyle import HEAD
25 +
from buildbot.steps.http_oldstyle import OPTIONS
26 +
from buildbot.steps.http_oldstyle import POST
27 +
from buildbot.steps.http_oldstyle import PUT
28 +
from buildbot.steps.http_oldstyle import HTTPStep
29 +
from buildbot.steps.http_oldstyle import closeSession
30 +
from buildbot.steps.http_oldstyle import getSession
31 +
from buildbot.steps.http_oldstyle import setSession
23 32
24 33
# use the 'requests' lib: https://requests.readthedocs.io/en/master/
25 34
try:
@@ -28,36 +37,24 @@
Loading
28 37
except ImportError:
29 38
    txrequests = None
30 39
40 +
_hush_pyflakes = [
41 +
    DELETE,
42 +
    GET,
43 +
    HEAD,
44 +
    HTTPStep,
45 +
    OPTIONS,
46 +
    POST,
47 +
    PUT,
48 +
    closeSession,
49 +
    getSession,
50 +
    setSession,
51 +
]
52 +
del _hush_pyflakes
31 53
32 -
# This step uses a global Session object, which encapsulates a thread pool as
33 -
# well as state such as cookies and authentication.  This state may pose
34 -
# problems for users, where one step may get a cookie that is subsequently used
35 -
# by another step in a different build.
54 +
# TODO: move session singleton handling back to this module from http_oldstyle
36 55
37 -
_session = None
38 56
39 -
40 -
def getSession():
41 -
    global _session
42 -
    if _session is None:
43 -
        _session = txrequests.Session()
44 -
        reactor.addSystemEventTrigger("before", "shutdown", closeSession)
45 -
    return _session
46 -
47 -
48 -
def setSession(session):
49 -
    global _session
50 -
    _session = session
51 -
52 -
53 -
def closeSession():
54 -
    global _session
55 -
    if _session is not None:
56 -
        _session.close()
57 -
        _session = None
58 -
59 -
60 -
class HTTPStep(BuildStep):
57 +
class HTTPStepNewStyle(BuildStep):
61 58
62 59
    name = 'HTTPStep'
63 60
    description = 'Requesting'
@@ -167,37 +164,37 @@
Loading
167 164
        yield content_log.addStdout(response.text)
168 165
169 166
170 -
class POST(HTTPStep):
167 +
class POSTNewStyle(HTTPStepNewStyle):
171 168
172 169
    def __init__(self, url, **kwargs):
173 170
        super().__init__(url, method='POST', **kwargs)
174 171
175 172
176 -
class GET(HTTPStep):
173 +
class GETNewStyle(HTTPStepNewStyle):
177 174
178 175
    def __init__(self, url, **kwargs):
179 176
        super().__init__(url, method='GET', **kwargs)
180 177
181 178
182 -
class PUT(HTTPStep):
179 +
class PUTNewStyle(HTTPStepNewStyle):
183 180
184 181
    def __init__(self, url, **kwargs):
185 182
        super().__init__(url, method='PUT', **kwargs)
186 183
187 184
188 -
class DELETE(HTTPStep):
185 +
class DELETENewStyle(HTTPStepNewStyle):
189 186
190 187
    def __init__(self, url, **kwargs):
191 188
        super().__init__(url, method='DELETE', **kwargs)
192 189
193 190
194 -
class HEAD(HTTPStep):
191 +
class HEADNewStyle(HTTPStepNewStyle):
195 192
196 193
    def __init__(self, url, **kwargs):
197 194
        super().__init__(url, method='HEAD', **kwargs)
198 195
199 196
200 -
class OPTIONS(HTTPStep):
197 +
class OPTIONSNewStyle(HTTPStepNewStyle):
201 198
202 199
    def __init__(self, url, **kwargs):
203 200
        super().__init__(url, method='OPTIONS', **kwargs)

@@ -19,10 +19,10 @@
Loading
19 19
20 20
21 21
from buildbot.steps.package import util as pkgutil
22 -
from buildbot.steps.shell import Test
22 +
from buildbot.steps.shell import TestNewStyle
23 23
24 24
25 -
class RpmLint(Test):
25 +
class RpmLint(TestNewStyle):
26 26
27 27
    """
28 28
    Rpmlint build step.

@@ -26,10 +26,16 @@
Loading
26 26
from buildbot.process.buildstep import FAILURE
27 27
from buildbot.process.buildstep import SUCCESS
28 28
from buildbot.process.buildstep import BuildStep
29 +
from buildbot.steps.master_oldstyle import MasterShellCommand
29 30
from buildbot.util import deferwaiter
30 31
32 +
_hush_pyflakes = [
33 +
    MasterShellCommand
34 +
]
35 +
del _hush_pyflakes
31 36
32 -
class MasterShellCommand(BuildStep):
37 +
38 +
class MasterShellCommandNewStyle(BuildStep):
33 39
34 40
    """
35 41
    Run a shell command locally - on the buildmaster.  The shell command

@@ -0,0 +1,203 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
from twisted.internet import defer
17 +
from twisted.internet import reactor
18 +
19 +
from buildbot import config
20 +
from buildbot.process.buildstep import FAILURE
21 +
from buildbot.process.buildstep import SUCCESS
22 +
from buildbot.process.buildstep import BuildStep
23 +
24 +
# use the 'requests' lib: https://requests.readthedocs.io/en/master/
25 +
try:
26 +
    import txrequests
27 +
    import requests
28 +
except ImportError:
29 +
    txrequests = None
30 +
31 +
32 +
# This step uses a global Session object, which encapsulates a thread pool as
33 +
# well as state such as cookies and authentication.  This state may pose
34 +
# problems for users, where one step may get a cookie that is subsequently used
35 +
# by another step in a different build.
36 +
37 +
_session = None
38 +
39 +
40 +
def getSession():
41 +
    global _session
42 +
    if _session is None:
43 +
        _session = txrequests.Session()
44 +
        reactor.addSystemEventTrigger("before", "shutdown", closeSession)
45 +
    return _session
46 +
47 +
48 +
def setSession(session):
49 +
    global _session
50 +
    _session = session
51 +
52 +
53 +
def closeSession():
54 +
    global _session
55 +
    if _session is not None:
56 +
        _session.close()
57 +
        _session = None
58 +
59 +
60 +
class HTTPStep(BuildStep):
61 +
62 +
    name = 'HTTPStep'
63 +
    description = 'Requesting'
64 +
    descriptionDone = 'Requested'
65 +
    requestsParams = ["params", "data", "json", "headers",
66 +
                      "cookies", "files", "auth",
67 +
                      "timeout", "allow_redirects", "proxies",
68 +
                      "hooks", "stream", "verify", "cert"]
69 +
    renderables = requestsParams + ["method", "url"]
70 +
    session = None
71 +
72 +
    def __init__(self, url, method, **kwargs):
73 +
        if txrequests is None:
74 +
            config.error(
75 +
                "Need to install txrequest to use this step:\n\n pip install txrequests")
76 +
77 +
        if method not in ('POST', 'GET', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'):
78 +
            config.error("Wrong method given: '{}' is not known".format(method))
79 +
80 +
        self.method = method
81 +
        self.url = url
82 +
83 +
        for param in self.requestsParams:
84 +
            setattr(self, param, kwargs.pop(param, None))
85 +
86 +
        super().__init__(**kwargs)
87 +
88 +
    @defer.inlineCallbacks
89 +
    def run(self):
90 +
        # create a new session if it doesn't exist
91 +
        self.session = getSession()
92 +
93 +
        requestkwargs = {
94 +
            'method': self.method,
95 +
            'url': self.url
96 +
        }
97 +
98 +
        for param in self.requestsParams:
99 +
            value = getattr(self, param, None)
100 +
            if value is not None:
101 +
                requestkwargs[param] = value
102 +
103 +
        log = yield self.addLog('log')
104 +
105 +
        # known methods already tested in __init__
106 +
107 +
        yield log.addHeader('Performing {} request to {}\n'.format(self.method, self.url))
108 +
        if self.params:
109 +
            yield log.addHeader('Parameters:\n')
110 +
            params = requestkwargs.get("params", {})
111 +
            if params:
112 +
                params = sorted(params.items(), key=lambda x: x[0])
113 +
                requestkwargs['params'] = params
114 +
            for k, v in params:
115 +
                yield log.addHeader('\t{}: {}\n'.format(k, v))
116 +
        data = requestkwargs.get("data", None)
117 +
        if data:
118 +
            yield log.addHeader('Data:\n')
119 +
            if isinstance(data, dict):
120 +
                for k, v in data.items():
121 +
                    yield log.addHeader('\t{}: {}\n'.format(k, v))
122 +
            else:
123 +
                yield log.addHeader('\t{}\n'.format(data))
124 +
125 +
        try:
126 +
            r = yield self.session.request(**requestkwargs)
127 +
        except requests.exceptions.ConnectionError as e:
128 +
            yield log.addStderr('An exception occurred while performing the request: {}'.format(e))
129 +
            return FAILURE
130 +
131 +
        if r.history:
132 +
            yield log.addStdout('\nRedirected %d times:\n\n' % len(r.history))
133 +
            for rr in r.history:
134 +
                yield self.log_response(log, rr)
135 +
                yield log.addStdout('=' * 60 + '\n')
136 +
137 +
        yield self.log_response(log, r)
138 +
139 +
        yield log.finish()
140 +
141 +
        self.descriptionDone = ["Status code: %d" % r.status_code]
142 +
        if (r.status_code < 400):
143 +
            return SUCCESS
144 +
        else:
145 +
            return FAILURE
146 +
147 +
    @defer.inlineCallbacks
148 +
    def log_response(self, log, response):
149 +
150 +
        yield log.addHeader('Request Header:\n')
151 +
        for k, v in response.request.headers.items():
152 +
            yield log.addHeader('\t{}: {}\n'.format(k, v))
153 +
154 +
        yield log.addStdout('URL: {}\n'.format(response.url))
155 +
156 +
        if response.status_code == requests.codes.ok:
157 +
            yield log.addStdout('Status: {}\n'.format(response.status_code))
158 +
        else:
159 +
            yield log.addStderr('Status: {}\n'.format(response.status_code))
160 +
161 +
        yield log.addHeader('Response Header:\n')
162 +
        for k, v in response.headers.items():
163 +
            yield log.addHeader('\t{}: {}\n'.format(k, v))
164 +
165 +
        yield log.addStdout(' ------ Content ------\n{}'.format(response.text))
166 +
        content_log = yield self.addLog('content')
167 +
        yield content_log.addStdout(response.text)
168 +
169 +
170 +
class POST(HTTPStep):
171 +
172 +
    def __init__(self, url, **kwargs):
173 +
        super().__init__(url, method='POST', **kwargs)
174 +
175 +
176 +
class GET(HTTPStep):
177 +
178 +
    def __init__(self, url, **kwargs):
179 +
        super().__init__(url, method='GET', **kwargs)
180 +
181 +
182 +
class PUT(HTTPStep):
183 +
184 +
    def __init__(self, url, **kwargs):
185 +
        super().__init__(url, method='PUT', **kwargs)
186 +
187 +
188 +
class DELETE(HTTPStep):
189 +
190 +
    def __init__(self, url, **kwargs):
191 +
        super().__init__(url, method='DELETE', **kwargs)
192 +
193 +
194 +
class HEAD(HTTPStep):
195 +
196 +
    def __init__(self, url, **kwargs):
197 +
        super().__init__(url, method='HEAD', **kwargs)
198 +
199 +
200 +
class OPTIONS(HTTPStep):
201 +
202 +
    def __init__(self, url, **kwargs):
203 +
        super().__init__(url, method='OPTIONS', **kwargs)

@@ -0,0 +1,673 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
import re
17 +
18 +
from twisted.python import failure
19 +
from twisted.python import log
20 +
21 +
from buildbot import config
22 +
from buildbot.process import buildstep
23 +
from buildbot.process import logobserver
24 +
from buildbot.process import remotecommand
25 +
# for existing configurations that import WithProperties from here.  We like
26 +
# to move this class around just to keep our readers guessing.
27 +
from buildbot.process.properties import WithProperties
28 +
from buildbot.process.results import FAILURE
29 +
from buildbot.process.results import SUCCESS
30 +
from buildbot.process.results import WARNINGS
31 +
from buildbot.process.results import Results
32 +
from buildbot.process.results import worst_status
33 +
from buildbot.steps.worker import CompositeStepMixin
34 +
from buildbot.util import command_to_string
35 +
from buildbot.util import flatten
36 +
from buildbot.util import join_list
37 +
38 +
_hush_pyflakes = [WithProperties]
39 +
del _hush_pyflakes
40 +
41 +
42 +
class ShellCommand(buildstep.LoggingBuildStep):
43 +
44 +
    """I run a single shell command on the worker. I return FAILURE if
45 +
    the exit code of that command is non-zero, SUCCESS otherwise. To change
46 +
    this behavior, override my .evaluateCommand method, or customize
47 +
    decodeRC argument
48 +
49 +
    By default, a failure of this step will mark the whole build as FAILURE.
50 +
    To override this, give me an argument of flunkOnFailure=False .
51 +
52 +
    I create a single Log named 'log' which contains the output of the
53 +
    command. To create additional summary Logs, override my .createSummary
54 +
    method.
55 +
56 +
    The shell command I run (a list of argv strings) can be provided in
57 +
    several ways:
58 +
      - a class-level .command attribute
59 +
      - a command= parameter to my constructor (overrides .command)
60 +
      - set explicitly with my .setCommand() method (overrides both)
61 +
62 +
    @ivar command: a list of renderable objects (typically strings or
63 +
                   WithProperties instances). This will be used by start()
64 +
                   to create a RemoteShellCommand instance.
65 +
66 +
    @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
67 +
                    of their corresponding logfiles. The contents of the file
68 +
                    named FILENAME will be put into a LogFile named NAME, ina
69 +
                    something approximating real-time. (note that logfiles=
70 +
                    is actually handled by our parent class LoggingBuildStep)
71 +
72 +
    @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
73 +
                        `lazily', meaning they will only be added when and if
74 +
                        they are written to. Empty or nonexistent logfiles
75 +
                        will be omitted. (Also handled by class
76 +
                        LoggingBuildStep.)
77 +
    """
78 +
79 +
    name = "shell"
80 +
    renderables = [
81 +
        'command',
82 +
        'flunkOnFailure',
83 +
        'haltOnFailure',
84 +
        'remote_kwargs',
85 +
        'workerEnvironment'
86 +
    ]
87 +
88 +
    command = None  # set this to a command, or set in kwargs
89 +
    # logfiles={} # you can also set 'logfiles' to a dictionary, and it
90 +
    #               will be merged with any logfiles= argument passed in
91 +
    #               to __init__
92 +
93 +
    # override this on a specific ShellCommand if you want to let it fail
94 +
    # without dooming the entire build to a status of FAILURE
95 +
    flunkOnFailure = True
96 +
97 +
    def __init__(self, workdir=None,
98 +
                 command=None,
99 +
                 usePTY=None,
100 +
                 **kwargs):
101 +
        # most of our arguments get passed through to the RemoteShellCommand
102 +
        # that we create, but first strip out the ones that we pass to
103 +
        # BuildStep (like haltOnFailure and friends), and a couple that we
104 +
        # consume ourselves.
105 +
106 +
        if command:
107 +
            self.setCommand(command)
108 +
109 +
        if self.__class__ is ShellCommand and not command:
110 +
            # ShellCommand class is directly instantiated.
111 +
            # Explicitly check that command is set to prevent runtime error
112 +
            # later.
113 +
            config.error("ShellCommand's `command' argument is not specified")
114 +
115 +
        # pull out the ones that LoggingBuildStep wants, then upcall
116 +
        buildstep_kwargs = {}
117 +
        # workdir is here first positional argument, but it belongs to
118 +
        # BuildStep parent
119 +
        kwargs['workdir'] = workdir
120 +
        for k in list(kwargs):
121 +
            if k in self.__class__.parms:
122 +
                buildstep_kwargs[k] = kwargs[k]
123 +
                del kwargs[k]
124 +
        super().__init__(**buildstep_kwargs)
125 +
126 +
        # check validity of arguments being passed to RemoteShellCommand
127 +
        invalid_args = []
128 +
        valid_rsc_args = [
129 +
            'env',
130 +
            'want_stdout',
131 +
            'want_stderr',
132 +
            'timeout',
133 +
            'maxTime',
134 +
            'sigtermTime',
135 +
            'logfiles',
136 +
            'usePTY',
137 +
            'logEnviron',
138 +
            'collectStdout',
139 +
            'collectStderr',
140 +
            'interruptSignal',
141 +
            'initialStdin',
142 +
            'decodeRC',
143 +
            'stdioLogName',
144 +
        ]
145 +
        for arg in kwargs:
146 +
            if arg not in valid_rsc_args:
147 +
                invalid_args.append(arg)
148 +
        # Raise Configuration error in case invalid arguments are present
149 +
        if invalid_args:
150 +
            config.error("Invalid argument(s) passed to RemoteShellCommand: " +
151 +
                         ', '.join(invalid_args))
152 +
153 +
        # everything left over goes to the RemoteShellCommand
154 +
        kwargs['usePTY'] = usePTY
155 +
        self.remote_kwargs = kwargs
156 +
        self.remote_kwargs['workdir'] = workdir
157 +
158 +
    def setBuild(self, build):
159 +
        super().setBuild(build)
160 +
        # Set this here, so it gets rendered when we start the step
161 +
        self.workerEnvironment = self.build.workerEnvironment
162 +
163 +
    def setCommand(self, command):
164 +
        self.command = command
165 +
166 +
    def _describe(self, done=False):
167 +
        return None
168 +
169 +
    def describe(self, done=False):
170 +
        if self.stopped and not self.rendered:
171 +
            return "stopped early"
172 +
        assert(self.rendered)
173 +
        desc = self._describe(done)
174 +
        if not desc:
175 +
            return None
176 +
        if self.descriptionSuffix:
177 +
            desc = desc + ' ' + join_list(self.descriptionSuffix)
178 +
        return desc
179 +
180 +
    def getCurrentSummary(self):
181 +
        cmdsummary = self._getLegacySummary(False)
182 +
        if cmdsummary:
183 +
            return {'step': cmdsummary}
184 +
        return super().getCurrentSummary()
185 +
186 +
    def getResultSummary(self):
187 +
        cmdsummary = self._getLegacySummary(True)
188 +
189 +
        if cmdsummary:
190 +
            if self.results != SUCCESS:
191 +
                cmdsummary += ' ({})'.format(Results[self.results])
192 +
            return {'step': cmdsummary}
193 +
194 +
        return super().getResultSummary()
195 +
196 +
    def _getLegacySummary(self, done):
197 +
        # defer to the describe method, if set
198 +
        description = self.describe(done)
199 +
        if description:
200 +
            return join_list(description)
201 +
202 +
        # defer to descriptions, if they're set
203 +
        if (not done and self.description) or (done and self.descriptionDone):
204 +
            return None
205 +
206 +
        try:
207 +
            # if self.cmd is set, then use the RemoteCommand's info
208 +
            if self.cmd:
209 +
                command = self.cmd.remote_command
210 +
            # otherwise, if we were configured with a command, use that
211 +
            elif self.command:
212 +
                command = self.command
213 +
            else:
214 +
                return None
215 +
216 +
            rv = command_to_string(command)
217 +
218 +
            # add the descriptionSuffix, if one was given
219 +
            if self.descriptionSuffix:
220 +
                rv = rv + ' ' + join_list(self.descriptionSuffix)
221 +
222 +
            return rv
223 +
224 +
        except Exception:
225 +
            log.err(failure.Failure(), "Error describing step")
226 +
            return None
227 +
228 +
    def setupEnvironment(self, cmd):
229 +
        # merge in anything from workerEnvironment (which comes from the builder
230 +
        # config) Environment variables passed in by a BuildStep override those
231 +
        # passed in at the Builder level, so if we have any from the builder,
232 +
        # apply those and then update with the args from the buildstep
233 +
        # (cmd.args)
234 +
        workerEnv = self.workerEnvironment
235 +
        if workerEnv:
236 +
            if cmd.args['env'] is None:
237 +
                cmd.args['env'] = {}
238 +
            fullWorkerEnv = workerEnv.copy()
239 +
            fullWorkerEnv.update(cmd.args['env'])
240 +
            cmd.args['env'] = fullWorkerEnv
241 +
            # note that each RemoteShellCommand gets its own copy of the
242 +
            # dictionary, so we shouldn't be affecting anyone but ourselves.
243 +
244 +
    def buildCommandKwargs(self, warnings):
245 +
        kwargs = super().buildCommandKwargs()
246 +
        kwargs.update(self.remote_kwargs)
247 +
        kwargs['workdir'] = self.workdir
248 +
249 +
        kwargs['command'] = flatten(self.command, (list, tuple))
250 +
251 +
        # check for the usePTY flag
252 +
        if 'usePTY' in kwargs and kwargs['usePTY'] is not None:
253 +
            if self.workerVersionIsOlderThan("shell", "2.7"):
254 +
                warnings.append(
255 +
                    "NOTE: worker does not allow master to override usePTY\n")
256 +
                del kwargs['usePTY']
257 +
258 +
        # check for the interruptSignal flag
259 +
        if "interruptSignal" in kwargs and self.workerVersionIsOlderThan("shell", "2.15"):
260 +
            warnings.append(
261 +
                "NOTE: worker does not allow master to specify interruptSignal\n")
262 +
            del kwargs['interruptSignal']
263 +
264 +
        return kwargs
265 +
266 +
    def start(self):
267 +
        # this block is specific to ShellCommands. subclasses that don't need
268 +
        # to set up an argv array, an environment, or extra logfiles= (like
269 +
        # the Source subclasses) can just skip straight to startCommand()
270 +
271 +
        warnings = []
272 +
273 +
        # create the actual RemoteShellCommand instance now
274 +
        kwargs = self.buildCommandKwargs(warnings)
275 +
        cmd = remotecommand.RemoteShellCommand(**kwargs)
276 +
        self.setupEnvironment(cmd)
277 +
278 +
        self.startCommand(cmd, warnings)
279 +
280 +
281 +
class TreeSize(ShellCommand):
282 +
    name = "treesize"
283 +
    command = ["du", "-s", "-k", "."]
284 +
    description = "measuring tree size"
285 +
    kib = None
286 +
287 +
    def __init__(self, **kwargs):
288 +
        super().__init__(**kwargs)
289 +
        self.observer = logobserver.BufferLogObserver(wantStdout=True,
290 +
                                                      wantStderr=True)
291 +
        self.addLogObserver('stdio', self.observer)
292 +
293 +
    def commandComplete(self, cmd):
294 +
        out = self.observer.getStdout()
295 +
        m = re.search(r'^(\d+)', out)
296 +
        if m:
297 +
            self.kib = int(m.group(1))
298 +
            self.setProperty("tree-size-KiB", self.kib, "treesize")
299 +
300 +
    def evaluateCommand(self, cmd):
301 +
        if cmd.didFail():
302 +
            return FAILURE
303 +
        if self.kib is None:
304 +
            return WARNINGS  # not sure how 'du' could fail, but whatever
305 +
        return SUCCESS
306 +
307 +
    def _describe(self, done=False):
308 +
        if self.kib is not None:
309 +
            return ["treesize", "%d KiB" % self.kib]
310 +
        return ["treesize", "unknown"]
311 +
312 +
313 +
class SetPropertyFromCommand(ShellCommand):
314 +
    name = "setproperty"
315 +
    renderables = ['property']
316 +
317 +
    def __init__(self, property=None, extract_fn=None, strip=True,
318 +
                 includeStdout=True, includeStderr=False, **kwargs):
319 +
        self.property = property
320 +
        self.extract_fn = extract_fn
321 +
        self.strip = strip
322 +
        self.includeStdout = includeStdout
323 +
        self.includeStderr = includeStderr
324 +
325 +
        if not ((property is not None) ^ (extract_fn is not None)):
326 +
            config.error(
327 +
                "Exactly one of property and extract_fn must be set")
328 +
329 +
        super().__init__(**kwargs)
330 +
331 +
        if self.extract_fn:
332 +
            self.includeStderr = True
333 +
334 +
        self.observer = logobserver.BufferLogObserver(
335 +
            wantStdout=self.includeStdout,
336 +
            wantStderr=self.includeStderr)
337 +
        self.addLogObserver('stdio', self.observer)
338 +
339 +
        self.property_changes = {}
340 +
341 +
    def commandComplete(self, cmd):
342 +
        if self.property:
343 +
            if cmd.didFail():
344 +
                return
345 +
            result = self.observer.getStdout()
346 +
            if self.strip:
347 +
                result = result.strip()
348 +
            propname = self.property
349 +
            self.setProperty(propname, result, "SetPropertyFromCommand Step")
350 +
            self.property_changes[propname] = result
351 +
        else:
352 +
            new_props = self.extract_fn(cmd.rc,
353 +
                                        self.observer.getStdout(),
354 +
                                        self.observer.getStderr())
355 +
            for k, v in new_props.items():
356 +
                self.setProperty(k, v, "SetPropertyFromCommand Step")
357 +
            self.property_changes = new_props
358 +
359 +
    def createSummary(self, log):
360 +
        if self.property_changes:
361 +
            props_set = ["{}: {}".format(k, repr(v))
362 +
                         for k, v in sorted(self.property_changes.items())]
363 +
            self.addCompleteLog('property changes', "\n".join(props_set))
364 +
365 +
    def describe(self, done=False):
366 +
        if len(self.property_changes) > 1:
367 +
            return ["%d properties set" % len(self.property_changes)]
368 +
        elif len(self.property_changes) == 1:
369 +
            return ["property '{}' set".format(list(self.property_changes)[0])]
370 +
        # else:
371 +
        # let ShellCommand describe
372 +
        return super().describe(done)
373 +
374 +
375 +
class Configure(ShellCommand):
376 +
377 +
    name = "configure"
378 +
    haltOnFailure = 1
379 +
    flunkOnFailure = 1
380 +
    description = ["configuring"]
381 +
    descriptionDone = ["configure"]
382 +
    command = ["./configure"]
383 +
384 +
385 +
class WarningCountingShellCommand(ShellCommand, CompositeStepMixin):
386 +
    renderables = [
387 +
                    'suppressionFile',
388 +
                    'suppressionList',
389 +
                    'warningPattern',
390 +
                    'directoryEnterPattern',
391 +
                    'directoryLeavePattern',
392 +
                    'maxWarnCount',
393 +
    ]
394 +
395 +
    warnCount = 0
396 +
    warningPattern = '(?i).*warning[: ].*'
397 +
    # The defaults work for GNU Make.
398 +
    directoryEnterPattern = ("make.*: Entering directory "
399 +
                             "[\u2019\"`'](.*)[\u2019'`\"]")
400 +
    directoryLeavePattern = "make.*: Leaving directory"
401 +
    suppressionFile = None
402 +
403 +
    commentEmptyLineRe = re.compile(r"^\s*(#.*)?$")
404 +
    suppressionLineRe = re.compile(
405 +
        r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$")
406 +
407 +
    def __init__(self,
408 +
                 warningPattern=None, warningExtractor=None, maxWarnCount=None,
409 +
                 directoryEnterPattern=None, directoryLeavePattern=None,
410 +
                 suppressionFile=None, suppressionList=None, **kwargs):
411 +
        # See if we've been given a regular expression to use to match
412 +
        # warnings. If not, use a default that assumes any line with "warning"
413 +
        # present is a warning. This may lead to false positives in some cases.
414 +
        if warningPattern:
415 +
            self.warningPattern = warningPattern
416 +
        if directoryEnterPattern:
417 +
            self.directoryEnterPattern = directoryEnterPattern
418 +
        if directoryLeavePattern:
419 +
            self.directoryLeavePattern = directoryLeavePattern
420 +
        if suppressionFile:
421 +
            self.suppressionFile = suppressionFile
422 +
        # self.suppressions is already taken, so use something else
423 +
        self.suppressionList = suppressionList
424 +
        if warningExtractor:
425 +
            self.warningExtractor = warningExtractor
426 +
        else:
427 +
            self.warningExtractor = WarningCountingShellCommand.warnExtractWholeLine
428 +
        self.maxWarnCount = maxWarnCount
429 +
430 +
        # And upcall to let the base class do its work
431 +
        super().__init__(**kwargs)
432 +
433 +
        if self.__class__ is WarningCountingShellCommand and \
434 +
                not kwargs.get('command'):
435 +
            # WarningCountingShellCommand class is directly instantiated.
436 +
            # Explicitly check that command is set to prevent runtime error
437 +
            # later.
438 +
            config.error("WarningCountingShellCommand's `command' argument "
439 +
                         "is not specified")
440 +
441 +
        self.suppressions = []
442 +
        self.directoryStack = []
443 +
444 +
        self.warnCount = 0
445 +
        self.loggedWarnings = []
446 +
447 +
        self.addLogObserver(
448 +
            'stdio',
449 +
            logobserver.LineConsumerLogObserver(self.warningLogConsumer))
450 +
451 +
    def addSuppression(self, suppressionList):
452 +
        """
453 +
        This method can be used to add patters of warnings that should
454 +
        not be counted.
455 +
456 +
        It takes a single argument, a list of patterns.
457 +
458 +
        Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END).
459 +
460 +
        FILE-RE is a regular expression (string or compiled regexp), or None.
461 +
        If None, the pattern matches all files, else only files matching the
462 +
        regexp. If directoryEnterPattern is specified in the class constructor,
463 +
        matching is against the full path name, eg. src/main.c.
464 +
465 +
        WARN-RE is similarly a regular expression matched against the
466 +
        text of the warning, or None to match all warnings.
467 +
468 +
        START and END form an inclusive line number range to match against. If
469 +
        START is None, there is no lower bound, similarly if END is none there
470 +
        is no upper bound."""
471 +
472 +
        for fileRe, warnRe, start, end in suppressionList:
473 +
            if fileRe is not None and isinstance(fileRe, str):
474 +
                fileRe = re.compile(fileRe)
475 +
            if warnRe is not None and isinstance(warnRe, str):
476 +
                warnRe = re.compile(warnRe)
477 +
            self.suppressions.append((fileRe, warnRe, start, end))
478 +
479 +
    def warnExtractWholeLine(self, line, match):
480 +
        """
481 +
        Extract warning text as the whole line.
482 +
        No file names or line numbers."""
483 +
        return (None, None, line)
484 +
485 +
    def warnExtractFromRegexpGroups(self, line, match):
486 +
        """
487 +
        Extract file name, line number, and warning text as groups (1,2,3)
488 +
        of warningPattern match."""
489 +
        file = match.group(1)
490 +
        lineNo = match.group(2)
491 +
        if lineNo is not None:
492 +
            lineNo = int(lineNo)
493 +
        text = match.group(3)
494 +
        return (file, lineNo, text)
495 +
496 +
    def warningLogConsumer(self):
497 +
        # Now compile a regular expression from whichever warning pattern we're
498 +
        # using
499 +
        wre = self.warningPattern
500 +
        if isinstance(wre, str):
501 +
            wre = re.compile(wre)
502 +
503 +
        directoryEnterRe = self.directoryEnterPattern
504 +
        if (directoryEnterRe is not None and
505 +
                isinstance(directoryEnterRe, str)):
506 +
            directoryEnterRe = re.compile(directoryEnterRe)
507 +
508 +
        directoryLeaveRe = self.directoryLeavePattern
509 +
        if (directoryLeaveRe is not None and
510 +
                isinstance(directoryLeaveRe, str)):
511 +
            directoryLeaveRe = re.compile(directoryLeaveRe)
512 +
513 +
        # Check if each line in the output from this command matched our
514 +
        # warnings regular expressions. If did, bump the warnings count and
515 +
        # add the line to the collection of lines with warnings
516 +
        self.loggedWarnings = []
517 +
        while True:
518 +
            stream, line = yield
519 +
            if directoryEnterRe:
520 +
                match = directoryEnterRe.search(line)
521 +
                if match:
522 +
                    self.directoryStack.append(match.group(1))
523 +
                    continue
524 +
            if (directoryLeaveRe and
525 +
                self.directoryStack and
526 +
                    directoryLeaveRe.search(line)):
527 +
                self.directoryStack.pop()
528 +
                continue
529 +
530 +
            match = wre.match(line)
531 +
            if match:
532 +
                self.maybeAddWarning(self.loggedWarnings, line, match)
533 +
534 +
    def maybeAddWarning(self, warnings, line, match):
535 +
        if self.suppressions:
536 +
            (file, lineNo, text) = self.warningExtractor(self, line, match)
537 +
            lineNo = lineNo and int(lineNo)
538 +
539 +
            if file is not None and file != "" and self.directoryStack:
540 +
                currentDirectory = '/'.join(self.directoryStack)
541 +
                if currentDirectory is not None and currentDirectory != "":
542 +
                    file = "{}/{}".format(currentDirectory, file)
543 +
544 +
            # Skip adding the warning if any suppression matches.
545 +
            for fileRe, warnRe, start, end in self.suppressions:
546 +
                if not (file is None or fileRe is None or fileRe.match(file)):
547 +
                    continue
548 +
                if not (warnRe is None or warnRe.search(text)):
549 +
                    continue
550 +
                if ((start is not None and end is not None) and
551 +
                   not (lineNo is not None and start <= lineNo <= end)):
552 +
                    continue
553 +
                return
554 +
555 +
        warnings.append(line)
556 +
        self.warnCount += 1
557 +
558 +
    def start(self):
559 +
        if self.suppressionList is not None:
560 +
            self.addSuppression(self.suppressionList)
561 +
        if self.suppressionFile is None:
562 +
            return super().start()
563 +
        d = self.getFileContentFromWorker(
564 +
            self.suppressionFile, abandonOnFailure=True)
565 +
        d.addCallback(self.uploadDone)
566 +
        d.addErrback(self.failed)
567 +
        return None
568 +
569 +
    def uploadDone(self, data):
570 +
        lines = data.split("\n")
571 +
572 +
        list = []
573 +
        for line in lines:
574 +
            if self.commentEmptyLineRe.match(line):
575 +
                continue
576 +
            match = self.suppressionLineRe.match(line)
577 +
            if (match):
578 +
                file, test, start, end = match.groups()
579 +
                if (end is not None):
580 +
                    end = int(end)
581 +
                if (start is not None):
582 +
                    start = int(start)
583 +
                    if end is None:
584 +
                        end = start
585 +
                list.append((file, test, start, end))
586 +
587 +
        self.addSuppression(list)
588 +
        return super().start()
589 +
590 +
    def createSummary(self, log):
591 +
        """
592 +
        Match log lines against warningPattern.
593 +
594 +
        Warnings are collected into another log for this step, and the
595 +
        build-wide 'warnings-count' is updated."""
596 +
597 +
        # If there were any warnings, make the log if lines with warnings
598 +
        # available
599 +
        if self.warnCount:
600 +
            self.addCompleteLog("warnings (%d)" % self.warnCount,
601 +
                                "\n".join(self.loggedWarnings) + "\n")
602 +
603 +
        warnings_stat = self.getStatistic('warnings', 0)
604 +
        self.setStatistic('warnings', warnings_stat + self.warnCount)
605 +
606 +
        old_count = self.getProperty("warnings-count", 0)
607 +
        self.setProperty(
608 +
            "warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")
609 +
610 +
    def evaluateCommand(self, cmd):
611 +
        result = cmd.results()
612 +
        if (self.maxWarnCount is not None and self.warnCount > self.maxWarnCount):
613 +
            result = worst_status(result, FAILURE)
614 +
        elif self.warnCount:
615 +
            result = worst_status(result, WARNINGS)
616 +
        return result
617 +
618 +
619 +
class Compile(WarningCountingShellCommand):
620 +
621 +
    name = "compile"
622 +
    haltOnFailure = 1
623 +
    flunkOnFailure = 1
624 +
    description = ["compiling"]
625 +
    descriptionDone = ["compile"]
626 +
    command = ["make", "all"]
627 +
628 +
629 +
class Test(WarningCountingShellCommand):
630 +
631 +
    name = "test"
632 +
    warnOnFailure = 1
633 +
    description = ["testing"]
634 +
    descriptionDone = ["test"]
635 +
    command = ["make", "test"]
636 +
637 +
    def setTestResults(self, total=0, failed=0, passed=0, warnings=0):
638 +
        """
639 +
        Called by subclasses to set the relevant statistics; this actually
640 +
        adds to any statistics already present
641 +
        """
642 +
        total += self.getStatistic('tests-total', 0)
643 +
        self.setStatistic('tests-total', total)
644 +
        failed += self.getStatistic('tests-failed', 0)
645 +
        self.setStatistic('tests-failed', failed)
646 +
        warnings += self.getStatistic('tests-warnings', 0)
647 +
        self.setStatistic('tests-warnings', warnings)
648 +
        passed += self.getStatistic('tests-passed', 0)
649 +
        self.setStatistic('tests-passed', passed)
650 +
651 +
    def describe(self, done=False):
652 +
        description = super().describe(done)
653 +
        if done:
654 +
            if not description:
655 +
                description = []
656 +
            description = description[:]  # make a private copy
657 +
            if self.hasStatistic('tests-total'):
658 +
                total = self.getStatistic("tests-total", 0)
659 +
                failed = self.getStatistic("tests-failed", 0)
660 +
                passed = self.getStatistic("tests-passed", 0)
661 +
                warnings = self.getStatistic("tests-warnings", 0)
662 +
                if not total:
663 +
                    total = failed + passed + warnings
664 +
665 +
                if total:
666 +
                    description.append('%d tests' % total)
667 +
                if passed:
668 +
                    description.append('%d passed' % passed)
669 +
                if warnings:
670 +
                    description.append('%d warnings' % warnings)
671 +
                if failed:
672 +
                    description.append('%d failed' % failed)
673 +
        return description

@@ -16,15 +16,12 @@
Loading
16 16
import re
17 17
18 18
from twisted.internet import defer
19 -
from twisted.python import failure
20 -
from twisted.python import log
21 19
from twisted.python.deprecate import deprecatedModuleAttribute
22 20
from twisted.python.versions import Version
23 21
24 22
from buildbot import config
25 23
from buildbot.process import buildstep
26 24
from buildbot.process import logobserver
27 -
from buildbot.process import remotecommand
28 25
# for existing configurations that import WithProperties from here.  We like
29 26
# to move this class around just to keep our readers guessing.
30 27
from buildbot.process.properties import WithProperties
@@ -33,254 +30,27 @@
Loading
33 30
from buildbot.process.results import WARNINGS
34 31
from buildbot.process.results import Results
35 32
from buildbot.process.results import worst_status
33 +
from buildbot.steps.shell_oldstyle import Compile
34 +
from buildbot.steps.shell_oldstyle import Configure
35 +
from buildbot.steps.shell_oldstyle import SetPropertyFromCommand
36 +
from buildbot.steps.shell_oldstyle import ShellCommand
37 +
from buildbot.steps.shell_oldstyle import Test
38 +
from buildbot.steps.shell_oldstyle import WarningCountingShellCommand
36 39
from buildbot.steps.worker import CompositeStepMixin
37 -
from buildbot.util import command_to_string
38 -
from buildbot.util import flatten
39 40
from buildbot.util import join_list
40 41
41 -
_hush_pyflakes = [WithProperties]
42 +
_hush_pyflakes = [
43 +
    WithProperties,
44 +
    Configure,
45 +
    Compile,
46 +
    ShellCommand,
47 +
    SetPropertyFromCommand,
48 +
    Test,
49 +
    WarningCountingShellCommand,
50 +
]
42 51
del _hush_pyflakes
43 52
44 53
45 -
class ShellCommand(buildstep.LoggingBuildStep):
46 -
47 -
    """I run a single shell command on the worker. I return FAILURE if
48 -
    the exit code of that command is non-zero, SUCCESS otherwise. To change
49 -
    this behavior, override my .evaluateCommand method, or customize
50 -
    decodeRC argument
51 -
52 -
    By default, a failure of this step will mark the whole build as FAILURE.
53 -
    To override this, give me an argument of flunkOnFailure=False .
54 -
55 -
    I create a single Log named 'log' which contains the output of the
56 -
    command. To create additional summary Logs, override my .createSummary
57 -
    method.
58 -
59 -
    The shell command I run (a list of argv strings) can be provided in
60 -
    several ways:
61 -
      - a class-level .command attribute
62 -
      - a command= parameter to my constructor (overrides .command)
63 -
      - set explicitly with my .setCommand() method (overrides both)
64 -
65 -
    @ivar command: a list of renderable objects (typically strings or
66 -
                   WithProperties instances). This will be used by start()
67 -
                   to create a RemoteShellCommand instance.
68 -
69 -
    @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
70 -
                    of their corresponding logfiles. The contents of the file
71 -
                    named FILENAME will be put into a LogFile named NAME, ina
72 -
                    something approximating real-time. (note that logfiles=
73 -
                    is actually handled by our parent class LoggingBuildStep)
74 -
75 -
    @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
76 -
                        `lazily', meaning they will only be added when and if
77 -
                        they are written to. Empty or nonexistent logfiles
78 -
                        will be omitted. (Also handled by class
79 -
                        LoggingBuildStep.)
80 -
    """
81 -
82 -
    name = "shell"
83 -
    renderables = [
84 -
        'command',
85 -
        'flunkOnFailure',
86 -
        'haltOnFailure',
87 -
        'remote_kwargs',
88 -
        'workerEnvironment'
89 -
    ]
90 -
91 -
    command = None  # set this to a command, or set in kwargs
92 -
    # logfiles={} # you can also set 'logfiles' to a dictionary, and it
93 -
    #               will be merged with any logfiles= argument passed in
94 -
    #               to __init__
95 -
96 -
    # override this on a specific ShellCommand if you want to let it fail
97 -
    # without dooming the entire build to a status of FAILURE
98 -
    flunkOnFailure = True
99 -
100 -
    def __init__(self, workdir=None,
101 -
                 command=None,
102 -
                 usePTY=None,
103 -
                 **kwargs):
104 -
        # most of our arguments get passed through to the RemoteShellCommand
105 -
        # that we create, but first strip out the ones that we pass to
106 -
        # BuildStep (like haltOnFailure and friends), and a couple that we
107 -
        # consume ourselves.
108 -
109 -
        if command:
110 -
            self.setCommand(command)
111 -
112 -
        if self.__class__ is ShellCommand and not command:
113 -
            # ShellCommand class is directly instantiated.
114 -
            # Explicitly check that command is set to prevent runtime error
115 -
            # later.
116 -
            config.error("ShellCommand's `command' argument is not specified")
117 -
118 -
        # pull out the ones that LoggingBuildStep wants, then upcall
119 -
        buildstep_kwargs = {}
120 -
        # workdir is here first positional argument, but it belongs to
121 -
        # BuildStep parent
122 -
        kwargs['workdir'] = workdir
123 -
        for k in list(kwargs):
124 -
            if k in self.__class__.parms:
125 -
                buildstep_kwargs[k] = kwargs[k]
126 -
                del kwargs[k]
127 -
        super().__init__(**buildstep_kwargs)
128 -
129 -
        # check validity of arguments being passed to RemoteShellCommand
130 -
        invalid_args = []
131 -
        valid_rsc_args = [
132 -
            'env',
133 -
            'want_stdout',
134 -
            'want_stderr',
135 -
            'timeout',
136 -
            'maxTime',
137 -
            'sigtermTime',
138 -
            'logfiles',
139 -
            'usePTY',
140 -
            'logEnviron',
141 -
            'collectStdout',
142 -
            'collectStderr',
143 -
            'interruptSignal',
144 -
            'initialStdin',
145 -
            'decodeRC',
146 -
            'stdioLogName',
147 -
        ]
148 -
        for arg in kwargs:
149 -
            if arg not in valid_rsc_args:
150 -
                invalid_args.append(arg)
151 -
        # Raise Configuration error in case invalid arguments are present
152 -
        if invalid_args:
153 -
            config.error("Invalid argument(s) passed to RemoteShellCommand: " +
154 -
                         ', '.join(invalid_args))
155 -
156 -
        # everything left over goes to the RemoteShellCommand
157 -
        kwargs['usePTY'] = usePTY
158 -
        self.remote_kwargs = kwargs
159 -
        self.remote_kwargs['workdir'] = workdir
160 -
161 -
    def setBuild(self, build):
162 -
        super().setBuild(build)
163 -
        # Set this here, so it gets rendered when we start the step
164 -
        self.workerEnvironment = self.build.workerEnvironment
165 -
166 -
    def setCommand(self, command):
167 -
        self.command = command
168 -
169 -
    def _describe(self, done=False):
170 -
        return None
171 -
172 -
    def describe(self, done=False):
173 -
        if self.stopped and not self.rendered:
174 -
            return "stopped early"
175 -
        assert(self.rendered)
176 -
        desc = self._describe(done)
177 -
        if not desc:
178 -
            return None
179 -
        if self.descriptionSuffix:
180 -
            desc = desc + ' ' + join_list(self.descriptionSuffix)
181 -
        return desc
182 -
183 -
    def getCurrentSummary(self):
184 -
        cmdsummary = self._getLegacySummary(False)
185 -
        if cmdsummary:
186 -
            return {'step': cmdsummary}
187 -
        return super().getCurrentSummary()
188 -
189 -
    def getResultSummary(self):
190 -
        cmdsummary = self._getLegacySummary(True)
191 -
192 -
        if cmdsummary:
193 -
            if self.results != SUCCESS:
194 -
                cmdsummary += ' ({})'.format(Results[self.results])
195 -
            return {'step': cmdsummary}
196 -
197 -
        return super().getResultSummary()
198 -
199 -
    def _getLegacySummary(self, done):
200 -
        # defer to the describe method, if set
201 -
        description = self.describe(done)
202 -
        if description:
203 -
            return join_list(description)
204 -
205 -
        # defer to descriptions, if they're set
206 -
        if (not done and self.description) or (done and self.descriptionDone):
207 -
            return None
208 -
209 -
        try:
210 -
            # if self.cmd is set, then use the RemoteCommand's info
211 -
            if self.cmd:
212 -
                command = self.cmd.remote_command
213 -
            # otherwise, if we were configured with a command, use that
214 -
            elif self.command:
215 -
                command = self.command
216 -
            else:
217 -
                return None
218 -
219 -
            rv = command_to_string(command)
220 -
221 -
            # add the descriptionSuffix, if one was given
222 -
            if self.descriptionSuffix:
223 -
                rv = rv + ' ' + join_list(self.descriptionSuffix)
224 -
225 -
            return rv
226 -
227 -
        except Exception:
228 -
            log.err(failure.Failure(), "Error describing step")
229 -
            return None
230 -
231 -
    def setupEnvironment(self, cmd):
232 -
        # merge in anything from workerEnvironment (which comes from the builder
233 -
        # config) Environment variables passed in by a BuildStep override those
234 -
        # passed in at the Builder level, so if we have any from the builder,
235 -
        # apply those and then update with the args from the buildstep
236 -
        # (cmd.args)
237 -
        workerEnv = self.workerEnvironment
238 -
        if workerEnv:
239 -
            if cmd.args['env'] is None:
240 -
                cmd.args['env'] = {}
241 -
            fullWorkerEnv = workerEnv.copy()
242 -
            fullWorkerEnv.update(cmd.args['env'])
243 -
            cmd.args['env'] = fullWorkerEnv
244 -
            # note that each RemoteShellCommand gets its own copy of the
245 -
            # dictionary, so we shouldn't be affecting anyone but ourselves.
246 -
247 -
    def buildCommandKwargs(self, warnings):
248 -
        kwargs = super().buildCommandKwargs()
249 -
        kwargs.update(self.remote_kwargs)
250 -
        kwargs['workdir'] = self.workdir
251 -
252 -
        kwargs['command'] = flatten(self.command, (list, tuple))
253 -
254 -
        # check for the usePTY flag
255 -
        if 'usePTY' in kwargs and kwargs['usePTY'] is not None:
256 -
            if self.workerVersionIsOlderThan("shell", "2.7"):
257 -
                warnings.append(
258 -
                    "NOTE: worker does not allow master to override usePTY\n")
259 -
                del kwargs['usePTY']
260 -
261 -
        # check for the interruptSignal flag
262 -
        if "interruptSignal" in kwargs and self.workerVersionIsOlderThan("shell", "2.15"):
263 -
            warnings.append(
264 -
                "NOTE: worker does not allow master to specify interruptSignal\n")
265 -
            del kwargs['interruptSignal']
266 -
267 -
        return kwargs
268 -
269 -
    def start(self):
270 -
        # this block is specific to ShellCommands. subclasses that don't need
271 -
        # to set up an argv array, an environment, or extra logfiles= (like
272 -
        # the Source subclasses) can just skip straight to startCommand()
273 -
274 -
        warnings = []
275 -
276 -
        # create the actual RemoteShellCommand instance now
277 -
        kwargs = self.buildCommandKwargs(warnings)
278 -
        cmd = remotecommand.RemoteShellCommand(**kwargs)
279 -
        self.setupEnvironment(cmd)
280 -
281 -
        self.startCommand(cmd, warnings)
282 -
283 -
284 54
class TreeSize(buildstep.ShellMixin, buildstep.BuildStep):
285 55
    name = "treesize"
286 56
    command = ["du", "-s", "-k", "."]
@@ -320,7 +90,7 @@
Loading
320 90
        return SUCCESS
321 91
322 92
323 -
class SetPropertyFromCommand(buildstep.ShellMixin, buildstep.BuildStep):
93 +
class SetPropertyFromCommandNewStyle(buildstep.ShellMixin, buildstep.BuildStep):
324 94
    name = "setproperty"
325 95
    renderables = ['property']
326 96
@@ -399,6 +169,41 @@
Loading
399 169
class ShellCommandNewStyle(buildstep.ShellMixin, buildstep.BuildStep):
400 170
    # This is a temporary class until old ShellCommand is retired
401 171
    def __init__(self, **kwargs):
172 +
173 +
        if self.__class__ is ShellCommandNewStyle:
174 +
            if 'command' not in kwargs:
175 +
                config.error("ShellCommandNewStyle's `command' argument is not specified")
176 +
177 +
            # check validity of arguments being passed to RemoteShellCommand
178 +
            valid_rsc_args = [
179 +
                'command',
180 +
                'env',
181 +
                'want_stdout',
182 +
                'want_stderr',
183 +
                'timeout',
184 +
                'maxTime',
185 +
                'sigtermTime',
186 +
                'logfiles',
187 +
                'usePTY',
188 +
                'logEnviron',
189 +
                'collectStdout',
190 +
                'collectStderr',
191 +
                'interruptSignal',
192 +
                'initialStdin',
193 +
                'decodeRC',
194 +
                'stdioLogName',
195 +
                'workdir',
196 +
            ] + buildstep.BuildStep.parms
197 +
198 +
            invalid_args = []
199 +
            for arg in kwargs:
200 +
                if arg not in valid_rsc_args:
201 +
                    invalid_args.append(arg)
202 +
203 +
            if invalid_args:
204 +
                config.error("Invalid argument(s) passed to ShellCommandNewStyle: " +
205 +
                             ', '.join(invalid_args))
206 +
402 207
        kwargs = self.setupShellMixin(kwargs)
403 208
        super().__init__(**kwargs)
404 209
@@ -409,7 +214,7 @@
Loading
409 214
        return cmd.results()
410 215
411 216
412 -
class Configure(ShellCommandNewStyle):
217 +
class ConfigureNewStyle(ShellCommandNewStyle):
413 218
    name = "configure"
414 219
    haltOnFailure = 1
415 220
    flunkOnFailure = 1
@@ -418,7 +223,8 @@
Loading
418 223
    command = ["./configure"]
419 224
420 225
421 -
class WarningCountingShellCommand(buildstep.ShellMixin, CompositeStepMixin, buildstep.BuildStep):
226 +
class WarningCountingShellCommandNewStyle(buildstep.ShellMixin, CompositeStepMixin,
227 +
                                          buildstep.BuildStep):
422 228
    renderables = [
423 229
        'suppressionFile',
424 230
        'suppressionList',
@@ -463,12 +269,12 @@
Loading
463 269
            self.warningExtractor = WarningCountingShellCommand.warnExtractWholeLine
464 270
        self.maxWarnCount = maxWarnCount
465 271
466 -
        if self.__class__ is WarningCountingShellCommand and \
272 +
        if self.__class__ is WarningCountingShellCommandNewStyle and \
467 273
                not kwargs.get('command'):
468 -
            # WarningCountingShellCommand class is directly instantiated.
274 +
            # WarningCountingShellCommandNewStyle class is directly instantiated.
469 275
            # Explicitly check that command is set to prevent runtime error
470 276
            # later.
471 -
            config.error("WarningCountingShellCommand's `command' argument "
277 +
            config.error("WarningCountingShellCommandNewStyle's `command' argument "
472 278
                         "is not specified")
473 279
474 280
        kwargs = self.setupShellMixin(kwargs)
@@ -662,7 +468,7 @@
Loading
662 468
        return result
663 469
664 470
665 -
class Compile(WarningCountingShellCommand):
471 +
class CompileNewStyle(WarningCountingShellCommandNewStyle):
666 472
667 473
    name = "compile"
668 474
    haltOnFailure = 1
@@ -672,7 +478,7 @@
Loading
672 478
    command = ["make", "all"]
673 479
674 480
675 -
class Test(WarningCountingShellCommand):
481 +
class TestNewStyle(WarningCountingShellCommandNewStyle):
676 482
677 483
    name = "test"
678 484
    warnOnFailure = 1
@@ -770,7 +576,7 @@
Loading
770 576
                self.total = int(mo.group(1))
771 577
772 578
773 -
class PerlModuleTest(Test):
579 +
class PerlModuleTest(TestNewStyle):
774 580
    command = ["prove", "--lib", "lib", "-r", "t"]
775 581
    total = 0
776 582

@@ -0,0 +1,176 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
import os
17 +
import re
18 +
19 +
from twisted.internet import defer
20 +
from twisted.internet import error
21 +
from twisted.internet import reactor
22 +
from twisted.internet.protocol import ProcessProtocol
23 +
from twisted.python import runtime
24 +
25 +
from buildbot.process.buildstep import FAILURE
26 +
from buildbot.process.buildstep import SUCCESS
27 +
from buildbot.process.buildstep import BuildStep
28 +
from buildbot.util import deferwaiter
29 +
30 +
31 +
class MasterShellCommand(BuildStep):
32 +
33 +
    """
34 +
    Run a shell command locally - on the buildmaster.  The shell command
35 +
    COMMAND is specified just as for a RemoteShellCommand.  Note that extra
36 +
    logfiles are not supported.
37 +
    """
38 +
    name = 'MasterShellCommand'
39 +
    description = 'Running'
40 +
    descriptionDone = 'Ran'
41 +
    descriptionSuffix = None
42 +
    renderables = ['command', 'env']
43 +
    haltOnFailure = True
44 +
    flunkOnFailure = True
45 +
46 +
    def __init__(self, command, **kwargs):
47 +
        self.env = kwargs.pop('env', None)
48 +
        self.usePTY = kwargs.pop('usePTY', 0)
49 +
        self.interruptSignal = kwargs.pop('interruptSignal', 'KILL')
50 +
        self.logEnviron = kwargs.pop('logEnviron', True)
51 +
52 +
        super().__init__(**kwargs)
53 +
54 +
        self.command = command
55 +
        self.masterWorkdir = self.workdir
56 +
        self._deferwaiter = deferwaiter.DeferWaiter()
57 +
        self._status_object = None
58 +
59 +
    class LocalPP(ProcessProtocol):
60 +
61 +
        def __init__(self, step):
62 +
            self.step = step
63 +
            self._finish_d = defer.Deferred()
64 +
            self.step._deferwaiter.add(self._finish_d)
65 +
66 +
        def outReceived(self, data):
67 +
            self.step._deferwaiter.add(self.step.stdio_log.addStdout(data))
68 +
69 +
        def errReceived(self, data):
70 +
            self.step._deferwaiter.add(self.step.stdio_log.addStderr(data))
71 +
72 +
        def processEnded(self, status_object):
73 +
            if status_object.value.exitCode is not None:
74 +
                msg = "exit status {}\n".format(status_object.value.exitCode)
75 +
                self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg))
76 +
77 +
            if status_object.value.signal is not None:
78 +
                msg = "signal {}\n".format(status_object.value.signal)
79 +
                self.step._deferwaiter.add(self.step.stdio_log.addHeader(msg))
80 +
81 +
            self.step._status_object = status_object
82 +
            self._finish_d.callback(None)
83 +
84 +
    @defer.inlineCallbacks
85 +
    def run(self):
86 +
        # render properties
87 +
        command = self.command
88 +
        # set up argv
89 +
        if isinstance(command, (str, bytes)):
90 +
            if runtime.platformType == 'win32':
91 +
                # allow %COMSPEC% to have args
92 +
                argv = os.environ['COMSPEC'].split()
93 +
                if '/c' not in argv:
94 +
                    argv += ['/c']
95 +
                argv += [command]
96 +
            else:
97 +
                # for posix, use /bin/sh. for other non-posix, well, doesn't
98 +
                # hurt to try
99 +
                argv = ['/bin/sh', '-c', command]
100 +
        else:
101 +
            if runtime.platformType == 'win32':
102 +
                # allow %COMSPEC% to have args
103 +
                argv = os.environ['COMSPEC'].split()
104 +
                if '/c' not in argv:
105 +
                    argv += ['/c']
106 +
                argv += list(command)
107 +
            else:
108 +
                argv = command
109 +
110 +
        self.stdio_log = yield self.addLog("stdio")
111 +
112 +
        if isinstance(command, (str, bytes)):
113 +
            yield self.stdio_log.addHeader(command.strip() + "\n\n")
114 +
        else:
115 +
            yield self.stdio_log.addHeader(" ".join(command) + "\n\n")
116 +
        yield self.stdio_log.addHeader("** RUNNING ON BUILDMASTER **\n")
117 +
        yield self.stdio_log.addHeader(" in dir {}\n".format(os.getcwd()))
118 +
        yield self.stdio_log.addHeader(" argv: {}\n".format(argv))
119 +
120 +
        if self.env is None:
121 +
            env = os.environ
122 +
        else:
123 +
            assert isinstance(self.env, dict)
124 +
            env = self.env
125 +
            for key, v in self.env.items():
126 +
                if isinstance(v, list):
127 +
                    # Need to do os.pathsep translation.  We could either do that
128 +
                    # by replacing all incoming ':'s with os.pathsep, or by
129 +
                    # accepting lists.  I like lists better.
130 +
                    # If it's not a string, treat it as a sequence to be
131 +
                    # turned in to a string.
132 +
                    self.env[key] = os.pathsep.join(self.env[key])
133 +
134 +
            # do substitution on variable values matching pattern: ${name}
135 +
            p = re.compile(r'\${([0-9a-zA-Z_]*)}')
136 +
137 +
            def subst(match):
138 +
                return os.environ.get(match.group(1), "")
139 +
            newenv = {}
140 +
            for key, v in env.items():
141 +
                if v is not None:
142 +
                    if not isinstance(v, (str, bytes)):
143 +
                        raise RuntimeError(("'env' values must be strings or "
144 +
                                            "lists; key '{}' is incorrect").format(key))
145 +
                    newenv[key] = p.sub(subst, env[key])
146 +
            env = newenv
147 +
148 +
        if self.logEnviron:
149 +
            yield self.stdio_log.addHeader(" env: %r\n" % (env,))
150 +
151 +
        # TODO add a timeout?
152 +
        self.process = reactor.spawnProcess(self.LocalPP(self), argv[0], argv,
153 +
                                            path=self.masterWorkdir, usePTY=self.usePTY, env=env)
154 +
155 +
        # self._deferwaiter will yield only after LocalPP finishes
156 +
157 +
        yield self._deferwaiter.wait()
158 +
159 +
        status_value = self._status_object.value
160 +
        if status_value.signal is not None:
161 +
            self.descriptionDone = ["killed ({})".format(status_value.signal)]
162 +
            return FAILURE
163 +
        elif status_value.exitCode != 0:
164 +
            self.descriptionDone = ["failed ({})".format(status_value.exitCode)]
165 +
            return FAILURE
166 +
        else:
167 +
            return SUCCESS
168 +
169 +
    def interrupt(self, reason):
170 +
        try:
171 +
            self.process.signalProcess(self.interruptSignal)
172 +
        except KeyError:  # Process not started yet
173 +
            pass
174 +
        except error.ProcessExitedAlready:
175 +
            pass
176 +
        super().interrupt(reason)
Files Coverage
master/buildbot 90.18%
worker/buildbot_worker 85.09%
Project Totals (352 files) 89.88%
13418.3
TRAVIS_PYTHON_VERSION=3.8
TRAVIS_OS_NAME=linux
13418.2
TRAVIS_PYTHON_VERSION=3.8
TRAVIS_OS_NAME=linux
Untitled
13418.1
TRAVIS_PYTHON_VERSION=3.8
TRAVIS_OS_NAME=linux

No yaml found.

Create your codecov.yml to customize your Codecov experience

Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading