buildbot / buildbot
1
# This file is part of Buildbot.  Buildbot is free software: you can
2
# redistribute it and/or modify it under the terms of the GNU General Public
3
# License as published by the Free Software Foundation, version 2.
4
#
5
# This program is distributed in the hope that it will be useful, but WITHOUT
6
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8
# details.
9
#
10
# You should have received a copy of the GNU General Public License along with
11
# this program; if not, write to the Free Software Foundation, Inc., 51
12
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13
#
14
# Portions Copyright Buildbot Team Members
15
# Portions Copyright Canonical Ltd. 2009
16

17 6
import time
18

19 6
from twisted.internet import defer
20 6
from twisted.python import log
21 6
from twisted.python.reflect import namedModule
22 6
from zope.interface import implementer
23

24 6
from buildbot import config
25 6
from buildbot.interfaces import IWorker
26 6
from buildbot.process import metrics
27 6
from buildbot.process.properties import Properties
28 6
from buildbot.util import Notifier
29 6
from buildbot.util import bytes2unicode
30 6
from buildbot.util import service
31

32

33 6
@implementer(IWorker)
34 6
class AbstractWorker(service.BuildbotService):
35

36
    """This is the master-side representative for a remote buildbot worker.
37
    There is exactly one for each worker described in the config file (the
38
    c['workers'] list). When buildbots connect in (.attach), they get a
39
    reference to this instance. The BotMaster object is stashed as the
40
    .botmaster attribute. The BotMaster is also our '.parent' Service.
41

42
    I represent a worker -- a remote machine capable of
43
    running builds.  I am instantiated by the configuration file, and can be
44
    subclassed to add extra functionality."""
45

46
    # reconfig workers after builders
47 6
    reconfig_priority = 64
48

49 6
    quarantine_timer = None
50 6
    quarantine_timeout = quarantine_initial_timeout = 10
51 6
    quarantine_max_timeout = 60 * 60
52 6
    start_missing_on_startup = True
53 6
    DEFAULT_MISSING_TIMEOUT = 3600
54 6
    DEFAULT_KEEPALIVE_INTERVAL = 3600
55

56
    # override to True if isCompatibleWithBuild may return False
57 6
    builds_may_be_incompatible = False
58

59 6
    def checkConfig(self, name, password, max_builds=None,
60
                    notify_on_missing=None,
61
                    missing_timeout=None,
62
                    properties=None, defaultProperties=None,
63
                    locks=None,
64
                    keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL,
65
                    machine_name=None):
66
        """
67
        @param name: botname this machine will supply when it connects
68
        @param password: password this machine will supply when
69
                         it connects
70
        @param max_builds: maximum number of simultaneous builds that will
71
                           be run concurrently on this worker (the
72
                           default is None for no limit)
73
        @param properties: properties that will be applied to builds run on
74
                           this worker
75
        @type properties: dictionary
76
        @param defaultProperties: properties that will be applied to builds
77
                                  run on this worker only if the property
78
                                  has not been set by another source
79
        @type defaultProperties: dictionary
80
        @param locks: A list of locks that must be acquired before this worker
81
                      can be used
82
        @type locks: dictionary
83
        @param machine_name: The name of the machine to associate with the
84
                             worker.
85
        """
86 6
        self.name = name = bytes2unicode(name)
87 6
        self.machine_name = machine_name
88

89 6
        self.password = password
90

91
        # protocol registration
92 6
        self.registration = None
93

94 6
        self._graceful = False
95 6
        self._paused = False
96

97
        # these are set when the service is started
98 6
        self.manager = None
99 6
        self.workerid = None
100

101 6
        self.info = Properties()
102 6
        self.worker_commands = None
103 6
        self.workerforbuilders = {}
104 6
        self.max_builds = max_builds
105 6
        self.access = []
106 6
        if locks:
107 6
            self.access = locks
108 6
        self.lock_subscriptions = []
109

110 6
        self.properties = Properties()
111 6
        self.properties.update(properties or {}, "Worker")
112 6
        self.properties.setProperty("workername", name, "Worker")
113 6
        self.defaultProperties = Properties()
114 6
        self.defaultProperties.update(defaultProperties or {}, "Worker")
115

116 6
        if self.machine_name is not None:
117 6
            self.properties.setProperty('machine_name', self.machine_name,
118
                                        'Worker')
119 6
        self.machine = None
120

121 6
        self.lastMessageReceived = 0
122

123 6
        if notify_on_missing is None:
124 6
            notify_on_missing = []
125 6
        if isinstance(notify_on_missing, str):
126 6
            notify_on_missing = [notify_on_missing]
127 6
        self.notify_on_missing = notify_on_missing
128 6
        for i in notify_on_missing:
129 6
            if not isinstance(i, str):
130 6
                config.error(
131
                    'notify_on_missing arg %r is not a string' % (i,))
132

133 6
        self.missing_timeout = missing_timeout
134 6
        self.missing_timer = None
135

136
        # a protocol connection, if we're currently connected
137 6
        self.conn = None
138

139
        # during disconnection self.conn will be set to None before all disconnection notifications
140
        # are delivered. During that period _pending_conn_shutdown_notifier will be set to
141
        # a notifier and allows interested users to wait until all disconnection notifications are
142
        # delivered.
143 6
        self._pending_conn_shutdown_notifier = None
144

145 6
        self._old_builder_list = None
146 6
        self._configured_builderid_list = None
147

148
    def __repr__(self):
149
        return "<{} {}>".format(self.__class__.__name__, repr(self.name))
150

151 6
    @property
152 6
    def workername(self):
153
        # workername is now an alias to twisted.Service's name
154 6
        return self.name
155

156 6
    @property
157 6
    def botmaster(self):
158 6
        if self.master is None:
159 0
            return None
160 6
        return self.master.botmaster
161

162 6
    @defer.inlineCallbacks
163 6
    def updateLocks(self):
164
        """Convert the L{LockAccess} objects in C{self.locks} into real lock
165
        objects, while also maintaining the subscriptions to lock releases."""
166
        # unsubscribe from any old locks
167 6
        for s in self.lock_subscriptions:
168 6
            s.unsubscribe()
169

170
        # convert locks into their real form
171 6
        locks = yield self.botmaster.getLockFromLockAccesses(self.access, self.config_version)
172

173 6
        self.locks = [(l.getLockForWorker(self.workername), la)
174
                      for l, la in locks]
175 6
        self.lock_subscriptions = [l.subscribeToReleases(self._lockReleased)
176
                                   for l, la in self.locks]
177

178 6
    def locksAvailable(self):
179
        """
180
        I am called to see if all the locks I depend on are available,
181
        in which I return True, otherwise I return False
182
        """
183 6
        if not self.locks:
184 6
            return True
185 0
        for lock, access in self.locks:
186 0
            if not lock.isAvailable(self, access):
187 0
                return False
188 0
        return True
189

190 6
    def acquireLocks(self):
191
        """
192
        I am called when a build is preparing to run. I try to claim all
193
        the locks that are needed for a build to happen. If I can't, then
194
        my caller should give up the build and try to get another worker
195
        to look at it.
196
        """
197 6
        log.msg("acquireLocks(worker {}, locks {})".format(self, self.locks))
198 6
        if not self.locksAvailable():
199 0
            log.msg("worker {} can't lock, giving up".format(self))
200 0
            return False
201
        # all locks are available, claim them all
202 6
        for lock, access in self.locks:
203 0
            lock.claim(self, access)
204 6
        return True
205

206 6
    def releaseLocks(self):
207
        """
208
        I am called to release any locks after a build has finished
209
        """
210 6
        log.msg("releaseLocks({}): {}".format(self, self.locks))
211 6
        for lock, access in self.locks:
212 0
            lock.release(self, access)
213

214 6
    def _lockReleased(self):
215
        """One of the locks for this worker was released; try scheduling
216
        builds."""
217 0
        if not self.botmaster:
218 0
            return  # oh well..
219 0
        self.botmaster.maybeStartBuildsForWorker(self.name)
220

221 6
    def _applyWorkerInfo(self, info):
222 6
        if not info:
223 6
            return
224

225
        # set defaults
226 6
        self.info.setProperty("version", "(unknown)", "Worker")
227

228
        # store everything as Properties
229 6
        for k, v in info.items():
230 6
            if k in ('environ', 'worker_commands'):
231 6
                continue
232 6
            self.info.setProperty(k, v, "Worker")
233

234 6
    @defer.inlineCallbacks
235 6
    def _getWorkerInfo(self):
236 6
        worker = yield self.master.data.get(
237
            ('workers', self.workerid))
238 6
        self._applyWorkerInfo(worker['workerinfo'])
239

240 6
    def setServiceParent(self, parent):
241
        # botmaster needs to set before setServiceParent which calls
242
        # startService
243

244 6
        self.manager = parent
245 6
        return super().setServiceParent(parent)
246

247 6
    @defer.inlineCallbacks
248 6
    def startService(self):
249
        # tracks config version for locks
250 6
        self.config_version = self.master.config_version
251

252 6
        self.updateLocks()
253 6
        self.workerid = yield self.master.data.updates.findWorkerId(
254
            self.name)
255

256 6
        self.workerActionConsumer = yield self.master.mq.startConsuming(self.controlWorker,
257
                                                                        ("control", "worker",
258
                                                                        str(self.workerid),
259
                                                                        None))
260

261 6
        yield self._getWorkerInfo()
262 6
        yield super().startService()
263

264
        # startMissingTimer wants the service to be running to really start
265 6
        if self.start_missing_on_startup:
266 6
            self.startMissingTimer()
267

268 6
    @defer.inlineCallbacks
269 6
    def reconfigService(self, name, password, max_builds=None,
270
                        notify_on_missing=None, missing_timeout=DEFAULT_MISSING_TIMEOUT,
271
                        properties=None, defaultProperties=None,
272
                        locks=None,
273
                        keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL,
274
                        machine_name=None):
275
        # Given a Worker config arguments, configure this one identically.
276
        # Because Worker objects are remotely referenced, we can't replace them
277
        # without disconnecting the worker, yet there's no reason to do that.
278

279 6
        assert self.name == name
280 6
        self.password = password
281

282
        # adopt new instance's configuration parameters
283 6
        self.max_builds = max_builds
284 6
        self.access = []
285 6
        if locks:
286 6
            self.access = locks
287 6
        if notify_on_missing is None:
288 6
            notify_on_missing = []
289 6
        if isinstance(notify_on_missing, str):
290 6
            notify_on_missing = [notify_on_missing]
291 6
        self.notify_on_missing = notify_on_missing
292

293 6
        if self.missing_timeout != missing_timeout:
294 6
            running_missing_timer = self.missing_timer
295 6
            self.stopMissingTimer()
296 6
            self.missing_timeout = missing_timeout
297 6
            if running_missing_timer:
298 6
                self.startMissingTimer()
299

300 6
        self.properties = Properties()
301 6
        self.properties.update(properties or {}, "Worker")
302 6
        self.properties.setProperty("workername", name, "Worker")
303 6
        self.defaultProperties = Properties()
304 6
        self.defaultProperties.update(defaultProperties or {}, "Worker")
305

306
        # Note that before first reconfig self.machine will always be None and
307
        # out of sync with self.machine_name, thus more complex logic is needed.
308 6
        if self.machine is not None and self.machine_name != machine_name:
309 6
            self.machine.unregisterWorker(self)
310 6
            self.machine = None
311

312 6
        self.machine_name = machine_name
313 6
        if self.machine is None and self.machine_name is not None:
314 6
            self.machine = self.master.machine_manager.getMachineByName(self.machine_name)
315 6
            if self.machine is not None:
316 6
                self.machine.registerWorker(self)
317 6
                self.properties.setProperty("machine_name", self.machine_name,
318
                                            "Worker")
319
            else:
320 6
                log.err("Unknown machine '{}' for worker '{}'".format(
321
                    self.machine_name, self.name))
322

323
        # update our records with the worker manager
324 6
        if not self.registration:
325 6
            self.registration = yield self.master.workers.register(self)
326 6
        yield self.registration.update(self, self.master.config)
327

328
        # tracks config version for locks
329 6
        self.config_version = self.master.config_version
330 6
        self.updateLocks()
331

332 6
    @defer.inlineCallbacks
333 6
    def reconfigServiceWithSibling(self, sibling):
334
        # reconfigServiceWithSibling will only reconfigure the worker when it is configured
335
        # differently.
336
        # However, the worker configuration depends on which builder it is configured
337 6
        yield super().reconfigServiceWithSibling(sibling)
338

339
        # update the attached worker's notion of which builders are attached.
340
        # This assumes that the relevant builders have already been configured,
341
        # which is why the reconfig_priority is set low in this class.
342 6
        bids = [
343
            b.getBuilderId() for b in self.botmaster.getBuildersForWorker(self.name)]
344 6
        bids = yield defer.gatherResults(bids, consumeErrors=True)
345 6
        if self._configured_builderid_list != bids:
346 6
            yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid,
347
                                                            bids)
348 6
            yield self.updateWorker()
349 6
            self._configured_builderid_list = bids
350

351 6
    @defer.inlineCallbacks
352 6
    def stopService(self):
353 6
        if self.registration:
354 6
            yield self.registration.unregister()
355 6
            self.registration = None
356 6
        self.workerActionConsumer.stopConsuming()
357 6
        self.stopMissingTimer()
358 6
        self.stopQuarantineTimer()
359
        # mark this worker as configured for zero builders in this master
360 6
        yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, [])
361

362
        # during master shutdown we need to wait until the disconnection notification deliveries
363
        # are completed, otherwise some of the events may still be firing long after the master
364
        # is completely shut down.
365 6
        yield self.disconnect()
366 6
        yield self.waitForCompleteShutdown()
367

368 6
        yield super().stopService()
369

370 6
    def isCompatibleWithBuild(self, build_props):
371
        # given a build properties object, determines whether the build is
372
        # compatible with the currently running worker or not. This is most
373
        # often useful for latent workers where it's possible to request
374
        # different kinds of workers.
375 0
        return defer.succeed(True)
376

377 6
    def startMissingTimer(self):
378 6
        if self.missing_timeout and self.parent and self.running:
379 6
            self.stopMissingTimer()  # in case it's already running
380 6
            self.missing_timer = self.master.reactor.callLater(self.missing_timeout,
381
                                                               self._missing_timer_fired)
382

383 6
    def stopMissingTimer(self):
384 6
        if self.missing_timer:
385 6
            if self.missing_timer.active():
386 6
                self.missing_timer.cancel()
387 6
            self.missing_timer = None
388

389 6
    def isConnected(self):
390 0
        return self.conn
391

392 6
    def _missing_timer_fired(self):
393 6
        self.missing_timer = None
394
        # notify people, but only if we're still in the config
395 6
        if not self.parent:
396 0
            return
397 6
        last_connection = time.ctime(time.time() - self.missing_timeout)
398 6
        self.master.data.updates.workerMissing(
399
            workerid=self.workerid,
400
            masterid=self.master.masterid,
401
            last_connection=last_connection,
402
            notify=self.notify_on_missing
403
        )
404

405 6
    def updateWorker(self):
406
        """Called to add or remove builders after the worker has connected.
407

408
        @return: a Deferred that indicates when an attached worker has
409
        accepted the new builders and/or released the old ones."""
410 6
        if self.conn:
411 6
            return self.sendBuilderList()
412
        # else:
413 6
        return defer.succeed(None)
414

415 6
    @defer.inlineCallbacks
416 6
    def attached(self, conn):
417
        """This is called when the worker connects."""
418

419 6
        assert self.conn is None
420

421 6
        metrics.MetricCountEvent.log("AbstractWorker.attached_workers", 1)
422

423
        # now we go through a sequence of calls, gathering information, then
424
        # tell the Botmaster that it can finally give this worker to all the
425
        # Builders that care about it.
426

427
        # Reset graceful shutdown status
428 6
        self._graceful = False
429

430 6
        self.conn = conn
431 6
        self._old_builder_list = None  # clear builder list before proceed
432

433 6
        self._applyWorkerInfo(conn.info)
434 6
        self.worker_commands = conn.info.get("worker_commands", {})
435 6
        self.worker_environ = conn.info.get("environ", {})
436 6
        self.worker_basedir = conn.info.get("basedir", None)
437 6
        self.worker_system = conn.info.get("system", None)
438

439
        # The _detach_sub member is only ever used from tests.
440 6
        self._detached_sub = self.conn.notifyOnDisconnect(self.detached)
441

442 6
        workerinfo = {
443
            'admin': conn.info.get('admin'),
444
            'host': conn.info.get('host'),
445
            'access_uri': conn.info.get('access_uri'),
446
            'version': conn.info.get('version')
447
        }
448

449 6
        yield self.master.data.updates.workerConnected(
450
            workerid=self.workerid,
451
            masterid=self.master.masterid,
452
            workerinfo=workerinfo
453
        )
454

455 6
        if self.worker_system == "nt":
456 6
            self.path_module = namedModule("ntpath")
457
        else:
458
            # most everything accepts / as separator, so posix should be a
459
            # reasonable fallback
460 6
            self.path_module = namedModule("posixpath")
461 6
        log.msg("bot attached")
462 6
        self.messageReceivedFromWorker()
463 6
        self.stopMissingTimer()
464 6
        yield self.updateWorker()
465 6
        yield self.botmaster.maybeStartBuildsForWorker(self.name)
466 6
        self.updateState()
467

468 6
    def messageReceivedFromWorker(self):
469 6
        now = time.time()
470 6
        self.lastMessageReceived = now
471

472 6
    def setupProperties(self, props):
473 6
        for name in self.properties.properties:
474 6
            props.setProperty(
475
                name, self.properties.getProperty(name), "Worker")
476 6
        for name in self.defaultProperties.properties:
477 6
            if name not in props:
478 6
                props.setProperty(
479
                    name, self.defaultProperties.getProperty(name), "Worker")
480

481 6
    @defer.inlineCallbacks
482 6
    def _handle_conn_shutdown_notifier(self, conn):
483 6
        self._pending_conn_shutdown_notifier = Notifier()
484 6
        yield conn.waitShutdown()
485 6
        self._pending_conn_shutdown_notifier.notify(None)
486 6
        self._pending_conn_shutdown_notifier = None
487

488 6
    @defer.inlineCallbacks
489 6
    def detached(self):
490 6
        conn = self.conn
491 6
        self.conn = None
492 6
        self._handle_conn_shutdown_notifier(conn)
493

494
        # Note that _pending_conn_shutdown_notifier will not be fired until detached()
495
        # is complete.
496

497 6
        metrics.MetricCountEvent.log("AbstractWorker.attached_workers", -1)
498

499 6
        self._old_builder_list = []
500 6
        log.msg("Worker.detached({})".format(self.name))
501 6
        self.releaseLocks()
502 6
        yield self.master.data.updates.workerDisconnected(
503
            workerid=self.workerid,
504
            masterid=self.master.masterid,
505
        )
506

507 6
    def disconnect(self):
508
        """Forcibly disconnect the worker.
509

510
        This severs the TCP connection and returns a Deferred that will fire
511
        (with None) when the connection is probably gone.
512

513
        If the worker is still alive, they will probably try to reconnect
514
        again in a moment.
515

516
        This is called in two circumstances. The first is when a worker is
517
        removed from the config file. In this case, when they try to
518
        reconnect, they will be rejected as an unknown worker. The second is
519
        when we wind up with two connections for the same worker, in which
520
        case we disconnect the older connection.
521
        """
522 6
        if self.conn is None:
523 6
            return defer.succeed(None)
524 6
        log.msg("disconnecting old worker {} now".format(self.name))
525
        # When this Deferred fires, we'll be ready to accept the new worker
526 6
        return self._disconnect(self.conn)
527

528 6
    def waitForCompleteShutdown(self):
529
        # This function waits until the disconnection to happen and the disconnection
530
        # notifications have been delivered and acted upon.
531 6
        return self._waitForCompleteShutdownImpl(self.conn)
532

533 6
    @defer.inlineCallbacks
534 6
    def _waitForCompleteShutdownImpl(self, conn):
535 6
        if conn:
536 6
            yield conn.wait_shutdown_started()
537 6
            yield conn.waitShutdown()
538 6
        elif self._pending_conn_shutdown_notifier is not None:
539 6
            yield self._pending_conn_shutdown_notifier.wait()
540

541 6
    @defer.inlineCallbacks
542 6
    def _disconnect(self, conn):
543
        # This function waits until the disconnection to happen and the disconnection
544
        # notifications have been delivered and acted upon
545 6
        d = self._waitForCompleteShutdownImpl(conn)
546 6
        conn.loseConnection()
547 6
        log.msg("waiting for worker to finish disconnecting")
548 6
        yield d
549

550 6
    @defer.inlineCallbacks
551 6
    def sendBuilderList(self):
552 6
        our_builders = self.botmaster.getBuildersForWorker(self.name)
553

554 6
        blist = [(b.name, b.config.workerbuilddir) for b in our_builders]
555

556 6
        if blist == self._old_builder_list:
557 0
            return
558

559 6
        slist = yield self.conn.remoteSetBuilderList(builders=blist)
560

561 6
        self._old_builder_list = blist
562

563
        # Nothing has changed, so don't need to re-attach to everything
564 6
        if not slist:
565 6
            return
566

567 6
        dl = []
568 6
        for name in slist:
569
            # use get() since we might have changed our mind since then
570 6
            b = self.botmaster.builders.get(name)
571 6
            if b:
572 6
                d1 = self.attachBuilder(b)
573 6
                dl.append(d1)
574 6
        yield defer.DeferredList(dl)
575

576 6
    def attachBuilder(self, builder):
577 6
        return builder.attached(self, self.worker_commands)
578

579 6
    def controlWorker(self, key, params):
580 6
        log.msg("worker {} wants to {}: {}".format(self.name, key[-1], params))
581 6
        if key[-1] == "stop":
582 6
            return self.shutdownRequested()
583 6
        if key[-1] == "pause":
584 6
            self.pause()
585 6
        if key[-1] == "unpause":
586 6
            self.unpause()
587 6
        if key[-1] == "kill":
588 6
            self.shutdown()
589 6
        return None
590

591 6
    def shutdownRequested(self):
592 6
        self._graceful = True
593 6
        self.maybeShutdown()
594 6
        self.updateState()
595

596 6
    def addWorkerForBuilder(self, wfb):
597 6
        self.workerforbuilders[wfb.builder_name] = wfb
598

599 6
    def removeWorkerForBuilder(self, wfb):
600 6
        try:
601 6
            del self.workerforbuilders[wfb.builder_name]
602 0
        except KeyError:
603
            pass
604

605 6
    def buildFinished(self, wfb):
606
        """This is called when a build on this worker is finished."""
607 6
        self.botmaster.maybeStartBuildsForWorker(self.name)
608

609 6
    def canStartBuild(self):
610
        """
611
        I am called when a build is requested to see if this worker
612
        can start a build.  This function can be used to limit overall
613
        concurrency on the worker.
614

615
        Note for subclassers: if a worker can become willing to start a build
616
        without any action on that worker (for example, by a resource in use on
617
        another worker becoming available), then you must arrange for
618
        L{maybeStartBuildsForWorker} to be called at that time, or builds on
619
        this worker will not start.
620
        """
621

622
        # If we're waiting to shutdown gracefully, paused or quarantined then we shouldn't
623
        # accept any new jobs.
624 6
        if self._graceful or self._paused or self.quarantine_timer:
625 6
            return False
626

627 6
        if self.max_builds:
628 6
            active_builders = [wfb for wfb in self.workerforbuilders.values()
629
                               if wfb.isBusy()]
630 6
            if len(active_builders) >= self.max_builds:
631 6
                return False
632

633 6
        if not self.locksAvailable():
634 0
            return False
635

636 6
        return True
637

638 6
    @defer.inlineCallbacks
639 6
    def shutdown(self):
640
        """Shutdown the worker"""
641 6
        if not self.conn:
642 6
            log.msg("no remote; worker is already shut down")
643 6
            return
644

645 6
        yield self.conn.remoteShutdown()
646

647 6
    def maybeShutdown(self):
648
        """Shut down this worker if it has been asked to shut down gracefully,
649
        and has no active builders."""
650 6
        if not self._graceful:
651 6
            return
652 6
        active_builders = [wfb for wfb in self.workerforbuilders.values()
653
                           if wfb.isBusy()]
654 6
        if active_builders:
655 0
            return
656 6
        d = self.shutdown()
657 6
        d.addErrback(log.err, 'error while shutting down worker')
658

659 6
    def updateState(self):
660 6
        self.master.data.updates.setWorkerState(self.workerid, self._paused, self._graceful)
661

662 6
    def pause(self):
663
        """Stop running new builds on the worker."""
664 6
        self._paused = True
665 6
        self.updateState()
666

667 6
    def unpause(self):
668
        """Restart running new builds on the worker."""
669 6
        self._paused = False
670 6
        self.stopQuarantineTimer()
671 6
        self.botmaster.maybeStartBuildsForWorker(self.name)
672 6
        self.updateState()
673

674 6
    def isPaused(self):
675 0
        return self._paused
676

677 6
    def resetQuarantine(self):
678 6
        self.quarantine_timeout = self.quarantine_initial_timeout
679

680 6
    def putInQuarantine(self):
681 6
        if self.quarantine_timer:  # already in quarantine
682 6
            return
683

684 6
        self.quarantine_timer = self.master.reactor.callLater(
685
            self.quarantine_timeout, self.exitQuarantine)
686 6
        log.msg("{} has been put in quarantine for {}s".format(
687
            self.name, self.quarantine_timeout))
688
        # next we will wait twice as long
689 6
        self.quarantine_timeout *= 2
690 6
        if self.quarantine_timeout > self.quarantine_max_timeout:
691
            # unless we hit the max timeout
692 6
            self.quarantine_timeout = self.quarantine_max_timeout
693

694 6
    def exitQuarantine(self):
695 6
        log.msg("{} has left quarantine".format(self.name))
696 6
        self.quarantine_timer = None
697 6
        self.botmaster.maybeStartBuildsForWorker(self.name)
698

699 6
    def stopQuarantineTimer(self):
700 6
        if self.quarantine_timer is not None:
701 6
            self.quarantine_timer.cancel()
702 6
            self.exitQuarantine()
703

704

705 6
class Worker(AbstractWorker):
706

707 6
    @defer.inlineCallbacks
708 6
    def detached(self):
709 6
        yield super().detached()
710 6
        self.botmaster.workerLost(self)
711 6
        self.startMissingTimer()
712

713 6
    @defer.inlineCallbacks
714 6
    def attached(self, bot):
715 6
        try:
716 6
            yield super().attached(bot)
717 0
        except Exception as e:
718 0
            log.err(e, "worker {} cannot attach".format(self.name))
719 0
            return
720

721 6
    def buildFinished(self, wfb):
722
        """This is called when a build on this worker is finished."""
723 6
        super().buildFinished(wfb)
724

725
        # If we're gracefully shutting down, and we have no more active
726
        # builders, then it's safe to disconnect
727 6
        self.maybeShutdown()

Read our documentation on viewing source code .

Loading