buildbot / buildbot

@@ -0,0 +1,133 @@
Loading
1 +
# This file is part of Buildbot.  Buildbot is free software: you can
2 +
# redistribute it and/or modify it under the terms of the GNU General Public
3 +
# License as published by the Free Software Foundation, version 2.
4 +
#
5 +
# This program is distributed in the hope that it will be useful, but WITHOUT
6 +
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 +
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8 +
# details.
9 +
#
10 +
# You should have received a copy of the GNU General Public License along with
11 +
# this program; if not, write to the Free Software Foundation, Inc., 51
12 +
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 +
#
14 +
# Copyright Buildbot Team Members
15 +
16 +
#
17 +
# Parts of this code were copied from Twisted Python.
18 +
# Copyright (c) Twisted Matrix Laboratories.
19 +
#
20 +
21 +
from twisted.internet import defer
22 +
from twisted.internet import interfaces
23 +
from twisted.internet import protocol
24 +
from zope.interface import implementer
25 +
26 +
27 +
class HTTPTunnelClient(protocol.Protocol):
28 +
    """
29 +
    This protocol handles the HTTP communication with the proxy server
30 +
    and subsequent creation of the tunnel.
31 +
32 +
    Once the tunnel is established, all incoming communication is forwarded
33 +
    directly to the wrapped protocol.
34 +
    """
35 +
36 +
    def __init__(self, connectedDeferred):
37 +
        # this gets set once the tunnel is ready
38 +
        self._proxyWrappedProtocol = None
39 +
        self._connectedDeferred = connectedDeferred
40 +
41 +
    def connectionMade(self):
42 +
        request = "CONNECT {}:{} HTTP/1.1\r\n\r\n".format(
43 +
            self.factory.host, self.factory.port)
44 +
        self.transport.write(request.encode())
45 +
46 +
    def connectionLost(self, reason):
47 +
        if self._proxyWrappedProtocol:
48 +
            # Proxy connectionLost to the wrapped protocol
49 +
            self._proxyWrappedProtocol.connectionLost(reason)
50 +
51 +
    def dataReceived(self, data):
52 +
        if self._proxyWrappedProtocol is not None:
53 +
            # If tunnel is already established, proxy dataReceived()
54 +
            # calls to the wrapped protocol
55 +
            return self._proxyWrappedProtocol.dataReceived(data)
56 +
57 +
        # process data from the proxy server
58 +
        _, status, _ = data.split(b"\r\n")[0].split(b" ", 2)
59 +
        if status != b"200":
60 +
            return self.transport.loseConnection()
61 +
62 +
        self._proxyWrappedProtocol = (
63 +
            self.factory._proxyWrappedFactory.buildProtocol(
64 +
                self.transport.getPeer()))
65 +
        self._proxyWrappedProtocol.makeConnection(self.transport)
66 +
        self._connectedDeferred.callback(self._proxyWrappedProtocol)
67 +
68 +
        # forward all traffic directly to the wrapped protocol
69 +
        self.transport.protocol = self._proxyWrappedProtocol
70 +
71 +
        # In case the server sent some data together with its response,
72 +
        # forward those to the wrapped protocol.
73 +
        remaining_data = data.split(b"\r\n\r\n", 2)[1]
74 +
        if remaining_data:
75 +
            return self._proxyWrappedProtocol.dataReceived(remaining_data)
76 +
77 +
        return None
78 +
79 +
80 +
class HTTPTunnelFactory(protocol.ClientFactory):
81 +
    """The protocol factory for the HTTP tunnel.
82 +
83 +
    It is used as a wrapper for BotFactory, which can hence be shielded
84 +
    from all the proxy business.
85 +
    """
86 +
    protocol = HTTPTunnelClient
87 +
88 +
    def __init__(self, host, port, wrappedFactory):
89 +
        self.host = host
90 +
        self.port = port
91 +
92 +
        self._proxyWrappedFactory = wrappedFactory
93 +
        self._onConnection = defer.Deferred()
94 +
95 +
    def doStart(self):
96 +
        super().doStart()
97 +
        # forward start notifications through to the wrapped factory.
98 +
        self._proxyWrappedFactory.doStart()
99 +
100 +
    def doStop(self):
101 +
        # forward stop notifications through to the wrapped factory.
102 +
        self._proxyWrappedFactory.doStop()
103 +
        super().doStop()
104 +
105 +
    def buildProtocol(self, addr):
106 +
        proto = self.protocol(self._onConnection)
107 +
        proto.factory = self
108 +
        return proto
109 +
110 +
    def clientConnectionFailed(self, connector, reason):
111 +
        if not self._onConnection.called:
112 +
            self._onConnection.errback(reason)
113 +
114 +
115 +
@implementer(interfaces.IStreamClientEndpoint)
116 +
class HTTPTunnelEndpoint(object):
117 +
    """This handles the connection to buildbot master on given 'host'
118 +
    and 'port' through the proxy server given as 'proxyEndpoint'.
119 +
    """
120 +
121 +
    def __init__(self, host, port, proxyEndpoint):
122 +
        self.host = host
123 +
        self.port = port
124 +
        self.proxyEndpoint = proxyEndpoint
125 +
126 +
    def connect(self, protocolFactory):
127 +
        """Connect to remote server through an HTTP tunnel."""
128 +
        tunnel = HTTPTunnelFactory(self.host, self.port, protocolFactory)
129 +
        d = self.proxyEndpoint.connect(tunnel)
130 +
        # once tunnel connection is established,
131 +
        # defer the subsequent server connection
132 +
        d.addCallback(lambda result: tunnel._onConnection)
133 +
        return d

@@ -60,12 +60,14 @@
Loading
60 60
maxretries = %(maxretries)s
61 61
use_tls = %(use-tls)s
62 62
delete_leftover_dirs = %(delete-leftover-dirs)s
63 +
proxy_connection_string = %(proxy-connection-string)s
63 64
64 65
s = Worker(buildmaster_host, port, workername, passwd, basedir,
65 66
           keepalive, umask=umask, maxdelay=maxdelay,
66 67
           numcpus=numcpus, allow_shutdown=allow_shutdown,
67 68
           maxRetries=maxretries, useTls=use_tls,
68 -
           delete_leftover_dirs=delete_leftover_dirs)
69 +
           delete_leftover_dirs=delete_leftover_dirs,
70 +
           proxy_connection_string=proxy_connection_string)
69 71
s.setServiceParent(application)
70 72
"""]
71 73

@@ -36,6 +36,7 @@
Loading
36 36
from buildbot_worker.base import WorkerForBuilderBase
37 37
from buildbot_worker.compat import unicode2bytes
38 38
from buildbot_worker.pbutil import AutoLoginPBFactory
39 +
from buildbot_worker.tunnel import HTTPTunnelEndpoint
39 40
40 41
41 42
class UnknownCommand(pb.Error):
@@ -180,7 +181,7 @@
Loading
180 181
                 keepalive, usePTY=None, keepaliveTimeout=None, umask=None,
181 182
                 maxdelay=None, numcpus=None, unicode_encoding=None, useTls=None,
182 183
                 allow_shutdown=None, maxRetries=None, connection_string=None,
183 -
                 delete_leftover_dirs=False):
184 +
                 delete_leftover_dirs=False, proxy_connection_string=None):
184 185
185 186
        assert usePTY is None, "worker-side usePTY is not supported anymore"
186 187
        assert (connection_string is None or
@@ -212,17 +213,30 @@
Loading
212 213
        bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay)
213 214
        bf.startLogin(
214 215
            credentials.UsernamePassword(name, passwd), client=self.bot)
215 -
        if connection_string is None:
216 +
217 +
        def get_connection_string(host, port):
216 218
            if useTls:
217 219
                connection_type = 'tls'
218 220
            else:
219 221
                connection_type = 'tcp'
220 222
221 -
            connection_string = '{}:host={}:port={}'.format(
223 +
            return '{}:host={}:port={}'.format(
222 224
                connection_type,
223 -
                buildmaster_host.replace(':', r'\:'),  # escape ipv6 addresses
225 +
                host.replace(':', r'\:'),  # escape ipv6 addresses
224 226
                port)
225 -
        endpoint = clientFromString(reactor, connection_string)
227 +
228 +
        assert not (proxy_connection_string and connection_string), (
229 +
            "If you want to use HTTP tunneling, then supply build master "
230 +
            "host and port rather than a connection string")
231 +
232 +
        if proxy_connection_string:
233 +
            log.msg("Using HTTP tunnel to connect through proxy")
234 +
            proxy_endpoint = clientFromString(reactor, proxy_connection_string)
235 +
            endpoint = HTTPTunnelEndpoint(buildmaster_host, port, proxy_endpoint)
236 +
        else:
237 +
            if connection_string is None:
238 +
                connection_string = get_connection_string(buildmaster_host, port)
239 +
            endpoint = clientFromString(reactor, connection_string)
226 240
227 241
        def policy(attempt):
228 242
            if maxRetries and attempt >= maxRetries:

@@ -130,7 +130,9 @@
Loading
130 130
         "(None for unlimited)"],
131 131
        ["allow-shutdown", "a", None,
132 132
         "Allows the worker to initiate a graceful shutdown. One of "
133 -
         "'signal' or 'file'"]
133 +
         "'signal' or 'file'"],
134 +
        ["proxy-connection-string", None, None,
135 +
         "Address of HTTP proxy to tunnel through"]
134 136
    ]
135 137
136 138
    longdesc = textwrap.dedent("""
Files Coverage
master/buildbot 92.32%
worker/buildbot_worker 85.68%
Project Totals (333 files) 91.91%
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading