scrapy / scrapy
1 7
from urllib.parse import urljoin
2

3 7
from twisted.web import server, resource, static, util
4

5

6 7
class SiteTest:
7

8 7
    def setUp(self):
9 7
        from twisted.internet import reactor
10 7
        super().setUp()
11 7
        self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
12 7
        self.baseurl = f"http://localhost:{self.site.getHost().port}/"
13

14 7
    def tearDown(self):
15 7
        super().tearDown()
16 7
        self.site.stopListening()
17

18 7
    def url(self, path):
19 7
        return urljoin(self.baseurl, path)
20

21

22 7
class NoMetaRefreshRedirect(util.Redirect):
23 7
    def render(self, request):
24 7
        content = util.Redirect.render(self, request)
25 7
        return content.replace(b'http-equiv=\"refresh\"',
26
            b'http-no-equiv=\"do-not-refresh-me\"')
27

28

29 7
def test_site():
30 7
    r = resource.Resource()
31 7
    r.putChild(b"text", static.Data(b"Works", "text/plain"))
32 7
    r.putChild(b"html", static.Data(b"<body><p class='one'>Works</p><p class='two'>World</p></body>", "text/html"))
33 7
    r.putChild(b"enc-gb18030", static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"))
34 7
    r.putChild(b"redirect", util.Redirect(b"/redirected"))
35 7
    r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))
36 7
    r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
37 7
    return server.Site(r)
38

39

40 7
if __name__ == '__main__':
41 0
    from twisted.internet import reactor
42 0
    port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
43 0
    print(f"http://localhost:{port.getHost().port}/")
44 0
    reactor.run()

Read our documentation on viewing source code .

Loading