scrapy / scrapy

@@ -1,7 +1,7 @@
Loading
1 1
import logging
2 2
import pprint
3 3
from collections import defaultdict, deque
4 -
from typing import Callable, Deque, Dict
4 +
from typing import Callable, Deque, Dict, Optional, cast, Iterable
5 5
6 6
from twisted.internet.defer import Deferred
7 7
@@ -9,7 +9,7 @@
Loading
9 9
from scrapy.exceptions import NotConfigured
10 10
from scrapy.settings import Settings
11 11
from scrapy.utils.misc import create_instance, load_object
12 -
from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
12 +
from scrapy.utils.defer import process_parallel, process_chain
13 13
14 14
logger = logging.getLogger(__name__)
15 15
@@ -21,7 +21,8 @@
Loading
21 21
22 22
    def __init__(self, *middlewares):
23 23
        self.middlewares = middlewares
24 -
        self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
24 +
        # Optional because process_spider_output and process_spider_exception can be None
25 +
        self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
25 26
        for mw in middlewares:
26 27
            self._add_middleware(mw)
27 28
@@ -64,14 +65,12 @@
Loading
64 65
            self.methods['close_spider'].appendleft(mw.close_spider)
65 66
66 67
    def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
67 -
        return process_parallel(self.methods[methodname], obj, *args)
68 +
        methods = cast(Iterable[Callable], self.methods[methodname])
69 +
        return process_parallel(methods, obj, *args)
68 70
69 71
    def _process_chain(self, methodname: str, obj, *args) -> Deferred:
70 -
        return process_chain(self.methods[methodname], obj, *args)
71 -
72 -
    def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
73 -
        return process_chain_both(self.methods[cb_methodname],
74 -
                                  self.methods[eb_methodname], obj, *args)
72 +
        methods = cast(Iterable[Callable], self.methods[methodname])
73 +
        return process_chain(methods, obj, *args)
75 74
76 75
    def open_spider(self, spider: Spider) -> Deferred:
77 76
        return self._process_parallel('open_spider', spider)

@@ -4,7 +4,7 @@
Loading
4 4
See documentation in docs/topics/spider-middleware.rst
5 5
"""
6 6
from itertools import islice
7 -
from typing import Any, Callable, Generator, Iterable, Union
7 +
from typing import Any, Callable, Generator, Iterable, Union, cast
8 8
9 9
from twisted.internet.defer import Deferred
10 10
from twisted.python.failure import Failure
@@ -47,6 +47,7 @@
Loading
47 47
    def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
48 48
                              spider: Spider) -> Any:
49 49
        for method in self.methods['process_spider_input']:
50 +
            method = cast(Callable, method)
50 51
            try:
51 52
                result = method(response=response, spider=spider)
52 53
                if result is not None:

@@ -3,7 +3,7 @@
Loading
3 3
4 4
See documentation in docs/topics/downloader-middleware.rst
5 5
"""
6 -
from typing import Callable, Union
6 +
from typing import Callable, Union, cast
7 7
8 8
from twisted.internet import defer
9 9
from twisted.python.failure import Failure
@@ -37,6 +37,7 @@
Loading
37 37
        @defer.inlineCallbacks
38 38
        def process_request(request: Request):
39 39
            for method in self.methods['process_request']:
40 +
                method = cast(Callable, method)
40 41
                response = yield deferred_from_coro(method(request=request, spider=spider))
41 42
                if response is not None and not isinstance(response, (Response, Request)):
42 43
                    raise _InvalidOutput(
@@ -55,6 +56,7 @@
Loading
55 56
                return response
56 57
57 58
            for method in self.methods['process_response']:
59 +
                method = cast(Callable, method)
58 60
                response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
59 61
                if not isinstance(response, (Response, Request)):
60 62
                    raise _InvalidOutput(
@@ -69,6 +71,7 @@
Loading
69 71
        def process_exception(failure: Failure):
70 72
            exception = failure.value
71 73
            for method in self.methods['process_exception']:
74 +
                method = cast(Callable, method)
72 75
                response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
73 76
                if response is not None and not isinstance(response, (Response, Request)):
74 77
                    raise _InvalidOutput(
Files Coverage
scrapy 88.57%
Project Totals (163 files) 88.57%
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading