scrapy / scrapy
1 7
import sys
2 7
import os
3 7
from importlib import import_module
4

5 7
from scrapy.utils.spider import iter_spider_classes
6 7
from scrapy.exceptions import UsageError
7 7
from scrapy.commands import BaseRunSpiderCommand
8

9

10 7
def _import_file(filepath):
11 7
    abspath = os.path.abspath(filepath)
12 7
    dirname, file = os.path.split(abspath)
13 7
    fname, fext = os.path.splitext(file)
14 7
    if fext not in ('.py', '.pyw'):
15 7
        raise ValueError(f"Not a Python source file: {abspath}")
16 7
    if dirname:
17 7
        sys.path = [dirname] + sys.path
18 7
    try:
19 7
        module = import_module(fname)
20
    finally:
21 7
        if dirname:
22 7
            sys.path.pop(0)
23 7
    return module
24

25

26 7
class Command(BaseRunSpiderCommand):
27

28 7
    requires_project = False
29 7
    default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
30

31 7
    def syntax(self):
32 7
        return "[options] <spider_file>"
33

34 7
    def short_desc(self):
35 0
        return "Run a self-contained spider (without creating a project)"
36

37 7
    def long_desc(self):
38 7
        return "Run the spider defined in the given file"
39

40 7
    def run(self, args, opts):
41 7
        if len(args) != 1:
42 0
            raise UsageError()
43 7
        filename = args[0]
44 7
        if not os.path.exists(filename):
45 7
            raise UsageError(f"File not found: {filename}\n")
46 7
        try:
47 7
            module = _import_file(filename)
48 7
        except (ImportError, ValueError) as e:
49 7
            raise UsageError(f"Unable to load {filename!r}: {e}\n")
50 7
        spclasses = list(iter_spider_classes(module))
51 7
        if not spclasses:
52 7
            raise UsageError(f"No spider found in file: {filename}\n")
53 7
        spidercls = spclasses.pop()
54

55 7
        self.crawler_process.crawl(spidercls, **opts.spargs)
56 7
        self.crawler_process.start()
57

58 7
        if self.crawler_process.bootstrap_failed:
59 7
            self.exitcode = 1

Read our documentation on viewing source code .

Loading