scrapy / scrapy
1 7
import re
2 7
import os
3 7
import string
4 7
from importlib import import_module
5 7
from os.path import join, exists, abspath
6 7
from shutil import ignore_patterns, move, copy2, copystat
7 7
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
8

9 7
import scrapy
10 7
from scrapy.commands import ScrapyCommand
11 7
from scrapy.utils.template import render_templatefile, string_camelcase
12 7
from scrapy.exceptions import UsageError
13

14

15 7
TEMPLATES_TO_RENDER = (
16
    ('scrapy.cfg',),
17
    ('${project_name}', 'settings.py.tmpl'),
18
    ('${project_name}', 'items.py.tmpl'),
19
    ('${project_name}', 'pipelines.py.tmpl'),
20
    ('${project_name}', 'middlewares.py.tmpl'),
21
)
22

23 7
IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
24

25

26 7
def _make_writable(path):
27 7
    current_permissions = os.stat(path).st_mode
28 7
    os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
29

30

31 7
class Command(ScrapyCommand):
32

33 7
    requires_project = False
34 7
    default_settings = {'LOG_ENABLED': False,
35
                        'SPIDER_LOADER_WARN_ONLY': True}
36

37 7
    def syntax(self):
38 7
        return "<project_name> [project_dir]"
39

40 7
    def short_desc(self):
41 7
        return "Create new project"
42

43 7
    def _is_valid_name(self, project_name):
44 7
        def _module_exists(module_name):
45 7
            try:
46 7
                import_module(module_name)
47 7
                return True
48 7
            except ImportError:
49 7
                return False
50

51 7
        if not re.search(r'^[_a-zA-Z]\w*$', project_name):
52 7
            print('Error: Project names must begin with a letter and contain'
53
                  ' only\nletters, numbers and underscores')
54 7
        elif _module_exists(project_name):
55 7
            print(f'Error: Module {project_name!r} already exists')
56
        else:
57 7
            return True
58 7
        return False
59

60 7
    def _copytree(self, src, dst):
61
        """
62
        Since the original function always creates the directory, to resolve
63
        the issue a new function had to be created. It's a simple copy and
64
        was reduced for this case.
65

66
        More info at:
67
        https://github.com/scrapy/scrapy/pull/2005
68
        """
69 7
        ignore = IGNORE
70 7
        names = os.listdir(src)
71 7
        ignored_names = ignore(src, names)
72

73 7
        if not os.path.exists(dst):
74 7
            os.makedirs(dst)
75

76 7
        for name in names:
77 7
            if name in ignored_names:
78 7
                continue
79

80 7
            srcname = os.path.join(src, name)
81 7
            dstname = os.path.join(dst, name)
82 7
            if os.path.isdir(srcname):
83 7
                self._copytree(srcname, dstname)
84
            else:
85 7
                copy2(srcname, dstname)
86 7
                _make_writable(dstname)
87

88 7
        copystat(src, dst)
89 7
        _make_writable(dst)
90

91 7
    def run(self, args, opts):
92 7
        if len(args) not in (1, 2):
93 7
            raise UsageError()
94

95 7
        project_name = args[0]
96 7
        project_dir = args[0]
97

98 7
        if len(args) == 2:
99 7
            project_dir = args[1]
100

101 7
        if exists(join(project_dir, 'scrapy.cfg')):
102 7
            self.exitcode = 1
103 7
            print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
104 7
            return
105

106 7
        if not self._is_valid_name(project_name):
107 7
            self.exitcode = 1
108 7
            return
109

110 7
        self._copytree(self.templates_dir, abspath(project_dir))
111 7
        move(join(project_dir, 'module'), join(project_dir, project_name))
112 7
        for paths in TEMPLATES_TO_RENDER:
113 7
            path = join(*paths)
114 7
            tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
115 7
            render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
116 7
        print(f"New Scrapy project '{project_name}', using template directory "
117
              f"'{self.templates_dir}', created in:")
118 7
        print(f"    {abspath(project_dir)}\n")
119 7
        print("You can start your first spider with:")
120 7
        print(f"    cd {project_dir}")
121 7
        print("    scrapy genspider example example.com")
122

123 7
    @property
124 4
    def templates_dir(self):
125 7
        return join(
126
            self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
127
            'project'
128
        )

Read our documentation on viewing source code .

Loading