1
#!/usr/bin/env python3
2

3
# Contest Management System - http://cms-dev.github.io/
4
# Copyright © 2011-2015 Luca Wehrstedt <luca.wehrstedt@gmail.com>
5
#
6
# This program is free software: you can redistribute it and/or modify
7
# it under the terms of the GNU Affero General Public License as
8
# published by the Free Software Foundation, either version 3 of the
9
# License, or (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU Affero General Public License for more details.
15
#
16
# You should have received a copy of the GNU Affero General Public License
17
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
18

19 1
import json
20 1
import logging
21 1
import os
22 1
import re
23

24 1
from gevent.lock import RLock
25

26 1
from cmsranking.Entity import Entity, InvalidKey, InvalidData
27

28

29 1
logger = logging.getLogger(__name__)
30

31

32
# Global shared lock for all Store instances.
33 1
LOCK = RLock()
34

35

36 1
class Store:
37
    """A store for entities.
38

39
    Provide methods to perform the CRUD operations (create, retrieve,
40
    update, delete) on a set of entities, accessed by their unique key.
41
    It's very similar to a dict, except that keys are strings, values
42
    are of a single type (defined at init-time) and it's possible to
43
    get notified when something changes by providing appropriate
44
    callbacks.
45

46
    """
47 1
    def __init__(self, entity, path, all_stores, depends=None):
48
        """Initialize an empty EntityStore.
49

50
        The entity definition given as argument will define what kind
51
        of entities will be stored. It cannot be changed.
52

53
        entity (type): the class definition of the entities that will
54
            be stored
55

56
        """
57 1
        if not issubclass(entity, Entity):
58 0
            raise ValueError("The 'entity' parameter "
59
                             "isn't a subclass of Entity")
60 1
        self._entity = entity
61 1
        self._path = path
62 1
        self._all_stores = all_stores
63 1
        self._depends = depends if depends is not None else []
64 1
        self._store = dict()
65 1
        self._create_callbacks = list()
66 1
        self._update_callbacks = list()
67 1
        self._delete_callbacks = list()
68

69 1
    def load_from_disk(self):
70
        """Load the initial data for this store from the disk.
71

72
        """
73 1
        try:
74 1
            os.mkdir(self._path)
75 0
        except OSError:
76
            # it's ok: it means the directory already exists
77 0
            pass
78

79 1
        try:
80 1
            for name in os.listdir(self._path):
81
                # TODO check that the key is '[A-Za-z0-9_]+'
82 0
                if name[-5:] == '.json' and name[:-5] != '':
83 0
                    with open(os.path.join(self._path, name), 'rb') as rec:
84 0
                        item = self._entity()
85 0
                        item.set(json.load(rec))
86 0
                        item.key = name[:-5]
87 0
                        self._store[name[:-5]] = item
88 0
        except OSError:
89
            # the path isn't a directory or is inaccessible
90 0
            logger.error("Path is not a directory or is not accessible "
91
                         "(or other I/O error occurred)", exc_info=True)
92 0
        except ValueError:
93 0
            logger.error("Invalid JSON", exc_info=False,
94
                         extra={'location': os.path.join(self._path, name)})
95 0
        except InvalidData as exc:
96 0
            logger.error(str(exc), exc_info=False,
97
                         extra={'location': os.path.join(self._path, name)})
98

99 1
    def add_create_callback(self, callback):
100
        """Add a callback to be called when entities are created.
101

102
        Callbacks can be any kind of callable objects. They must accept
103
        a single argument: the key of the entity.
104

105
        """
106 1
        self._create_callbacks.append(callback)
107

108 1
    def add_update_callback(self, callback):
109
        """Add a callback to be called when entities are updated.
110

111
        Callbacks can be any kind of callable objects. They must accept
112
        a single argument: the key of the entity.
113

114
        """
115 1
        self._update_callbacks.append(callback)
116

117 1
    def add_delete_callback(self, callback):
118
        """Add a callback to be called when entities are deleted.
119

120
        Callbacks can be any kind of callable objects. They must accept
121
        a single argument: the key of the entity.
122

123
        """
124 1
        self._delete_callbacks.append(callback)
125

126 1
    def create(self, key, data):
127
        """Create a new entity.
128

129
        Create a new entity with the given key and the given data.
130

131
        key (unicode): the key with which the entity will be later
132
            accessed
133
        data (dict): the properties of the entity
134

135
        raise (InvalidKey): if key isn't a unicode or if an entity
136
            with the same key is already present in the store.
137
        raise (InvalidData): if data cannot be parsed, if it's missing
138
            some properties or if properties are of the wrong type.
139

140
        """
141 0
        if not isinstance(key, str) or key in self._store:
142 0
            raise InvalidKey("Key already in store.")
143

144
        # create entity
145 0
        with LOCK:
146 0
            item = self._entity()
147 0
            item.set(data)
148 0
            if not item.consistent(self._all_stores):
149 0
                raise InvalidData("Inconsistent data")
150 0
            item.key = key
151 0
            self._store[key] = item
152
            # notify callbacks
153 0
            for callback in self._create_callbacks:
154 0
                callback(key, item)
155
            # reflect changes on the persistent storage
156 0
            try:
157 0
                path = os.path.join(self._path, key + '.json')
158 0
                with open(path, 'wt', encoding="utf-8") as rec:
159 0
                    json.dump(self._store[key].get(), rec)
160 0
            except OSError:
161 0
                logger.error("I/O error occured while creating entity",
162
                             exc_info=True)
163

164 1
    def update(self, key, data):
165
        """Update an entity.
166

167
        Update an existing entity with the given key and the given
168
        data.
169

170
        key (unicode): the key of the entity that has to be updated
171
        data (dict): the new properties of the entity
172

173
        raise (InvalidKey): if key isn't a unicode or if no entity
174
            with that key is present in the store.
175
        raise (InvalidData): if data cannot be parsed, if it's missing
176
            some properties or if properties are of the wrong type.
177

178
        """
179 0
        if not isinstance(key, str) or key not in self._store:
180 0
            raise InvalidKey("Key not in store.")
181

182
        # update entity
183 0
        with LOCK:
184 0
            item = self._entity()
185 0
            item.set(data)
186 0
            if not item.consistent(self._all_stores):
187 0
                raise InvalidData("Inconsistent data")
188 0
            item.key = key
189 0
            old_item = self._store[key]
190 0
            self._store[key] = item
191
            # notify callbacks
192 0
            for callback in self._update_callbacks:
193 0
                callback(key, old_item, item)
194
            # reflect changes on the persistent storage
195 0
            try:
196 0
                path = os.path.join(self._path, key + '.json')
197 0
                with open(path, 'wt', encoding="utf-8") as rec:
198 0
                    json.dump(self._store[key].get(), rec)
199 0
            except OSError:
200 0
                logger.error("I/O error occured while updating entity",
201
                             exc_info=True)
202

203 1
    def merge_list(self, data_dict):
204
        """Merge a list of entities.
205

206
        Take a dictionary of entites and, for each of them:
207
         - if it's not present in the store, create it
208
         - if it's present, update it
209

210
        data_dict (dict): the dictionary of entities
211

212
        raise (InvalidData) if data cannot be parsed, if an entity is
213
            missing some properties or if properties are of the wrong
214
            type.
215

216
        """
217 1
        with LOCK:
218 1
            if not isinstance(data_dict, dict):
219 0
                raise InvalidData("Not a dictionary")
220 1
            item_dict = dict()
221 1
            for key, value in data_dict.items():
222 1
                try:
223
                    # FIXME We should allow keys to be arbitrary unicode
224
                    # strings, so this just needs to be a non-empty check.
225 1
                    if not re.match('[A-Za-z0-9_]+', key):
226 0
                        raise InvalidData("Invalid key")
227 1
                    item = self._entity()
228 1
                    item.set(value)
229 1
                    if not item.consistent(self._all_stores):
230 0
                        raise InvalidData("Inconsistent data")
231 1
                    item.key = key
232 1
                    item_dict[key] = item
233 0
                except InvalidData as exc:
234 0
                    raise InvalidData("[entity %s] %s" % (key, exc))
235

236 1
            for key, value in item_dict.items():
237 1
                is_new = key not in self._store
238 1
                old_value = self._store.get(key)
239
                # insert entity
240 1
                self._store[key] = value
241
                # notify callbacks
242 1
                if is_new:
243 1
                    for callback in self._create_callbacks:
244 1
                        callback(key, value)
245
                else:
246 0
                    for callback in self._update_callbacks:
247 0
                        callback(key, old_value, value)
248
                # reflect changes on the persistent storage
249 1
                try:
250 1
                    path = os.path.join(self._path, key + '.json')
251 1
                    with open(path, 'wt', encoding="utf-8") as rec:
252 1
                        json.dump(value.get(), rec)
253 0
                except OSError:
254 0
                    logger.error(
255
                        "I/O error occured while merging entity lists",
256
                        exc_info=True)
257

258 1
    def delete(self, key):
259
        """Delete an entity.
260

261
        Delete an existing entity from the store.
262

263
        key (unicode): the key of the entity that has to be deleted
264

265
        raise (InvalidKey): if key isn't a unicode or if no entity
266
            with that key is present in the store.
267

268
        """
269 0
        if not isinstance(key, str) or key not in self._store:
270 0
            raise InvalidKey("Key not in store.")
271

272 0
        with LOCK:
273
            # delete entity
274 0
            old_value = self._store[key]
275 0
            del self._store[key]
276
            # enforce consistency
277 0
            for depend in self._depends:
278 0
                for o_key, o_value in list(depend._store.items()):
279 0
                    if not o_value.consistent(self._all_stores):
280 0
                        depend.delete(o_key)
281
            # notify callbacks
282 0
            for callback in self._delete_callbacks:
283 0
                callback(key, old_value)
284
            # reflect changes on the persistent storage
285 0
            try:
286 0
                os.remove(os.path.join(self._path, key + '.json'))
287 0
            except OSError:
288 0
                logger.error("Unable to delete entity", exc_info=True)
289

290 1
    def delete_list(self):
291
        """Delete all entities.
292

293
        Delete all existing entities from the store.
294

295
        """
296 0
        with LOCK:
297
            # delete all entities
298 0
            for key in list(self._store.keys()):
299 0
                self.delete(key)
300

301 1
    def retrieve(self, key):
302
        """Retrieve an entity.
303

304
        Retrieve an existing entity from the store.
305

306
        key (unicode): the key of the entity that has to be retrieved
307

308
        raise (InvalidKey): if key isn't a unicode or if no entity
309
            with that key is present in the store.
310

311
        """
312 1
        if not isinstance(key, str) or key not in self._store:
313 0
            raise InvalidKey("Key not in store.")
314

315
        # retrieve entity
316 1
        return self._store[key].get()
317

318 1
    def retrieve_list(self):
319
        """Retrieve a list of all entities."""
320 0
        result = dict()
321 0
        for key, value in self._store.items():
322 0
            result[key] = value.get()
323 0
        return result
324

325 1
    def __contains__(self, key):
326 1
        return key in self._store

Read our documentation on viewing source code .

Loading