Store.py at faba26f ⋅ cms-dev/cms

#!/usr/bin/env python3

# Contest Management System - http://cms-dev.github.io/
# Copyright © 2011-2015 Luca Wehrstedt <luca.wehrstedt@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json
import logging
import os
import re

from gevent.lock import RLock

from cmsranking.Entity import Entity, InvalidKey, InvalidData


logger = logging.getLogger(__name__)


# Global shared lock for all Store instances.
LOCK = RLock()


class Store:
    """A store for entities.

    Provide methods to perform the CRUD operations (create, retrieve,
    update, delete) on a set of entities, accessed by their unique key.
    It's very similar to a dict, except that keys are strings, values
    are of a single type (defined at init-time) and it's possible to
    get notified when something changes by providing appropriate
    callbacks.

    """
    def __init__(self, entity, path, all_stores, depends=None):
        """Initialize an empty EntityStore.

        The entity definition given as argument will define what kind
        of entities will be stored. It cannot be changed.

        entity (type): the class definition of the entities that will
            be stored

        """
        if not issubclass(entity, Entity):
            raise ValueError("The 'entity' parameter "
                             "isn't a subclass of Entity")
        self._entity = entity
        self._path = path
        self._all_stores = all_stores
        self._depends = depends if depends is not None else []
        self._store = dict()
        self._create_callbacks = list()
        self._update_callbacks = list()
        self._delete_callbacks = list()

    def load_from_disk(self):
        """Load the initial data for this store from the disk.

        """
        try:
            os.mkdir(self._path)
        except OSError:
            # it's ok: it means the directory already exists
            pass

        try:
            for name in os.listdir(self._path):
                # TODO check that the key is '[A-Za-z0-9_]+'
                if name[-5:] == '.json' and name[:-5] != '':
                    with open(os.path.join(self._path, name), 'rb') as rec:
                        item = self._entity()
                        item.set(json.load(rec))
                        item.key = name[:-5]
                        self._store[name[:-5]] = item
        except OSError:
            # the path isn't a directory or is inaccessible
            logger.error("Path is not a directory or is not accessible "
                         "(or other I/O error occurred)", exc_info=True)
        except ValueError:
            logger.error("Invalid JSON", exc_info=False,
                         extra={'location': os.path.join(self._path, name)})
        except InvalidData as exc:
            logger.error(str(exc), exc_info=False,
                         extra={'location': os.path.join(self._path, name)})

    def add_create_callback(self, callback):
        """Add a callback to be called when entities are created.

        Callbacks can be any kind of callable objects. They must accept
        a single argument: the key of the entity.

        """
        self._create_callbacks.append(callback)

    def add_update_callback(self, callback):
        """Add a callback to be called when entities are updated.

        Callbacks can be any kind of callable objects. They must accept
        a single argument: the key of the entity.

        """
        self._update_callbacks.append(callback)

    def add_delete_callback(self, callback):
        """Add a callback to be called when entities are deleted.

        Callbacks can be any kind of callable objects. They must accept
        a single argument: the key of the entity.

        """
        self._delete_callbacks.append(callback)

    def create(self, key, data):
        """Create a new entity.

        Create a new entity with the given key and the given data.

        key (unicode): the key with which the entity will be later
            accessed
        data (dict): the properties of the entity

        raise (InvalidKey): if key isn't a unicode or if an entity
            with the same key is already present in the store.
        raise (InvalidData): if data cannot be parsed, if it's missing
            some properties or if properties are of the wrong type.

        """
        if not isinstance(key, str) or key in self._store:
            raise InvalidKey("Key already in store.")

        # create entity
        with LOCK:
            item = self._entity()
            item.set(data)
            if not item.consistent(self._all_stores):
                raise InvalidData("Inconsistent data")
            item.key = key
            self._store[key] = item
            # notify callbacks
            for callback in self._create_callbacks:
                callback(key, item)
            # reflect changes on the persistent storage
            try:
                path = os.path.join(self._path, key + '.json')
                with open(path, 'wt', encoding="utf-8") as rec:
                    json.dump(self._store[key].get(), rec)
            except OSError:
                logger.error("I/O error occured while creating entity",
                             exc_info=True)

    def update(self, key, data):
        """Update an entity.

        Update an existing entity with the given key and the given
        data.

        key (unicode): the key of the entity that has to be updated
        data (dict): the new properties of the entity

        raise (InvalidKey): if key isn't a unicode or if no entity
            with that key is present in the store.
        raise (InvalidData): if data cannot be parsed, if it's missing
            some properties or if properties are of the wrong type.

        """
        if not isinstance(key, str) or key not in self._store:
            raise InvalidKey("Key not in store.")

        # update entity
        with LOCK:
            item = self._entity()
            item.set(data)
            if not item.consistent(self._all_stores):
                raise InvalidData("Inconsistent data")
            item.key = key
            old_item = self._store[key]
            self._store[key] = item
            # notify callbacks
            for callback in self._update_callbacks:
                callback(key, old_item, item)
            # reflect changes on the persistent storage
            try:
                path = os.path.join(self._path, key + '.json')
                with open(path, 'wt', encoding="utf-8") as rec:
                    json.dump(self._store[key].get(), rec)
            except OSError:
                logger.error("I/O error occured while updating entity",
                             exc_info=True)

    def merge_list(self, data_dict):
        """Merge a list of entities.

        Take a dictionary of entites and, for each of them:
         - if it's not present in the store, create it
         - if it's present, update it

        data_dict (dict): the dictionary of entities

        raise (InvalidData) if data cannot be parsed, if an entity is
            missing some properties or if properties are of the wrong
            type.

        """
        with LOCK:
            if not isinstance(data_dict, dict):
                raise InvalidData("Not a dictionary")
            item_dict = dict()
            for key, value in data_dict.items():
                try:
                    # FIXME We should allow keys to be arbitrary unicode
                    # strings, so this just needs to be a non-empty check.
                    if not re.match('[A-Za-z0-9_]+', key):
                        raise InvalidData("Invalid key")
                    item = self._entity()
                    item.set(value)
                    if not item.consistent(self._all_stores):
                        raise InvalidData("Inconsistent data")
                    item.key = key
                    item_dict[key] = item
                except InvalidData as exc:
                    raise InvalidData("[entity %s] %s" % (key, exc))

            for key, value in item_dict.items():
                is_new = key not in self._store
                old_value = self._store.get(key)
                # insert entity
                self._store[key] = value
                # notify callbacks
                if is_new:
                    for callback in self._create_callbacks:
                        callback(key, value)
                else:
                    for callback in self._update_callbacks:
                        callback(key, old_value, value)
                # reflect changes on the persistent storage
                try:
                    path = os.path.join(self._path, key + '.json')
                    with open(path, 'wt', encoding="utf-8") as rec:
                        json.dump(value.get(), rec)
                except OSError:
                    logger.error(
                        "I/O error occured while merging entity lists",
                        exc_info=True)

    def delete(self, key):
        """Delete an entity.

        Delete an existing entity from the store.

        key (unicode): the key of the entity that has to be deleted

        raise (InvalidKey): if key isn't a unicode or if no entity
            with that key is present in the store.

        """
        if not isinstance(key, str) or key not in self._store:
            raise InvalidKey("Key not in store.")

        with LOCK:
            # delete entity
            old_value = self._store[key]
            del self._store[key]
            # enforce consistency
            for depend in self._depends:
                for o_key, o_value in list(depend._store.items()):
                    if not o_value.consistent(self._all_stores):
                        depend.delete(o_key)
            # notify callbacks
            for callback in self._delete_callbacks:
                callback(key, old_value)
            # reflect changes on the persistent storage
            try:
                os.remove(os.path.join(self._path, key + '.json'))
            except OSError:
                logger.error("Unable to delete entity", exc_info=True)

    def delete_list(self):
        """Delete all entities.

        Delete all existing entities from the store.

        """
        with LOCK:
            # delete all entities
            for key in list(self._store.keys()):
                self.delete(key)

    def retrieve(self, key):
        """Retrieve an entity.

        Retrieve an existing entity from the store.

        key (unicode): the key of the entity that has to be retrieved

        raise (InvalidKey): if key isn't a unicode or if no entity
            with that key is present in the store.

        """
        if not isinstance(key, str) or key not in self._store:
            raise InvalidKey("Key not in store.")

        # retrieve entity
        return self._store[key].get()

    def retrieve_list(self):
        """Retrieve a list of all entities."""
        result = dict()
        for key, value in self._store.items():
            result[key] = value.get()
        return result

    def __contains__(self, key):
        return key in self._store

Read our documentation on viewing source code .

Navigation	Overlay
`t` Navigate files	`h` Toggle hits
`y` Change url to tip of branch	`m` Toggle misses
`b / v` Jump to prev/next hit line	`p` Toggle partial
`z / x` Jump to prev/next missed or partial line	`1..9` Toggle flags
`shift + o` Open current page in GitHub	`a` Toggle all on
`/ or ?` Show keyboard shortcuts dialog	`c` Toggle context lines or commits

1		#!/usr/bin/env python3
2
3		# Contest Management System - http://cms-dev.github.io/
4		# Copyright © 2011-2015 Luca Wehrstedt <luca.wehrstedt@gmail.com>
5		#
6		# This program is free software: you can redistribute it and/or modify
7		# it under the terms of the GNU Affero General Public License as
8		# published by the Free Software Foundation, either version 3 of the
9		# License, or (at your option) any later version.
10		#
11		# This program is distributed in the hope that it will be useful,
12		# but WITHOUT ANY WARRANTY; without even the implied warranty of
13		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14		# GNU Affero General Public License for more details.
15		#
16		# You should have received a copy of the GNU Affero General Public License
17		# along with this program. If not, see <http://www.gnu.org/licenses/>.
18
19	1	import json
20	1	import logging
21	1	import os
22	1	import re
23
24	1	from gevent.lock import RLock
25
26	1	from cmsranking.Entity import Entity, InvalidKey, InvalidData
27
28
29	1	logger = logging.getLogger(__name__)
30
31
32		# Global shared lock for all Store instances.
33	1	LOCK = RLock()
34
35
36	1	class Store:
37		"""A store for entities.
38
39		Provide methods to perform the CRUD operations (create, retrieve,
40		update, delete) on a set of entities, accessed by their unique key.
41		It's very similar to a dict, except that keys are strings, values
42		are of a single type (defined at init-time) and it's possible to
43		get notified when something changes by providing appropriate
44		callbacks.
45
46		"""
47	1	def __init__(self, entity, path, all_stores, depends=None):
48		"""Initialize an empty EntityStore.
49
50		The entity definition given as argument will define what kind
51		of entities will be stored. It cannot be changed.
52
53		entity (type): the class definition of the entities that will
54		be stored
55
56		"""
57	1	if not issubclass(entity, Entity):
58	0	raise ValueError("The 'entity' parameter "
59		"isn't a subclass of Entity")
60	1	self._entity = entity
61	1	self._path = path
62	1	self._all_stores = all_stores
63	1	self._depends = depends if depends is not None else []
64	1	self._store = dict()
65	1	self._create_callbacks = list()
66	1	self._update_callbacks = list()
67	1	self._delete_callbacks = list()
68
69	1	def load_from_disk(self):
70		"""Load the initial data for this store from the disk.
71
72		"""
73	1	try:
74	1	os.mkdir(self._path)
75	0	except OSError:
76		# it's ok: it means the directory already exists
77	0	pass
78
79	1	try:
80	1	for name in os.listdir(self._path):
81		# TODO check that the key is '[A-Za-z0-9_]+'
82	0	if name[-5:] == '.json' and name[:-5] != '':
83	0	with open(os.path.join(self._path, name), 'rb') as rec:
84	0	item = self._entity()
85	0	item.set(json.load(rec))
86	0	item.key = name[:-5]
87	0	self._store[name[:-5]] = item
88	0	except OSError:
89		# the path isn't a directory or is inaccessible
90	0	logger.error("Path is not a directory or is not accessible "
91		"(or other I/O error occurred)", exc_info=True)
92	0	except ValueError:
93	0	logger.error("Invalid JSON", exc_info=False,
94		extra={'location': os.path.join(self._path, name)})
95	0	except InvalidData as exc:
96	0	logger.error(str(exc), exc_info=False,
97		extra={'location': os.path.join(self._path, name)})
98
99	1	def add_create_callback(self, callback):
100		"""Add a callback to be called when entities are created.
101
102		Callbacks can be any kind of callable objects. They must accept
103		a single argument: the key of the entity.
104
105		"""
106	1	self._create_callbacks.append(callback)
107
108	1	def add_update_callback(self, callback):
109		"""Add a callback to be called when entities are updated.
110
111		Callbacks can be any kind of callable objects. They must accept
112		a single argument: the key of the entity.
113
114		"""
115	1	self._update_callbacks.append(callback)
116
117	1	def add_delete_callback(self, callback):
118		"""Add a callback to be called when entities are deleted.
119
120		Callbacks can be any kind of callable objects. They must accept
121		a single argument: the key of the entity.
122
123		"""
124	1	self._delete_callbacks.append(callback)
125
126	1	def create(self, key, data):
127		"""Create a new entity.
128
129		Create a new entity with the given key and the given data.
130
131		key (unicode): the key with which the entity will be later
132		accessed
133		data (dict): the properties of the entity
134
135		raise (InvalidKey): if key isn't a unicode or if an entity
136		with the same key is already present in the store.
137		raise (InvalidData): if data cannot be parsed, if it's missing
138		some properties or if properties are of the wrong type.
139
140		"""
141	0	if not isinstance(key, str) or key in self._store:
142	0	raise InvalidKey("Key already in store.")
143
144		# create entity
145	0	with LOCK:
146	0	item = self._entity()
147	0	item.set(data)
148	0	if not item.consistent(self._all_stores):
149	0	raise InvalidData("Inconsistent data")
150	0	item.key = key
151	0	self._store[key] = item
152		# notify callbacks
153	0	for callback in self._create_callbacks:
154	0	callback(key, item)
155		# reflect changes on the persistent storage
156	0	try:
157	0	path = os.path.join(self._path, key + '.json')
158	0	with open(path, 'wt', encoding="utf-8") as rec:
159	0	json.dump(self._store[key].get(), rec)
160	0	except OSError:
161	0	logger.error("I/O error occured while creating entity",
162		exc_info=True)
163
164	1	def update(self, key, data):
165		"""Update an entity.
166
167		Update an existing entity with the given key and the given
168		data.
169
170		key (unicode): the key of the entity that has to be updated
171		data (dict): the new properties of the entity
172
173		raise (InvalidKey): if key isn't a unicode or if no entity
174		with that key is present in the store.
175		raise (InvalidData): if data cannot be parsed, if it's missing
176		some properties or if properties are of the wrong type.
177
178		"""
179	0	if not isinstance(key, str) or key not in self._store:
180	0	raise InvalidKey("Key not in store.")
181
182		# update entity
183	0	with LOCK:
184	0	item = self._entity()
185	0	item.set(data)
186	0	if not item.consistent(self._all_stores):
187	0	raise InvalidData("Inconsistent data")
188	0	item.key = key
189	0	old_item = self._store[key]
190	0	self._store[key] = item
191		# notify callbacks
192	0	for callback in self._update_callbacks:
193	0	callback(key, old_item, item)
194		# reflect changes on the persistent storage
195	0	try:
196	0	path = os.path.join(self._path, key + '.json')
197	0	with open(path, 'wt', encoding="utf-8") as rec:
198	0	json.dump(self._store[key].get(), rec)
199	0	except OSError:
200	0	logger.error("I/O error occured while updating entity",
201		exc_info=True)
202
203	1	def merge_list(self, data_dict):
204		"""Merge a list of entities.
205
206		Take a dictionary of entites and, for each of them:
207		- if it's not present in the store, create it
208		- if it's present, update it
209
210		data_dict (dict): the dictionary of entities
211
212		raise (InvalidData) if data cannot be parsed, if an entity is
213		missing some properties or if properties are of the wrong
214		type.
215
216		"""
217	1	with LOCK:
218	1	if not isinstance(data_dict, dict):
219	0	raise InvalidData("Not a dictionary")
220	1	item_dict = dict()
221	1	for key, value in data_dict.items():
222	1	try:
223		# FIXME We should allow keys to be arbitrary unicode
224		# strings, so this just needs to be a non-empty check.
225	1	if not re.match('[A-Za-z0-9_]+', key):
226	0	raise InvalidData("Invalid key")
227	1	item = self._entity()
228	1	item.set(value)
229	1	if not item.consistent(self._all_stores):
230	0	raise InvalidData("Inconsistent data")
231	1	item.key = key
232	1	item_dict[key] = item
233	0	except InvalidData as exc:
234	0	raise InvalidData("[entity %s] %s" % (key, exc))
235
236	1	for key, value in item_dict.items():
237	1	is_new = key not in self._store
238	1	old_value = self._store.get(key)
239		# insert entity
240	1	self._store[key] = value
241		# notify callbacks
242	1	if is_new:
243	1	for callback in self._create_callbacks:
244	1	callback(key, value)
245		else:
246	0	for callback in self._update_callbacks:
247	0	callback(key, old_value, value)
248		# reflect changes on the persistent storage
249	1	try:
250	1	path = os.path.join(self._path, key + '.json')
251	1	with open(path, 'wt', encoding="utf-8") as rec:
252	1	json.dump(value.get(), rec)
253	0	except OSError:
254	0	logger.error(
255		"I/O error occured while merging entity lists",
256		exc_info=True)
257
258	1	def delete(self, key):
259		"""Delete an entity.
260
261		Delete an existing entity from the store.
262
263		key (unicode): the key of the entity that has to be deleted
264
265		raise (InvalidKey): if key isn't a unicode or if no entity
266		with that key is present in the store.
267
268		"""
269	0	if not isinstance(key, str) or key not in self._store:
270	0	raise InvalidKey("Key not in store.")
271
272	0	with LOCK:
273		# delete entity
274	0	old_value = self._store[key]
275	0	del self._store[key]
276		# enforce consistency
277	0	for depend in self._depends:
278	0	for o_key, o_value in list(depend._store.items()):
279	0	if not o_value.consistent(self._all_stores):
280	0	depend.delete(o_key)
281		# notify callbacks
282	0	for callback in self._delete_callbacks:
283	0	callback(key, old_value)
284		# reflect changes on the persistent storage
285	0	try:
286	0	os.remove(os.path.join(self._path, key + '.json'))
287	0	except OSError:
288	0	logger.error("Unable to delete entity", exc_info=True)
289
290	1	def delete_list(self):
291		"""Delete all entities.
292
293		Delete all existing entities from the store.
294
295		"""
296	0	with LOCK:
297		# delete all entities
298	0	for key in list(self._store.keys()):
299	0	self.delete(key)
300
301	1	def retrieve(self, key):
302		"""Retrieve an entity.
303
304		Retrieve an existing entity from the store.
305
306		key (unicode): the key of the entity that has to be retrieved
307
308		raise (InvalidKey): if key isn't a unicode or if no entity
309		with that key is present in the store.
310
311		"""
312	1	if not isinstance(key, str) or key not in self._store:
313	0	raise InvalidKey("Key not in store.")
314
315		# retrieve entity
316	1	return self._store[key].get()
317
318	1	def retrieve_list(self):
319		"""Retrieve a list of all entities."""
320	0	result = dict()
321	0	for key, value in self._store.items():
322	0	result[key] = value.get()
323	0	return result
324
325	1	def __contains__(self, key):
326	1	return key in self._store