|
1
|
|
# Python
|
|
2
|
3
|
import re
|
|
3
|
3
|
import json
|
|
4
|
3
|
import logging
|
|
5
|
|
|
|
6
|
|
# Django
|
|
7
|
3
|
from django.core.exceptions import FieldError, ValidationError
|
|
8
|
3
|
from django.db import models
|
|
9
|
3
|
from django.db.models import Q
|
|
10
|
3
|
from django.db.models.fields import FieldDoesNotExist
|
|
11
|
3
|
from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey
|
|
12
|
3
|
from django.contrib.contenttypes.models import ContentType
|
|
13
|
3
|
from django.contrib.contenttypes.fields import GenericForeignKey
|
|
14
|
3
|
from django.utils.encoding import force_text
|
|
15
|
3
|
from django.utils.translation import ugettext_lazy as _
|
|
16
|
|
|
|
17
|
|
# Django REST Framework
|
|
18
|
3
|
from rest_framework.exceptions import ParseError, PermissionDenied
|
|
19
|
3
|
from rest_framework.filters import BaseFilterBackend
|
|
20
|
|
|
|
21
|
|
# cyborgbackup
|
|
22
|
3
|
from cyborgbackup.main.utils.common import get_type_for_model, to_python_boolean, get_all_field_names
|
|
23
|
|
|
|
24
|
3
|
logger = logging.getLogger('cyborgbackup.main.tasks')
|
|
25
|
|
|
|
26
|
|
|
|
27
|
3
|
class V1CredentialFilterBackend(BaseFilterBackend):
|
|
28
|
|
'''
|
|
29
|
|
For /api/v1/ requests, filter out v2 (custom) credentials
|
|
30
|
|
'''
|
|
31
|
|
|
|
32
|
3
|
def filter_queryset(self, request, queryset, view):
|
|
33
|
0
|
from cyborgbackup.api.versioning import get_request_version
|
|
34
|
0
|
if get_request_version(request) == 1:
|
|
35
|
0
|
queryset = queryset.filter(credential_type__managed_by_cyborgbackup=True)
|
|
36
|
0
|
return queryset
|
|
37
|
|
|
|
38
|
|
|
|
39
|
3
|
class TypeFilterBackend(BaseFilterBackend):
|
|
40
|
|
'''
|
|
41
|
|
Filter on type field now returned with all objects.
|
|
42
|
|
'''
|
|
43
|
|
|
|
44
|
3
|
def filter_queryset(self, request, queryset, view):
|
|
45
|
3
|
try:
|
|
46
|
3
|
types = None
|
|
47
|
3
|
for key, value in request.query_params.items():
|
|
48
|
0
|
if key == 'type':
|
|
49
|
0
|
if ',' in value:
|
|
50
|
0
|
types = value.split(',')
|
|
51
|
|
else:
|
|
52
|
0
|
types = (value,)
|
|
53
|
3
|
if types:
|
|
54
|
0
|
types_map = {}
|
|
55
|
0
|
for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')):
|
|
56
|
0
|
ct_model = ct.model_class()
|
|
57
|
0
|
if not ct_model:
|
|
58
|
0
|
continue
|
|
59
|
0
|
ct_type = get_type_for_model(ct_model)
|
|
60
|
0
|
types_map[ct_type] = ct.pk
|
|
61
|
0
|
model = queryset.model
|
|
62
|
0
|
model_type = get_type_for_model(model)
|
|
63
|
0
|
if 'polymorphic_ctype' in get_all_field_names(model):
|
|
64
|
0
|
types_pks = set([v for k, v in types_map.items() if k in types])
|
|
65
|
0
|
queryset = queryset.filter(polymorphic_ctype_id__in=types_pks)
|
|
66
|
0
|
elif model_type in types:
|
|
67
|
0
|
queryset = queryset
|
|
68
|
|
else:
|
|
69
|
0
|
queryset = queryset.none()
|
|
70
|
3
|
return queryset
|
|
71
|
0
|
except FieldError as e:
|
|
72
|
|
# Return a 400 for invalid field names.
|
|
73
|
0
|
raise ParseError(*e.args)
|
|
74
|
|
|
|
75
|
|
|
|
76
|
3
|
class FieldLookupBackend(BaseFilterBackend):
|
|
77
|
|
'''
|
|
78
|
|
Filter using field lookups provided via query string parameters.
|
|
79
|
|
'''
|
|
80
|
|
|
|
81
|
3
|
RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by',
|
|
82
|
|
'search', 'type', 'host_filter', 'fields')
|
|
83
|
|
|
|
84
|
3
|
SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains',
|
|
85
|
|
'startswith', 'istartswith', 'endswith', 'iendswith',
|
|
86
|
|
'regex', 'iregex', 'gt', 'gte', 'lt', 'lte', 'in',
|
|
87
|
|
'isnull', 'search')
|
|
88
|
|
|
|
89
|
3
|
def get_field_from_lookup(self, model, lookup):
|
|
90
|
0
|
field = None
|
|
91
|
0
|
parts = lookup.split('__')
|
|
92
|
0
|
if parts and parts[-1] not in self.SUPPORTED_LOOKUPS:
|
|
93
|
0
|
parts.append('exact')
|
|
94
|
|
# FIXME: Could build up a list of models used across relationships, use
|
|
95
|
|
# those lookups combined with request.user.get_queryset(Model) to make
|
|
96
|
|
# sure user cannot query using objects he could not view.
|
|
97
|
0
|
new_parts = []
|
|
98
|
|
|
|
99
|
|
# Store of all the fields used to detect repeats
|
|
100
|
0
|
field_set = set([])
|
|
101
|
|
|
|
102
|
0
|
for name in parts[:-1]:
|
|
103
|
0
|
new_parts.append(name)
|
|
104
|
|
|
|
105
|
0
|
if name in getattr(model, 'PASSWORD_FIELDS', ()):
|
|
106
|
0
|
raise PermissionDenied(_('Filtering on password fields is not allowed.'))
|
|
107
|
0
|
elif name == 'pk':
|
|
108
|
0
|
field = model._meta.pk
|
|
109
|
|
else:
|
|
110
|
0
|
name_alt = name.replace("_", "")
|
|
111
|
0
|
if name_alt in model._meta.fields_map.keys():
|
|
112
|
0
|
field = model._meta.fields_map[name_alt]
|
|
113
|
0
|
new_parts.pop()
|
|
114
|
0
|
new_parts.append(name_alt)
|
|
115
|
|
else:
|
|
116
|
0
|
field = model._meta.get_field(name)
|
|
117
|
0
|
if 'auth' in name or 'token' in name:
|
|
118
|
0
|
raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
|
|
119
|
0
|
if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
|
|
120
|
0
|
raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
|
|
121
|
0
|
elif getattr(field, '__prevent_search__', False):
|
|
122
|
0
|
raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
|
|
123
|
0
|
if field in field_set:
|
|
124
|
|
# Field traversed twice, could create infinite JOINs, DoSing
|
|
125
|
0
|
raise ParseError(_('Loops not allowed in filters, detected on field {}.').format(field.name))
|
|
126
|
0
|
field_set.add(field)
|
|
127
|
0
|
model = getattr(field, 'related_model', None) or field.model
|
|
128
|
|
|
|
129
|
0
|
if parts:
|
|
130
|
0
|
new_parts.append(parts[-1])
|
|
131
|
0
|
new_lookup = '__'.join(new_parts)
|
|
132
|
0
|
return field, new_lookup
|
|
133
|
|
|
|
134
|
3
|
def to_python_related(self, value):
|
|
135
|
0
|
value = force_text(value)
|
|
136
|
0
|
if value.lower() in ('none', 'null'):
|
|
137
|
0
|
return None
|
|
138
|
|
else:
|
|
139
|
0
|
return int(value)
|
|
140
|
|
|
|
141
|
3
|
def value_to_python_for_field(self, field, value):
|
|
142
|
0
|
if isinstance(field, models.NullBooleanField):
|
|
143
|
0
|
return to_python_boolean(value, allow_none=True)
|
|
144
|
0
|
elif isinstance(field, models.BooleanField):
|
|
145
|
0
|
return to_python_boolean(value)
|
|
146
|
0
|
elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)):
|
|
147
|
0
|
try:
|
|
148
|
0
|
return self.to_python_related(value)
|
|
149
|
0
|
except ValueError:
|
|
150
|
0
|
raise ParseError(_('Invalid {field_name} id: {field_id}').format(
|
|
151
|
|
field_name=getattr(field, 'name', 'related field'),
|
|
152
|
|
field_id=value)
|
|
153
|
|
)
|
|
154
|
|
else:
|
|
155
|
0
|
return field.to_python(value)
|
|
156
|
|
|
|
157
|
3
|
def value_to_python(self, model, lookup, value):
|
|
158
|
0
|
try:
|
|
159
|
0
|
lookup = lookup.encode("ascii").decode('ascii')
|
|
160
|
0
|
except UnicodeEncodeError:
|
|
161
|
0
|
raise ValueError("%r is not an allowed field name. Must be ascii encodable." % lookup)
|
|
162
|
|
|
|
163
|
0
|
field, new_lookup = self.get_field_from_lookup(model, lookup)
|
|
164
|
|
|
|
165
|
0
|
if new_lookup.endswith('__isnull'):
|
|
166
|
0
|
value = to_python_boolean(value)
|
|
167
|
0
|
elif new_lookup.endswith('__in'):
|
|
168
|
0
|
items = []
|
|
169
|
0
|
if not value:
|
|
170
|
0
|
raise ValueError('cannot provide empty value for __in')
|
|
171
|
0
|
for item in value.split(','):
|
|
172
|
0
|
items.append(self.value_to_python_for_field(field, item))
|
|
173
|
0
|
value = items
|
|
174
|
0
|
elif new_lookup.endswith('__regex') or new_lookup.endswith('__iregex'):
|
|
175
|
0
|
try:
|
|
176
|
0
|
re.compile(value)
|
|
177
|
0
|
except re.error as e:
|
|
178
|
0
|
raise ValueError(e.args[0])
|
|
179
|
0
|
elif new_lookup.endswith('__search'):
|
|
180
|
0
|
related_model = getattr(field, 'related_model', None)
|
|
181
|
0
|
if not related_model:
|
|
182
|
0
|
raise ValueError('%s is not searchable' % new_lookup[:-8])
|
|
183
|
0
|
new_lookups = []
|
|
184
|
0
|
for rm_field in related_model._meta.fields:
|
|
185
|
0
|
if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description'):
|
|
186
|
0
|
new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
|
|
187
|
0
|
return value, new_lookups
|
|
188
|
|
else:
|
|
189
|
0
|
value = self.value_to_python_for_field(field, value)
|
|
190
|
0
|
return value, new_lookup
|
|
191
|
|
|
|
192
|
3
|
def filter_queryset(self, request, queryset, view):
|
|
193
|
3
|
try:
|
|
194
|
|
# Apply filters specified via query_params. Each entry in the lists
|
|
195
|
|
# below is (negate, field, value).
|
|
196
|
3
|
and_filters = []
|
|
197
|
3
|
or_filters = []
|
|
198
|
3
|
chain_filters = []
|
|
199
|
3
|
role_filters = []
|
|
200
|
3
|
search_filters = []
|
|
201
|
3
|
for key, values in request.query_params.lists():
|
|
202
|
0
|
if key in self.RESERVED_NAMES:
|
|
203
|
0
|
continue
|
|
204
|
|
|
|
205
|
|
# HACK: Make job event filtering by host name mostly work even
|
|
206
|
|
# when not capturing job event hosts M2M.
|
|
207
|
0
|
if queryset.model._meta.object_name == 'JobEvent' and key.startswith('hosts__name'):
|
|
208
|
0
|
key = key.replace('hosts__name', 'or__host__name')
|
|
209
|
0
|
or_filters.append((False, 'host__name__isnull', True))
|
|
210
|
|
|
|
211
|
|
# Custom __int filter suffix (internal use only).
|
|
212
|
0
|
q_int = False
|
|
213
|
0
|
if key.endswith('__int'):
|
|
214
|
0
|
key = key[:-5]
|
|
215
|
0
|
q_int = True
|
|
216
|
|
|
|
217
|
|
# RBAC filtering
|
|
218
|
0
|
if key == 'role_level':
|
|
219
|
0
|
role_filters.append(values[0])
|
|
220
|
0
|
continue
|
|
221
|
|
|
|
222
|
|
# Search across related objects.
|
|
223
|
0
|
if key.endswith('__search'):
|
|
224
|
0
|
for value in values:
|
|
225
|
0
|
search_value, new_keys = self.value_to_python(queryset.model, key, force_text(value))
|
|
226
|
0
|
assert isinstance(new_keys, list)
|
|
227
|
0
|
for new_key in new_keys:
|
|
228
|
0
|
search_filters.append((new_key, search_value))
|
|
229
|
0
|
continue
|
|
230
|
|
|
|
231
|
|
# Custom chain__ and or__ filters, mutually exclusive (both can
|
|
232
|
|
# precede not__).
|
|
233
|
0
|
q_chain = False
|
|
234
|
0
|
q_or = False
|
|
235
|
0
|
if key.startswith('chain__'):
|
|
236
|
0
|
key = key[7:]
|
|
237
|
0
|
q_chain = True
|
|
238
|
0
|
elif key.startswith('or__'):
|
|
239
|
0
|
key = key[4:]
|
|
240
|
0
|
q_or = True
|
|
241
|
|
|
|
242
|
|
# Custom not__ filter prefix.
|
|
243
|
0
|
q_not = False
|
|
244
|
0
|
if key.startswith('not__'):
|
|
245
|
0
|
key = key[5:]
|
|
246
|
0
|
q_not = True
|
|
247
|
|
|
|
248
|
|
# Convert value(s) to python and add to the appropriate list.
|
|
249
|
0
|
for value in values:
|
|
250
|
0
|
if q_int:
|
|
251
|
0
|
value = int(value)
|
|
252
|
0
|
value, new_key = self.value_to_python(queryset.model, key, value)
|
|
253
|
0
|
if q_chain:
|
|
254
|
0
|
chain_filters.append((q_not, new_key, value))
|
|
255
|
0
|
elif q_or:
|
|
256
|
0
|
or_filters.append((q_not, new_key, value))
|
|
257
|
|
else:
|
|
258
|
0
|
and_filters.append((q_not, new_key, value))
|
|
259
|
|
|
|
260
|
|
# Now build Q objects for database query filter.
|
|
261
|
3
|
if and_filters or or_filters or chain_filters or search_filters:
|
|
262
|
0
|
args = []
|
|
263
|
0
|
for n, k, v in and_filters:
|
|
264
|
0
|
if n:
|
|
265
|
0
|
args.append(~Q(**{k: v}))
|
|
266
|
|
else:
|
|
267
|
0
|
args.append(Q(**{k: v}))
|
|
268
|
0
|
if or_filters:
|
|
269
|
0
|
q = Q()
|
|
270
|
0
|
for n, k, v in or_filters:
|
|
271
|
0
|
if n:
|
|
272
|
0
|
q |= ~Q(**{k: v})
|
|
273
|
|
else:
|
|
274
|
0
|
q |= Q(**{k: v})
|
|
275
|
0
|
args.append(q)
|
|
276
|
0
|
if search_filters:
|
|
277
|
0
|
q = Q()
|
|
278
|
0
|
for k, v in search_filters:
|
|
279
|
0
|
q |= Q(**{k: v})
|
|
280
|
0
|
args.append(q)
|
|
281
|
0
|
for n, k, v in chain_filters:
|
|
282
|
0
|
if n:
|
|
283
|
0
|
q = ~Q(**{k: v})
|
|
284
|
|
else:
|
|
285
|
0
|
q = Q(**{k: v})
|
|
286
|
0
|
queryset = queryset.filter(q)
|
|
287
|
0
|
queryset = queryset.filter(*args).distinct()
|
|
288
|
3
|
return queryset
|
|
289
|
0
|
except (FieldError, FieldDoesNotExist, ValueError, TypeError) as e:
|
|
290
|
0
|
raise ParseError(e.args[0])
|
|
291
|
0
|
except ValidationError as e:
|
|
292
|
0
|
raise ParseError(json.dumps(e.messages, ensure_ascii=False))
|
|
293
|
|
|
|
294
|
|
|
|
295
|
3
|
class OrderByBackend(BaseFilterBackend):
|
|
296
|
|
'''
|
|
297
|
|
Filter to apply ordering based on query string parameters.
|
|
298
|
|
'''
|
|
299
|
|
|
|
300
|
3
|
def filter_queryset(self, request, queryset, view):
|
|
301
|
3
|
try:
|
|
302
|
3
|
order_by = None
|
|
303
|
3
|
for key, value in request.query_params.items():
|
|
304
|
0
|
if key in ('order', 'order_by'):
|
|
305
|
0
|
order_by = value
|
|
306
|
0
|
if ',' in value:
|
|
307
|
0
|
order_by = value.split(',')
|
|
308
|
|
else:
|
|
309
|
0
|
order_by = (value,)
|
|
310
|
3
|
if order_by:
|
|
311
|
0
|
order_by = self._strip_sensitive_model_fields(queryset.model, order_by)
|
|
312
|
|
|
|
313
|
|
# Special handling of the type field for ordering. In this
|
|
314
|
|
# case, we're not sorting exactly on the type field, but
|
|
315
|
|
# given the limited number of views with multiple types,
|
|
316
|
|
# sorting on polymorphic_ctype.model is effectively the same.
|
|
317
|
0
|
new_order_by = []
|
|
318
|
0
|
for field in order_by:
|
|
319
|
0
|
if field not in ('type', '-type'):
|
|
320
|
0
|
new_order_by.append(field)
|
|
321
|
0
|
queryset = queryset.order_by(*new_order_by)
|
|
322
|
3
|
return queryset
|
|
323
|
0
|
except FieldError as e:
|
|
324
|
|
# Return a 400 for invalid field names.
|
|
325
|
0
|
raise ParseError(*e.args)
|
|
326
|
|
|
|
327
|
3
|
def _strip_sensitive_model_fields(self, model, order_by):
|
|
328
|
0
|
for field_name in order_by:
|
|
329
|
|
# strip off the negation prefix `-` if it exists
|
|
330
|
0
|
_field_name = field_name.split('-')[-1]
|
|
331
|
0
|
try:
|
|
332
|
|
# if the field name is encrypted/sensitive, don't sort on it
|
|
333
|
0
|
if _field_name in getattr(model, 'PASSWORD_FIELDS', ()) or \
|
|
334
|
|
getattr(model._meta.get_field(_field_name), '__prevent_search__', False):
|
|
335
|
0
|
raise ParseError(_('cannot order by field %s') % _field_name)
|
|
336
|
0
|
except FieldDoesNotExist:
|
|
337
|
0
|
pass
|
|
338
|
0
|
yield field_name
|
Pelt10