@@ -1,12 +1,7 @@
Loading
1 -
# Python
2 -
import datetime
3 -
import logging
4 -
5 1
6 2
# Django
7 -
from django.core.management.base import BaseCommand, CommandError
3 +
from django.core.management.base import BaseCommand
8 4
from django.db import transaction
9 -
from django.utils.timezone import now
10 5
11 6
# CyBorgBackup
12 7
from cyborgbackup.main.models import Job, Repository
@@ -27,7 +22,6 @@
Loading
27 22
                            default=True,
28 23
                            help='Remove jobs')
29 24
30 -
31 25
    def cleanup_jobs(self):
32 26
        # Sanity check: Is there already a running job on the System?
33 27
        jobs = Job.objects.filter(status="running")
@@ -39,7 +33,7 @@
Loading
39 33
        repoArchives = []
40 34
        if repos.exists():
41 35
            for repo in repos:
42 -
                lines = self.launch_command(["borg", "list", "::"], repo, repo.repository_key, repo.path, **kwargs)
36 +
                lines = self.launch_command(["borg", "list", "::"], repo, repo.repository_key, repo.path)
43 37
44 38
                for line in lines:
45 39
                    archive_name = line.split(' ')[0]  #
@@ -78,4 +72,4 @@
Loading
78 72
                    self.logger.log(99, '%s: %d would be deleted, %d would be skipped.', m.replace('_', ' '),
79 73
                                    deleted, skipped)
80 74
                else:
81 -
                    self.logger.log(99, '%s: %d deleted, %d skipped.', m.replace('_', ' '), deleted, skipped)

@@ -7,7 +7,10 @@
Loading
7 7
class UriCleaner(object):
8 8
    REPLACE_STR = REPLACE_STR
9 9
    # https://regex101.com/r/sV2dO2/2
10 -
    SENSITIVE_URI_PATTERN = re.compile(r'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?\xab\xbb\u201c\u201d\u2018\u2019]))', re.MULTILINE)
10 +
    SENSITIVE_URI_PATTERN = re.compile(r'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s('
11 +
                                       r')<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s('
12 +
                                       r')<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?\xab\xbb\u201c\u201d\u2018\u2019]))',
13 +
                                       re.MULTILINE)
11 14
12 15
    @staticmethod
13 16
    def remove_sensitive(cleartext):

@@ -547,7 +547,7 @@
Loading
547 547
                    if isinstance(override_field_val, (set, list, QuerySet)):
548 548
                        getattr(obj2, field_name).add(*override_field_val)
549 549
                        continue
550 -
                    if override_field_val.__class__.__name__ is 'ManyRelatedManager':
550 +
                    if override_field_val.__class__.__name__ == 'ManyRelatedManager':
551 551
                        src_field_value = override_field_val
552 552
                dest_field = getattr(obj2, field_name)
553 553
                dest_field.add(*list(src_field_value.all().values_list('id', flat=True)))

@@ -11,7 +11,7 @@
Loading
11 11
        # issue is really that Django is missing to avoid masking other
12 12
        # exceptions on Python 2.
13 13
        try:
14 -
            import django
14 +
            import django # noqa
15 15
        except ImportError:
16 16
            raise ImportError(
17 17
                "Couldn't import Django. Are you sure it's installed and "

@@ -1,5 +1,4 @@
Loading
1 1
from .celery import app as celery_app
2 -
from pkg_resources import get_distribution
3 2
import warnings
4 3
import os
5 4
import sys

@@ -49,7 +49,7 @@
Loading
49 49
            search_kwargs = self._expand_search(k, v)
50 50
            if search_kwargs:
51 51
                kwargs.update(search_kwargs)
52 -
                q = reduce(lambda x, y: x | y, [django.db.models.Q(**{u'%s__contains' % _k: _v}) for _k, _v in kwargs.items()])
52 +
                q = reduce(lambda x, y: x | y, [django.db.models.Q(**{u'%s__contains' % _k: _v}) for _k, _v in kwargs.items()])  # noqa
53 53
                self.result = Host.objects.filter(q)
54 54
            else:
55 55
                kwargs[k] = v

@@ -452,7 +452,7 @@
Loading
452 452
    def update_raw_data(self, data):
453 453
        data = super(JobRelaunch, self).update_raw_data(data)
454 454
        try:
455 -
            obj = self.get_object()
455 +
            self.get_object()
456 456
        except PermissionDenied:
457 457
            return data
458 458
        return data

@@ -75,7 +75,7 @@
Loading
75 75
        catalogs_entries_raw = gzip.decompress(base64.b64decode(catalog_data))
76 76
        catalog_entries = json.loads(catalogs_entries_raw.decode('utf-8'))
77 77
        created = []
78 -
        es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
78 +
        # es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
79 79
        for entry in catalog_entries:
80 80
            entry.update({'archive_name': archive_name, 'job_id': job})
81 81
            created.append(self.objects.create(**entry))

@@ -1,6 +1,5 @@
Loading
1 1
import os
2 2
import re
3 -
import sys
4 3
import json
5 4
import stat
6 5
import tempfile
@@ -12,12 +11,17 @@
Loading
12 11
from distutils.version import LooseVersion as Version
13 12
from django.core.management.base import BaseCommand
14 13
from cyborgbackup.main.models import Job, Repository, Catalog
15 -
from django.contrib.auth import get_user_model
16 14
from cyborgbackup.main.expect import run
17 15
from cyborgbackup.main.models.settings import Setting
18 16
from cyborgbackup.main.utils.common import get_ssh_version
19 17
from cyborgbackup.main.utils.encryption import decrypt_field
20 18
19 +
OPENSSH_KEY_ERROR = u'''\
20 +
It looks like you're trying to use a private key in OpenSSH format, which \
21 +
isn't supported by the installed version of OpenSSH on this instance. \
22 +
Try upgrading OpenSSH or providing your private key in an different format. \
23 +
'''
24 +
21 25
22 26
class Command(BaseCommand):
23 27
    """Rebuild Catalog
@@ -181,7 +185,7 @@
Loading
181 185
                        if '{}-'.format(type) in archive_name:
182 186
                            repoArchives.append(archive_name)
183 187
184 -
            entries = Job.objects.filter(job_type='job',status='successful')
188 +
            entries = Job.objects.filter(job_type='job', status='successful')
185 189
            if entries.exists():
186 190
                for entry in entries:
187 191
                    if entry.archive_name != '' and entry.archive_name not in repoArchives:
@@ -196,17 +200,26 @@
Loading
196 200
                                          job_type='job').order_by('-finished')
197 201
                if jobs.exists():
198 202
                    for job in jobs:
199 -
                        if job.archive_name and job.archive_name != '' and job.archive_name == 'rootfs-dave.milkywan.cloud-2019-02-22_02-00':
200 -
                            lines = self.launch_command(["borg", "list", "--json-lines", "::{}".format(job.archive_name)], repo, repo.repository_key, repo.path, **kwargs)
201 -
                            elements = len(lines)
203 +
                        if job.archive_name \
204 +
                                and job.archive_name != '' \
205 +
                                and job.archive_name == 'rootfs-dave.milkywan.cloud-2019-02-22_02-00':
206 +
                            lines = self.launch_command(["borg",
207 +
                                                         "list",
208 +
                                                         "--json-lines",
209 +
                                                         "::{}".format(job.archive_name)],
210 +
                                                        repo,
211 +
                                                        repo.repository_key,
212 +
                                                        repo.path,
213 +
                                                        **kwargs)
214 +
                            # elements = len(lines)
202 215
                            hoursTimezone = round((round((datetime.now()-datetime.utcnow()).total_seconds())/1800)/2)
203 216
                            with transaction.atomic():
204 217
                                for line in lines:
205 218
                                    try:
206 219
                                        data = json.loads(line)
207 220
                                        entries = Catalog.objects.filter(archive_name=job.archive_name,
208 -
                                                                        path=data['path'],
209 -
                                                                        size=data['size'])
221 +
                                                                         path=data['path'],
222 +
                                                                         size=data['size'])
210 223
                                        if not entries.exists():
211 224
                                            entry = Catalog()
212 225
                                            entry.path = data['path']
@@ -218,7 +231,8 @@
Loading
218 231
                                            entry.type = data['type']
219 232
                                            entry.size = data['size']
220 233
                                            entry.healthy = data['healthy']
221 -
                                            entry.mtime = '{}+0{}00'.format(data['mtime'].replace('T', ' '), hoursTimezone)
234 +
                                            entry.mtime = '{}+0{}00'.format(data['mtime'].replace('T', ' '),
235 +
                                                                            hoursTimezone)
222 236
                                            entry.save()
223 237
                                    except Exception as e:
224 238
                                        print(e)

@@ -1,18 +1,15 @@
Loading
1 1
import os
2 2
import re
3 -
import sys
4 3
import json
5 4
import stat
6 5
import tempfile
7 6
from datetime import datetime
8 7
from io import StringIO
9 -
from django.db import transaction
10 8
from collections import OrderedDict
11 9
from django.conf import settings
12 10
from distutils.version import LooseVersion as Version
13 11
from django.core.management.base import BaseCommand
14 -
from cyborgbackup.main.models import Job, Repository, Catalog
15 -
from django.contrib.auth import get_user_model
12 +
from cyborgbackup.main.models import Job, Repository
16 13
from cyborgbackup.main.expect import run
17 14
from cyborgbackup.main.models.settings import Setting
18 15
from cyborgbackup.main.utils.common import get_ssh_version
@@ -20,6 +17,12 @@
Loading
20 17
from elasticsearch import Elasticsearch
21 18
22 19
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
20 +
OPENSSH_KEY_ERROR = u'''\
21 +
It looks like you're trying to use a private key in OpenSSH format, which \
22 +
isn't supported by the installed version of OpenSSH on this instance. \
23 +
Try upgrading OpenSSH or providing your private key in an different format. \
24 +
'''
25 +
23 26
24 27
class Command(BaseCommand):
25 28
    """Rebuild Catalog
@@ -183,14 +186,14 @@
Loading
183 186
                        if '{}-'.format(archtype) in archive_name:
184 187
                            repoArchives.append(archive_name)
185 188
186 -
            entries = Job.objects.filter(job_type='job',status='successful')
189 +
            entries = Job.objects.filter(job_type='job', status='successful')
187 190
            if entries.exists():
188 191
                for entry in entries:
189 192
                    if entry.archive_name != '' and entry.archive_name not in repoArchives:
190 193
                        print('Delete {} from catalog'.format(entry.archive_name))
191 -
                        #Catalog.objects.filter(archive_name=entry.archive_name).delete()
192 -
                        #entry.archive_name = ''
193 -
                        #entry.save()
194 +
                        # Catalog.objects.filter(archive_name=entry.archive_name).delete()
195 +
                        # entry.archive_name = ''
196 +
                        # entry.save()
194 197
195 198
            for repo in repos:
196 199
                jobs = Job.objects.filter(policy__repository_id=repo.pk,
@@ -206,8 +209,18 @@
Loading
206 209
                            }]}}}
207 210
                            res = es.search(index="catalog", body=search_object)
208 211
                            if res['hits']['total'] == 0:
209 -
                                lines = self.launch_command(["borg", "list", "--json-lines", "::{}".format(job.archive_name)], repo, repo.repository_key, repo.path, **kwargs)
210 -
                                hoursTimezone = round((round((datetime.now()-datetime.utcnow()).total_seconds())/1800)/2)
212 +
                                lines = self.launch_command(["borg",
213 +
                                                             "list",
214 +
                                                             "--json-lines",
215 +
                                                             "::{}".format(job.archive_name)],
216 +
                                                            repo,
217 +
                                                            repo.repository_key,
218 +
                                                            repo.path,
219 +
                                                            **kwargs)
220 +
                                hoursTimezone = round(
221 +
                                    (round(
222 +
                                        (datetime.now()-datetime.utcnow()).total_seconds())/1800)
223 +
                                    / 2)
211 224
                                for line in lines:
212 225
                                    data = None
213 226
                                    try:
@@ -219,7 +232,7 @@
Loading
219 232
                                             'term': {
220 233
                                                 'path.keyword': data['path']
221 234
                                             }
222 -
                                        },{
235 +
                                        }, {
223 236
                                             'term': {
224 237
                                                 'archive_name.keyword': job.archive_name,
225 238
                                             }
@@ -238,5 +251,6 @@
Loading
238 251
                                                'type': data['type'],
239 252
                                                'size': data['size'],
240 253
                                                'healthy': data['healthy'],
241 -
                                                'mtime': '{}+0{}00'.format(data['mtime'].replace('T', ' '), hoursTimezone)
242 -
                                            });
254 +
                                                'mtime': '{}+0{}00'.format(data['mtime'].replace('T', ' '),
255 +
                                                                           hoursTimezone)
256 +
                                            })

@@ -13,7 +13,7 @@
Loading
13 13
    1. Import the include() function: from django.conf.urls import url, include
14 14
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
15 15
"""
16 -
from django.conf.urls import url, include
16 +
from django.conf.urls import url
17 17
from django.contrib import admin
18 18
from django.conf import settings
19 19
from django.urls import include, path

@@ -10,7 +10,7 @@
Loading
10 10
11 11
logger = logging.getLogger('cyborgbackup.api.permissions')
12 12
13 -
__all__ = ['ModelAccessPermission', 'UserPermission', 'IsSuperUser', 'check_user_acccess']
13 +
__all__ = ['ModelAccessPermission', 'UserPermission', 'IsSuperUser', 'check_user_access']
14 14
15 15
16 16
def check_user_access(user, model_class, action, *args, **kwargs):

@@ -16,6 +16,7 @@
Loading
16 16
    char_filter=["html_strip"]
17 17
)
18 18
19 +
19 20
@catalog_index.doc_type
20 21
class CatalogDocument(Document):
21 22
    """Catalog elasticsearch document"""

@@ -1,13 +1,7 @@
Loading
1 1
from django_elasticsearch_dsl_drf.constants import (
2 -
    LOOKUP_FILTER_RANGE,
3 2
    LOOKUP_FILTER_REGEXP,
4 3
    LOOKUP_FILTER_TERM,
5 -
    LOOKUP_FILTER_WILDCARD,
6 -
    LOOKUP_QUERY_IN,
7 -
    LOOKUP_QUERY_GT,
8 -
    LOOKUP_QUERY_GTE,
9 -
    LOOKUP_QUERY_LT,
10 -
    LOOKUP_QUERY_LTE,
4 +
    LOOKUP_FILTER_WILDCARD
11 5
)
12 6
from django_elasticsearch_dsl_drf.filter_backends import (
13 7
    FilteringFilterBackend,
@@ -72,5 +66,3 @@
Loading
72 66
73 67
    # Specify default ordering
74 68
    ordering = ('archive_name', 'path')
75 -
76 -

@@ -444,7 +444,7 @@
Loading
444 444
            if node in active_queues:
445 445
                active_tasks = active_queues[node]
446 446
            else:
447 -
                if instance is None:
447 +
                if node is None:
448 448
                    logger.error("Execution node Instance {} not found in database. "
449 449
                                 "The node is currently executing jobs {}".format(
450 450
                                     node, [j.log_format for j in node_jobs]))

@@ -347,10 +347,11 @@
Loading
347 347
                    )
348 348
                if obj.id in self.context['capability_map']:
349 349
                    capabilities_cache = self.context['capability_map'][obj.id]
350 -
            return get_user_capabilities(
351 -
                view.request.user, obj, method_list=self.show_capabilities, parent_obj=parent_obj,
352 -
                capabilities_cache=capabilities_cache
353 -
            )
350 +
            return {parent_obj, capabilities_cache}
351 +
            # return get_user_capabilities(
352 +
            #     view.request.user, obj, method_list=self.show_capabilities, parent_obj=parent_obj,
353 +
            #     capabilities_cache=capabilities_cache
354 +
            # )
354 355
        else:
355 356
            # Contextual information to produce user_capabilities doesn't exist
356 357
            return {}
@@ -405,7 +406,8 @@
Loading
405 406
            field_kwargs['default'] = field_kwargs['initial'] = model_field.get_default()
406 407
407 408
        # Enforce minimum value of 0 for PositiveIntegerFields.
408 -
        if isinstance(model_field, (models.PositiveIntegerField, models.PositiveSmallIntegerField)) and 'choices' not in field_kwargs:
409 +
        if isinstance(model_field, (models.PositiveIntegerField, models.PositiveSmallIntegerField)) \
410 +
                and 'choices' not in field_kwargs:
409 411
            field_kwargs['min_value'] = 0
410 412
411 413
        # Use custom boolean field that allows null and empty string as False values.

@@ -2,6 +2,7 @@
Loading
2 2
3 3
from cyborgbackup.elasticsearch.documents.catalogs import CatalogDocument
4 4
5 +
5 6
class ESCatalogDocumentSerializer(DocumentSerializer):
6 7
7 8
    class Meta:

@@ -64,16 +64,17 @@
Loading
64 64
65 65
logger = logging.getLogger('cyborgbackup.main.tasks')
66 66
67 +
67 68
def humanbytes(B):
68 -
    'Return the given bytes as a human friendly KB, MB, GB, or TB string'
69 +
    '  Return the given bytes as a human friendly KB, MB, GB, or TB string'
69 70
    B = float(B)
70 71
    KB = float(1024)
71 -
    MB = float(KB ** 2) # 1,048,576
72 -
    GB = float(KB ** 3) # 1,073,741,824
73 -
    TB = float(KB ** 4) # 1,099,511,627,776
72 +
    MB = float(KB ** 2)  # 1,048,576
73 +
    GB = float(KB ** 3)  # 1,073,741,824
74 +
    TB = float(KB ** 4)  # 1,099,511,627,776
74 75
75 76
    if B < KB:
76 -
        return '{0} {1}'.format(B,'Bytes' if 0 == B > 1 else 'Byte')
77 +
        return '{0} {1}'.format(B, 'Bytes' if 0 == B > 1 else 'Byte')
77 78
    elif KB <= B < MB:
78 79
        return '{0:.2f} KB'.format(B/KB)
79 80
    elif MB <= B < GB:
@@ -83,6 +84,7 @@
Loading
83 84
    elif TB <= B:
84 85
        return '{0:.2f} TB'.format(B/TB)
85 86
87 +
86 88
def build_report(type):
87 89
    since = 24*60*60
88 90
    if type == 'daily':

@@ -2,7 +2,7 @@
Loading
2 2
from rest_framework import serializers
3 3
4 4
# CyBorgBackup
5 -
from cyborgbackup.main.fields import *
5 +
# from cyborgbackup.main.fields import *
6 6
7 7
__all__ = ['BooleanNullField', 'CharNullField', 'ChoiceNullField', 'VerbatimField']
8 8
Files Coverage
src 35.65%
Project Totals (74 files) 35.65%
84.2
TRAVIS_PYTHON_VERSION=3.6
TRAVIS_OS_NAME=linux
84.1
TRAVIS_PYTHON_VERSION=3.5
TRAVIS_OS_NAME=linux
84.3
TRAVIS_PYTHON_VERSION=3.7
TRAVIS_OS_NAME=linux

No yaml found.

Create your codecov.yml to customize your Codecov experience

Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading