Compare `f32c885 ... +249 ... e5f7121`

import sys
import textwrap
from hmac import HMAC, compare_digest
from hashlib import sha256, pbkdf2_hmac
from hashlib import sha256, sha512, pbkdf2_hmac

from .helpers import IntegrityError, get_keys_dir, Error, yes
import msgpack

from .helpers import StableDict, IntegrityError, get_keys_dir, get_security_dir, Error, yes, bin_to_hex
from .logger import create_logger
logger = create_logger()

from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
from .compress import Compressor
import msgpack
from .crypto import hkdf_hmac_sha512
from .compress import Compressor, CNONE

PREFIX = b'\0' * 8


@@ -30,6 +32,10 @@

    """Unsupported payload type {}. A newer version is required to access this repository."""


class UnsupportedManifestError(Error):
    """Unsupported manifest envelope. A newer version is required to access this repository."""


class KeyfileNotFoundError(Error):
    """No key file for repository {} found in {}."""


@@ -38,6 +44,32 @@

38	44		"""No key entry found in the config of repository {}."""
39	45
40	46
	47	+	class TAMRequiredError(IntegrityError):
	48	+	__doc__ = textwrap.dedent("""
	49	+	Manifest is unauthenticated, but it is required for this repository.
	50	+
	51	+	This either means that you are under attack, or that you modified this repository
	52	+	with a Borg version older than 1.0.9 after TAM authentication was enabled.
	53	+
	54	+	In the latter case, use "borg upgrade --tam --force '{}'" to re-authenticate the manifest.
	55	+	""").strip()
	56	+	traceback = False
	57	+
	58	+
	59	+	class TAMInvalid(IntegrityError):
	60	+	__doc__ = IntegrityError.__doc__
	61	+	traceback = False
	62	+
	63	+	def __init__(self):
	64	+	# Error message becomes: "Data integrity error: Manifest authentication did not verify"
	65	+	super().__init__('Manifest authentication did not verify')
	66	+
	67	+
	68	+	class TAMUnsupportedSuiteError(IntegrityError):
	69	+	"""Could not verify manifest: Unsupported suite {!r}; a newer version is needed."""
	70	+	traceback = False
	71	+
	72	+
41	73		def key_creator(repository, args):
42	74		if args.encryption == 'keyfile':
43	75		return KeyfileKey.create(repository, args)

@@ -63,6 +95,16 @@

        raise UnsupportedPayloadError(key_type)


def tam_required_file(repository):
    security_dir = get_security_dir(bin_to_hex(repository.id))
    return os.path.join(security_dir, 'tam_required')


def tam_required(repository):
    file = tam_required_file(repository)
    return os.path.isfile(file)


class KeyBase:
    TYPE = None  # override in subclasses


@@ -71,23 +113,90 @@

        self.repository = repository
        self.target = None  # key location file path / repo obj
        self.compressor = Compressor('none')
        self.tam_required = True

    def id_hash(self, data):
        """Return HMAC hash using the "id" HMAC key
        """

    def encrypt(self, data):
    def encrypt(self, data, none_compression=False):
        pass

    def decrypt(self, id, data):
        pass

    def _tam_key(self, salt, context):
        return hkdf_hmac_sha512(
            ikm=self.id_key + self.enc_key + self.enc_hmac_key,
            salt=salt,
            info=b'borg-metadata-authentication-' + context,
            output_length=64
        )

    def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest'):
        metadata_dict = StableDict(metadata_dict)
        tam = metadata_dict['tam'] = StableDict({
            'type': 'HKDF_HMAC_SHA512',
            'hmac': bytes(64),
            'salt': os.urandom(64),
        })
        packed = msgpack.packb(metadata_dict, unicode_errors='surrogateescape')
        tam_key = self._tam_key(tam['salt'], context)
        tam['hmac'] = HMAC(tam_key, packed, sha512).digest()
        return msgpack.packb(metadata_dict, unicode_errors='surrogateescape')

    def unpack_and_verify_manifest(self, data, force_tam_not_required=False):
        """Unpack msgpacked *data* and return (object, did_verify)."""
        if data.startswith(b'\xc1' * 4):
            # This is a manifest from the future, we can't read it.
            raise UnsupportedManifestError()
        tam_required = self.tam_required
        if force_tam_not_required and tam_required:
            logger.warning('Manifest authentication DISABLED.')
            tam_required = False
        data = bytearray(data)
        # Since we don't trust these bytes we use the slower Python unpacker,
        # which is assumed to have a lower probability of security issues.
        unpacked = msgpack.fallback.unpackb(data, object_hook=StableDict, unicode_errors='surrogateescape')
        if b'tam' not in unpacked:
            if tam_required:
                raise TAMRequiredError(self.repository._location.canonical_path())
            else:
                logger.debug('TAM not found and not required')
                return unpacked, False
        tam = unpacked.pop(b'tam', None)
        if not isinstance(tam, dict):
            raise TAMInvalid()
        tam_type = tam.get(b'type', b'<none>').decode('ascii', 'replace')
        if tam_type != 'HKDF_HMAC_SHA512':
            if tam_required:
                raise TAMUnsupportedSuiteError(repr(tam_type))
            else:
                logger.debug('Ignoring TAM made with unsupported suite, since TAM is not required: %r', tam_type)
                return unpacked, False
        tam_hmac = tam.get(b'hmac')
        tam_salt = tam.get(b'salt')
        if not isinstance(tam_salt, bytes) or not isinstance(tam_hmac, bytes):
            raise TAMInvalid()
        offset = data.index(tam_hmac)
        data[offset:offset + 64] = bytes(64)
        tam_key = self._tam_key(tam_salt, context=b'manifest')
        calculated_hmac = HMAC(tam_key, data, sha512).digest()
        if not compare_digest(calculated_hmac, tam_hmac):
            raise TAMInvalid()
        logger.debug('TAM-verified manifest')
        return unpacked, True


class PlaintextKey(KeyBase):
    TYPE = 0x02

    chunk_seed = 0

    def __init__(self, repository):
        super().__init__(repository)
        self.tam_required = False

    @classmethod
    def create(cls, repository, args):
        logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile" to enable encryption.')

@@ -100,17 +209,25 @@

    def id_hash(self, data):
        return sha256(data).digest()

    def encrypt(self, data):
        return b''.join([self.TYPE_STR, self.compressor.compress(data)])
    def encrypt(self, data, none_compression=False):
        if none_compression:
            compressed = CNONE().compress(data)
        else:
            compressed = self.compressor.compress(data)
        return b''.join([self.TYPE_STR, compressed])

    def decrypt(self, id, data):
        if data[0] != self.TYPE:
            raise IntegrityError('Invalid encryption envelope')
            id_str = bin_to_hex(id) if id is not None else '(unknown)'
            raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str)
        data = self.compressor.decompress(memoryview(data)[1:])
        if id and sha256(data).digest() != id:
            raise IntegrityError('Chunk id verification failed')
            raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id))
        return data

    def _tam_key(self, salt, context):
        return salt + context


class AESKeyBase(KeyBase):
    """Common base class shared by KeyfileKey and PassphraseKey

@@ -132,8 +249,11 @@

        """
        return HMAC(self.id_key, data, sha256).digest()

    def encrypt(self, data):
        data = self.compressor.compress(data)
    def encrypt(self, data, none_compression=False):
        if none_compression:
            data = CNONE().compress(data)
        else:
            data = self.compressor.compress(data)
        self.enc_cipher.reset()
        data = b''.join((self.enc_cipher.iv[8:], self.enc_cipher.encrypt(data)))
        hmac = HMAC(self.enc_hmac_key, data, sha256).digest()

@@ -142,24 +262,26 @@

    def decrypt(self, id, data):
        if not (data[0] == self.TYPE or
            data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
            raise IntegrityError('Invalid encryption envelope')
            id_str = bin_to_hex(id) if id is not None else '(unknown)'
            raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str)
        hmac_given = memoryview(data)[1:33]
        hmac_computed = memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest())
        if not compare_digest(hmac_computed, hmac_given):
            raise IntegrityError('Encryption envelope checksum mismatch')
            id_str = bin_to_hex(id) if id is not None else '(unknown)'
            raise IntegrityError('Chunk %s: Encryption envelope checksum mismatch' % id_str)
        self.dec_cipher.reset(iv=PREFIX + data[33:41])
        data = self.compressor.decompress(self.dec_cipher.decrypt(data[41:]))
        if id:
            hmac_given = id
            hmac_computed = HMAC(self.id_key, data, sha256).digest()
            if not compare_digest(hmac_computed, hmac_given):
                raise IntegrityError('Chunk id verification failed')
                raise IntegrityError('Chunk %s: Chunk id verification failed' % bin_to_hex(id))
        return data

    def extract_nonce(self, payload):
        if not (payload[0] == self.TYPE or
            payload[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
            raise IntegrityError('Invalid encryption envelope')
            raise IntegrityError('Manifest: Invalid encryption envelope')
        nonce = bytes_to_long(payload[33:41])
        return nonce


@@ -190,8 +312,9 @@


    @classmethod
    def verification(cls, passphrase):
        if yes('Do you want your passphrase to be displayed for verification? [yN]: ',
               env_var_override='BORG_DISPLAY_PASSPHRASE'):
        msg = 'Do you want your passphrase to be displayed for verification? [yN]: '
        if yes(msg, retry_msg=msg, invalid_msg='Invalid answer, try again.',
               retry=True, env_var_override='BORG_DISPLAY_PASSPHRASE'):
            print('Your passphrase (between double-quotes): "%s"' % passphrase,
                  file=sys.stderr)
            print('Make sure the passphrase displayed above is exactly what you wanted.',

@@ -200,7 +323,7 @@

                passphrase.encode('ascii')
            except UnicodeEncodeError:
                print('Your passphrase (UTF-8 encoding in hex): %s' %
                      hexlify(passphrase.encode('utf-8')).decode('ascii'),
                      bin_to_hex(passphrase.encode('utf-8')),
                      file=sys.stderr)
                print('As you have a non-ASCII passphrase, it is recommended to keep the UTF-8 encoding in hex together with the passphrase at a safe place.',
                      file=sys.stderr)

@@ -265,6 +388,7 @@

                key.decrypt(None, manifest_data)
                num_blocks = num_aes_blocks(len(manifest_data) - 41)
                key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
                key._passphrase = passphrase
                return key
            except IntegrityError:
                passphrase = Passphrase.getpass(prompt)

@@ -280,6 +404,7 @@

    def init(self, repository, passphrase):
        self.init_from_random_data(passphrase.kdf(repository.id, self.iterations, 100))
        self.init_ciphers()
        self.tam_required = False


class KeyfileKeyBase(AESKeyBase):

@@ -303,6 +428,7 @@

                raise PassphraseWrong
        num_blocks = num_aes_blocks(len(manifest_data) - 41)
        key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
        key._passphrase = passphrase
        return key

    def find_key(self):

@@ -323,6 +449,7 @@

            self.enc_hmac_key = key[b'enc_hmac_key']
            self.id_key = key[b'id_key']
            self.chunk_seed = key[b'chunk_seed']
            self.tam_required = key.get(b'tam_required', tam_required(self.repository))
            return True
        return False


@@ -359,15 +486,16 @@

            'enc_hmac_key': self.enc_hmac_key,
            'id_key': self.id_key,
            'chunk_seed': self.chunk_seed,
            'tam_required': self.tam_required,
        }
        data = self.encrypt_key_file(msgpack.packb(key), passphrase)
        key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))
        return key_data

    def change_passphrase(self):
        passphrase = Passphrase.new(allow_empty=True)
    def change_passphrase(self, passphrase=None):
        if passphrase is None:
            passphrase = Passphrase.new(allow_empty=True)
        self.save(self.target, passphrase)
        logger.info('Key updated')

    @classmethod
    def create(cls, repository, args):

@@ -426,7 +554,7 @@

    def save(self, target, passphrase):
        key_data = self._save(passphrase)
        with open(target, 'w') as fd:
            fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii')))
            fd.write('%s %s\n' % (self.FILE_ID, bin_to_hex(self.repository_id)))
            fd.write(key_data)
            fd.write('\n')
        self.target = target

@@ -433 +561 @@

@@ -1,6 +1,5 @@

from binascii import hexlify
from contextlib import contextmanager
from datetime import datetime, timezone
from datetime import datetime, timezone, timedelta
from getpass import getuser
from itertools import groupby
import errno

@@ -17,10 +16,10 @@

import time
from io import BytesIO
from . import xattr
from .helpers import Error, uid2user, user2uid, gid2group, group2gid, \
from .helpers import Error, uid2user, user2uid, gid2group, group2gid, bin_to_hex, \
    parse_timestamp, to_localtime, format_time, format_timedelta, remove_surrogates, \
    Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \
    ProgressIndicatorPercent
    ProgressIndicatorPercent, IntegrityError
from .platform import acl_get, acl_set
from .chunker import Chunker
from .hashindex import ChunkIndex

@@ -186,8 +185,8 @@

        """Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable."""

    def __init__(self, repository, key, manifest, name, cache=None, create=False,
                 checkpoint_interval=300, numeric_owner=False, progress=False,
                 chunker_params=CHUNKER_PARAMS, start=None, end=None):
                 checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, progress=False,
                 chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None):
        self.cwd = os.getcwd()
        self.key = key
        self.repository = repository

@@ -199,9 +198,14 @@

        self.name = name
        self.checkpoint_interval = checkpoint_interval
        self.numeric_owner = numeric_owner
        self.noatime = noatime
        self.noctime = noctime
        assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.'
        if start is None:
            start = datetime.utcnow()
            start_monotonic = time.monotonic()
        self.start = start
        self.start_monotonic = start_monotonic
        if end is None:
            end = datetime.utcnow()
        self.end = end

@@ -211,7 +215,7 @@

            self.chunker = Chunker(self.key.chunk_seed, *chunker_params)
            if name in manifest.archives:
                raise self.AlreadyExists(name)
            self.last_checkpoint = time.time()
            self.last_checkpoint = time.monotonic()
            i = 0
            while True:
                self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '')

@@ -227,7 +231,7 @@


    def _load_meta(self, id):
        data = self.key.decrypt(id, self.repository.get(id))
        metadata = msgpack.unpackb(data)
        metadata = msgpack.unpackb(data, unicode_errors='surrogateescape')
        if metadata[b'version'] != 1:
            raise Exception('Unknown archive metadata version')
        return metadata

@@ -254,7 +258,7 @@


    @property
    def fpr(self):
        return hexlify(self.id).decode('ascii')
        return bin_to_hex(self.id)

    @property
    def duration(self):

@@ -286,9 +290,9 @@

        if self.show_progress:
            self.stats.show_progress(item=item, dt=0.2)
        self.items_buffer.add(item)
        if time.time() - self.last_checkpoint > self.checkpoint_interval:
        if time.monotonic() - self.last_checkpoint > self.checkpoint_interval:
            self.write_checkpoint()
            self.last_checkpoint = time.time()
            self.last_checkpoint = time.monotonic()

    def write_checkpoint(self):
        self.save(self.checkpoint_name)

@@ -300,14 +304,17 @@

        if name in self.manifest.archives:
            raise self.AlreadyExists(name)
        self.items_buffer.flush(flush=True)
        duration = timedelta(seconds=time.monotonic() - self.start_monotonic)
        if timestamp is None:
            self.end = datetime.utcnow()
            self.start = self.end - duration
            start = self.start
            end = self.end
        else:
            self.end = timestamp
            start = timestamp
            end = timestamp  # we only have 1 value
            self.start = timestamp - duration
            end = timestamp
            start = self.start
        metadata = StableDict({
            'version': 1,
            'name': name,

@@ -318,7 +325,7 @@

            'time': start.isoformat(),
            'time_end': end.isoformat(),
        })
        data = msgpack.packb(metadata, unicode_errors='surrogateescape')
        data = self.key.pack_and_authenticate_metadata(metadata, context=b'archive')
        self.id = self.key.id_hash(data)
        self.cache.add_chunk(self.id, data, self.stats)
        self.manifest.archives[name] = {'id': self.id, 'time': metadata['time']}

@@ -522,7 +529,7 @@

            try:
                self.cache.chunk_decref(id, stats)
            except KeyError:
                cid = hexlify(id).decode('ascii')
                cid = bin_to_hex(id)
                raise ChunksIndexError(cid)
            except Repository.ObjectNotFound as e:
                # object not in repo - strange, but we wanted to delete it anyway.

@@ -572,10 +579,15 @@

            b'mode': st.st_mode,
            b'uid': st.st_uid, b'user': uid2user(st.st_uid),
            b'gid': st.st_gid, b'group': gid2group(st.st_gid),
            b'atime': int_to_bigint(st.st_atime_ns),
            b'ctime': int_to_bigint(st.st_ctime_ns),
            b'mtime': int_to_bigint(st.st_mtime_ns),
        }
        # borg can work with archives only having mtime (older attic archives do not have
        # atime/ctime). it can be useful to omit atime/ctime, if they change without the
        # file content changing - e.g. to get better metadata deduplication.
        if not self.noatime:
            item[b'atime'] = int_to_bigint(st.st_atime_ns)
        if not self.noctime:
            item[b'ctime'] = int_to_bigint(st.st_ctime_ns)
        if self.numeric_owner:
            item[b'user'] = item[b'group'] = None
        with backup_io():

@@ -610,7 +622,8 @@

            return 'b'  # block device

    def process_symlink(self, path, st):
        source = os.readlink(path)
        with backup_io():
            source = os.readlink(path)
        item = {b'path': make_path_safe(path), b'source': source}
        item.update(self.stat_attrs(st, path))
        self.add_item(item)

@@ -641,14 +654,12 @@

        # Is it a hard link?
        if st.st_nlink > 1:
            source = self.hard_links.get((st.st_ino, st.st_dev))
            if (st.st_ino, st.st_dev) in self.hard_links:
            if source is not None:
                item = self.stat_attrs(st, path)
                item.update({b'path': safe_path, b'source': source})
                self.add_item(item)
                status = 'h'  # regular file, hardlink (to already seen inodes)
                return status
            else:
                self.hard_links[st.st_ino, st.st_dev] = safe_path
        is_special_file = is_special(st.st_mode)
        if not is_special_file:
            path_hash = self.key.id_hash(os.path.join(self.cwd, path).encode('utf-8', 'surrogateescape'))

@@ -696,6 +707,9 @@

            item[b'mode'] = stat.S_IFREG | stat.S_IMODE(item[b'mode'])
        self.stats.nfiles += 1
        self.add_item(item)
        if st.st_nlink > 1 and source is None:
            # Add the hard link reference *after* the file has been added to the archive.
            self.hard_links[st.st_ino, st.st_dev] = safe_path
        return status

    @staticmethod

@@ -833,13 +847,22 @@

        self.repair = repair
        self.repository = repository
        self.init_chunks()
        if not self.chunks:
            logger.error('Repository contains no apparent data at all, cannot continue check/repair.')
            return False
        self.key = self.identify_key(repository)
        if Manifest.MANIFEST_ID not in self.chunks:
            logger.error("Repository manifest not found!")
            self.error_found = True
            self.manifest = self.rebuild_manifest()
        else:
            self.manifest, _ = Manifest.load(repository, key=self.key)
            try:
                self.manifest, _ = Manifest.load(repository, key=self.key)
            except IntegrityError as exc:
                logger.error('Repository manifest is corrupted: %s', exc)
                self.error_found = True
                del self.chunks[Manifest.MANIFEST_ID]
                self.manifest = self.rebuild_manifest()
        self.rebuild_refcounts(archive=archive, last=last, prefix=prefix)
        self.orphan_chunks_check()
        self.finish(save_space=save_space)

@@ -853,8 +876,9 @@

        """Fetch a list of all object keys from repository
        """
        # Explicitly set the initial hash table capacity to avoid performance issues
        # due to hash table "resonance"
        capacity = int(len(self.repository) * 1.35 + 1)  # > len * 1.0 / HASH_MAX_LOAD (see _hashindex.c)
        # due to hash table "resonance".
        # Since reconstruction of archive items can add some new chunks, add 10 % headroom
        capacity = int(len(self.repository) / ChunkIndex.MAX_LOAD_FACTOR * 1.1)
        self.chunks = ChunkIndex(capacity)
        marker = None
        while True:

@@ -895,7 +919,12 @@

        archive_keys_serialized = [msgpack.packb(name) for name in ARCHIVE_KEYS]
        for chunk_id, _ in self.chunks.iteritems():
            cdata = self.repository.get(chunk_id)
            data = self.key.decrypt(chunk_id, cdata)
            try:
                data = self.key.decrypt(chunk_id, cdata)
            except IntegrityError as exc:
                logger.error('Skipping corrupted chunk: %s', exc)
                self.error_found = True
                continue
            if not valid_msgpacked_dict(data, archive_keys_serialized):
                continue
            if b'cmdline' not in data or b'\xa7version\x01' not in data:

@@ -907,8 +936,18 @@

            except (TypeError, ValueError, StopIteration):
                continue
            if valid_archive(archive):
                logger.info('Found archive %s', archive[b'name'].decode('utf-8'))
                manifest.archives[archive[b'name'].decode('utf-8')] = {b'id': chunk_id, b'time': archive[b'time']}
                name = archive[b'name'].decode()
                logger.info('Found archive %s', name)
                if name in manifest.archives:
                    i = 1
                    while True:
                        new_name = '%s.%d' % (name, i)
                        if new_name not in manifest.archives:
                            break
                        i += 1
                    logger.warning('Duplicate archive name %s, storing as %s', name, new_name)
                    name = new_name
                manifest.archives[name] = {b'id': chunk_id, b'time': archive[b'time']}
        logger.info('Manifest rebuild complete.')
        return manifest


@@ -1008,16 +1047,26 @@

                return _state

            def report(msg, chunk_id, chunk_no):
                cid = hexlify(chunk_id).decode('ascii')
                cid = bin_to_hex(chunk_id)
                msg += ' [chunk: %06d_%s]' % (chunk_no, cid)  # see debug-dump-archive-items
                self.error_found = True
                logger.error(msg)

            def list_keys_safe(keys):
                return ', '.join((k.decode() if isinstance(k, bytes) else str(k) for k in keys))

            def valid_item(obj):
                if not isinstance(obj, StableDict):
                    return False
                    return False, 'not a dictionary'
                # A bug in Attic up to and including release 0.13 added a (meaningless) b'acl' key to every item.
                # We ignore it here, should it exist. See test_attic013_acl_bug for details.
                obj.pop(b'acl', None)
                keys = set(obj)
                return REQUIRED_ITEM_KEYS.issubset(keys) and keys.issubset(item_keys)
                if not REQUIRED_ITEM_KEYS.issubset(keys):
                    return False, 'missing required keys: ' + list_keys_safe(REQUIRED_ITEM_KEYS - keys)
                if not keys.issubset(item_keys):
                    return False, 'invalid keys: ' + list_keys_safe(keys - item_keys)
                return True, ''

            i = 0
            for state, items in groupby(archive[b'items'], missing_chunk_detector):

@@ -1033,10 +1082,11 @@

                    unpacker.feed(self.key.decrypt(chunk_id, cdata))
                    try:
                        for item in unpacker:
                            if valid_item(item):
                            valid, reason = valid_item(item)
                            if valid:
                                yield item
                            else:
                                report('Did not get expected metadata dict when unpacking item metadata', chunk_id, i)
                                report('Did not get expected metadata dict when unpacking item metadata (%s)' % reason, chunk_id, i)
                    except RobustUnpacker.UnpackerCrashed as err:
                        report('Unpacker crashed while unpacking item metadata, trying to resync...', chunk_id, i)
                        unpacker.resync()

@@ -1051,13 +1101,21 @@

                                   key=lambda name_info: name_info[1][b'time'])
            if prefix is not None:
                archive_items = [item for item in archive_items if item[0].startswith(prefix)]
                if not archive_items:
                    logger.warning('--prefix %s does not match any archives', prefix)
            num_archives = len(archive_items)
            end = None if last is None else min(num_archives, last)
            if last is not None and end < last:
                logger.warning('--last %d archives: only found %d archives', last, end)
        else:
            # we only want one specific archive
            archive_items = [item for item in self.manifest.archives.items() if item[0] == archive]
            num_archives = 1
            end = 1
            if not archive_items:
                logger.error('Archive %s does not exist', archive)
                self.error_found = True
                return

        with cache_if_remote(self.repository) as repository:
            for i, (name, info) in enumerate(archive_items[:end]):

@@ -1064 +1122 @@

@@ -1,5 +1,7 @@

import argparse
from binascii import hexlify
from collections import namedtuple
import contextlib
from functools import wraps
import grp
import os

@@ -10,6 +12,7 @@

from shutil import get_terminal_size
import sys
import platform
import signal
import threading
import time
import unicodedata

@@ -25,6 +28,7 @@

from operator import attrgetter

from . import __version__ as borg_version
from . import __version_tuple__ as borg_version_tuple
from . import hashindex
from . import chunker
from . import crypto

@@ -51,17 +55,23 @@

    # show a traceback?
    traceback = False

    def __init__(self, *args):
        super().__init__(*args)
        self.args = args

    def get_message(self):
        return type(self).__doc__.format(*self.args)

    __str__ = get_message


class ErrorWithTraceback(Error):
    """like Error, but show a traceback also"""
    traceback = True


class IntegrityError(ErrorWithTraceback):
    """Data integrity error"""
    """Data integrity error: {}"""


class ExtensionModuleError(Error):

@@ -78,13 +88,13 @@


def check_extension_modules():
    from . import platform
    if hashindex.API_VERSION != 2:
    if hashindex.API_VERSION != '1.0_01':
        raise ExtensionModuleError
    if chunker.API_VERSION != 2:
    if chunker.API_VERSION != '1.0_01':
        raise ExtensionModuleError
    if crypto.API_VERSION != 2:
    if crypto.API_VERSION != '1.0_01':
        raise ExtensionModuleError
    if platform.API_VERSION != 2:
    if platform.API_VERSION != '1.0_01':
        raise ExtensionModuleError



@@ -99,10 +109,12 @@

        self.key = key
        self.repository = repository
        self.item_keys = frozenset(item_keys) if item_keys is not None else ITEM_KEYS
        self.tam_verified = False
        self.timestamp = None

    @classmethod
    def load(cls, repository, key=None):
        from .key import key_factory
    def load(cls, repository, key=None, force_tam_not_required=False):
        from .key import key_factory, tam_required_file, tam_required
        from .repository import Repository
        from .archive import ITEM_KEYS
        try:

@@ -113,8 +125,8 @@

            key = key_factory(repository, cdata)
        manifest = cls(key, repository)
        data = key.decrypt(None, cdata)
        m, manifest.tam_verified = key.unpack_and_verify_manifest(data, force_tam_not_required=force_tam_not_required)
        manifest.id = key.id_hash(data)
        m = msgpack.unpackb(data)
        if not m.get(b'version') == 1:
            raise ValueError('Invalid manifest version')
        manifest.archives = dict((k.decode('utf-8'), v) for k, v in m[b'archives'].items())

@@ -124,19 +136,40 @@

        manifest.config = m[b'config']
        # valid item keys are whatever is known in the repo or every key we know
        manifest.item_keys = frozenset(m.get(b'item_keys', [])) | ITEM_KEYS

        if manifest.tam_verified:
            manifest_required = manifest.config.get(b'tam_required', False)
            security_required = tam_required(repository)
            if manifest_required and not security_required:
                logger.debug('Manifest is TAM verified and says TAM is required, updating security database...')
                file = tam_required_file(repository)
                open(file, 'w').close()
            if not manifest_required and security_required:
                logger.debug('Manifest is TAM verified and says TAM is *not* required, updating security database...')
                os.unlink(tam_required_file(repository))
        return manifest, key

    def write(self):
        self.timestamp = datetime.utcnow().isoformat()
        data = msgpack.packb(StableDict({
        if self.key.tam_required:
            self.config[b'tam_required'] = True
        # self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly
        if self.timestamp is None:
            self.timestamp = datetime.utcnow().isoformat()
        else:
            prev_ts = datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f")
            incremented = (prev_ts + timedelta(microseconds=1)).isoformat()
            self.timestamp = max(incremented, datetime.utcnow().isoformat())
        m = {
            'version': 1,
            'archives': self.archives,
            'archives': StableDict((name, StableDict(archive)) for name, archive in self.archives.items()),
            'timestamp': self.timestamp,
            'config': self.config,
            'item_keys': tuple(self.item_keys),
        }))
            'config': StableDict(self.config),
            'item_keys': tuple(sorted(self.item_keys)),
        }
        self.tam_verified = True
        data = self.key.pack_and_authenticate_metadata(m)
        self.id = self.key.id_hash(data)
        self.repository.put(self.MANIFEST_ID, self.key.encrypt(data))
        self.repository.put(self.MANIFEST_ID, self.key.encrypt(data, none_compression=True))

    def list_archive_infos(self, sort_by=None, reverse=False):
        # inexpensive Archive.list_archives replacement if we just need .name, .id, .ts

@@ -215,17 +248,21 @@

        return format_file_size(self.csize)

    def show_progress(self, item=None, final=False, stream=None, dt=None):
        now = time.time()
        now = time.monotonic()
        if dt is None or now - self.last_progress > dt:
            self.last_progress = now
            columns, lines = get_terminal_size()
            if not final:
                msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self)
                path = remove_surrogates(item[b'path']) if item else ''
                space = columns - len(msg)
                if space < len('...') + len(path):
                    path = '%s...%s' % (path[:(space // 2) - len('...')], path[-space // 2:])
                msg += "{0:<{space}}".format(path, space=space)
                if space < 12:
                    msg = ''
                    space = columns - len(msg)
                if space >= 8:
                    if space < len('...') + len(path):
                        path = '%s...%s' % (path[:(space // 2) - len('...')], path[-space // 2:])
                    msg += "{0:<{space}}".format(path, space=space)
            else:
                msg = ' ' * columns
            print(msg, file=stream or sys.stderr, end="\r", flush=True)

@@ -241,6 +278,18 @@

    return keys_dir


def get_security_dir(repository_id=None):
    """Determine where to store local security information."""
    xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config'))
    security_dir = os.environ.get('BORG_SECURITY_DIR', os.path.join(xdg_config, 'borg', 'security'))
    if repository_id:
        security_dir = os.path.join(security_dir, repository_id)
    if not os.path.exists(security_dir):
        os.makedirs(security_dir)
        os.chmod(security_dir, stat.S_IRWXU)
    return security_dir


def get_cache_dir():
    """Determine where to repository keys and cache"""
    xdg_cache = os.environ.get('XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache'))

@@ -578,6 +627,9 @@

        'utcnow': current_time.utcnow(),
        'user': uid2user(os.getuid(), os.getuid()),
        'borgversion': borg_version,
        'borgmajor': '%d' % borg_version_tuple[:1],
        'borgminor': '%d.%d' % borg_version_tuple[:2],
        'borgpatch': '%d.%d.%d' % borg_version_tuple[:3],
    }
    return format_line(text, data)


@@ -660,6 +712,10 @@

    """
    provide a thread-local buffer
    """

    class MemoryLimitExceeded(Error, OSError):
        """Requested buffer size {} is above the limit of {}."""

    def __init__(self, allocator, size=4096, limit=None):
        """
        Initialize the buffer: use allocator(size) call to allocate a buffer.

@@ -679,11 +735,11 @@

        """
        resize the buffer - to avoid frequent reallocation, we usually always grow (if needed).
        giving init=True it is possible to first-time initialize or shrink the buffer.
        if a buffer size beyond the limit is requested, raise ValueError.
        if a buffer size beyond the limit is requested, raise Buffer.MemoryLimitExceeded (OSError).
        """
        size = int(size)
        if self.limit is not None and size > self.limit:
            raise ValueError('Requested buffer size %d is above the limit of %d.' % (size, self.limit))
            raise Buffer.MemoryLimitExceeded(size, self.limit)
        if init or len(self) < size:
            self._thread_local.buffer = self.allocator(size)


@@ -753,24 +809,74 @@

    return s.encode(coding, errors)


def bin_to_hex(binary):
    return hexlify(binary).decode('ascii')


class Location:
    """Object representing a repository / archive location
    """
    proto = user = host = port = path = archive = None

    # user must not contain "@", ":" or "/".
    # Quoting adduser error message:
    # "To avoid problems, the username should consist only of letters, digits,
    # underscores, periods, at signs and dashes, and not start with a dash
    # (as defined by IEEE Std 1003.1-2001)."
    # We use "@" as separator between username and hostname, so we must
    # disallow it within the pure username part.
    optional_user_re = r"""
        (?:(?P<user>[^@:/]+)@)?
    """

    # path must not contain :: (it ends at :: or string end), but may contain single colons.
    # to avoid ambiguities with other regexes, it must also not start with ":" nor with "//" nor with "ssh://".
    path_re = r"""
        (?!(:|//|ssh://))                                   # not starting with ":" or // or ssh://
        (?P<path>([^:]|(:(?!:)))+)                          # any chars, but no "::"
        """
    # abs_path must not contain :: (it ends at :: or string end), but may contain single colons.
    # it must start with a / and that slash is part of the path.
    abs_path_re = r"""
        (?P<path>(/([^:]|(:(?!:)))+))                       # start with /, then any chars, but no "::"
        """

    # optional ::archive_name at the end, archive name must not contain "/".
    # borg mount's FUSE filesystem creates one level of directories from
    # the archive names. Thus, we must not accept "/" in archive names.
    ssh_re = re.compile(r'(?P<proto>ssh)://(?:(?P<user>[^@]+)@)?'
                        r'(?P<host>[^:/#]+)(?::(?P<port>\d+))?'
                        r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
    file_re = re.compile(r'(?P<proto>file)://'
                         r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
    scp_re = re.compile(r'((?:(?P<user>[^@]+)@)?(?P<host>[^:/]+):)?'
                        r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
    # get the repo from BORG_RE env and the optional archive from param.
    # the archive names and of course "/" is not valid in a directory name.
    optional_archive_re = r"""
        (?:
            ::                                              # "::" as separator
            (?P<archive>[^/]+)                              # archive name must not contain "/"
        )?$"""                                              # must match until the end

    # regexes for misc. kinds of supported location specifiers:
    ssh_re = re.compile(r"""
        (?P<proto>ssh)://                                   # ssh://
        """ + optional_user_re + r"""                       # user@  (optional)
        (?P<host>[^:/]+)(?::(?P<port>\d+))?                 # host or host:port
        """ + abs_path_re + optional_archive_re, re.VERBOSE)  # path or path::archive

    file_re = re.compile(r"""
        (?P<proto>file)://                                  # file://
        """ + path_re + optional_archive_re, re.VERBOSE)    # path or path::archive

    # note: scp_re is also use for local pathes
    scp_re = re.compile(r"""
        (
            """ + optional_user_re + r"""                   # user@  (optional)
            (?P<host>[^:/]+):                               # host: (don't match / in host to disambiguate from file:)
        )?                                                  # user@host: part is optional
        """ + path_re + optional_archive_re, re.VERBOSE)    # path with optional archive

    # get the repo from BORG_REPO env and the optional archive from param.
    # if the syntax requires giving REPOSITORY (see "borg mount"),
    # use "::" to let it use the env var.
    # if REPOSITORY argument is optional, it'll automatically use the env.
    env_re = re.compile(r'(?:::(?P<archive>[^/]+)?)?$')
    env_re = re.compile(r"""                                # the repo part is fetched from BORG_REPO
        (?:::$)                                             # just "::" is ok (when a pos. arg is required, no archive)
        |                                                   # or
        """ + optional_archive_re, re.VERBOSE)              # archive name (optional, may be empty)

    def __init__(self, text=''):
        self.orig = text

@@ -795,26 +901,32 @@

        return True

    def _parse(self, text):
        def normpath_special(p):
            # avoid that normpath strips away our relative path hack and even makes p absolute
            relative = p.startswith('/./')
            p = os.path.normpath(p)
            return ('/.' + p) if relative else p

        m = self.ssh_re.match(text)
        if m:
            self.proto = m.group('proto')
            self.user = m.group('user')
            self.host = m.group('host')
            self.port = m.group('port') and int(m.group('port')) or None
            self.path = os.path.normpath(m.group('path'))
            self.path = normpath_special(m.group('path'))
            self.archive = m.group('archive')
            return True
        m = self.file_re.match(text)
        if m:
            self.proto = m.group('proto')
            self.path = os.path.normpath(m.group('path'))
            self.path = normpath_special(m.group('path'))
            self.archive = m.group('archive')
            return True
        m = self.scp_re.match(text)
        if m:
            self.user = m.group('user')
            self.host = m.group('host')
            self.path = os.path.normpath(m.group('path'))
            self.path = normpath_special(m.group('path'))
            self.archive = m.group('archive')
            self.proto = self.host and 'ssh' or 'file'
            return True

@@ -845,9 +957,9 @@

            return self.path
        else:
            if self.path and self.path.startswith('~'):
                path = '/' + self.path
                path = '/' + self.path  # /~/x = path x relative to home dir
            elif self.path and not self.path.startswith('/'):
                path = '/~/' + self.path
                path = '/./' + self.path  # /./x = path x relative to cwd
            else:
                path = self.path
            return 'ssh://{}{}{}{}'.format('{}@'.format(self.user) if self.user else '',

@@ -959,9 +1071,8 @@

        default=False, retry=True, env_var_override=None, ofile=None, input=input):
    """Output <msg> (usually a question) and let user input an answer.
    Qualifies the answer according to falsish, truish and defaultish as True, False or <default>.
    If it didn't qualify and retry_msg is None (no retries wanted),
    return the default [which defaults to False]. Otherwise let user retry
    answering until answer is qualified.
    If it didn't qualify and retry is False (no retries wanted), return the default [which
    defaults to False]. If retry is True let user retry answering until answer is qualified.

    If env_var_override is given and this var is present in the environment, do not ask
    the user, but just use the env var contents as answer as if it was typed in.

@@ -1156,3 +1267,49 @@

1156	1267		except OSError:
1157	1268		pass
1158	1269		return len(s)
	1270	+
	1271	+
	1272	+	class SignalException(BaseException):
	1273	+	"""base class for all signal-based exceptions"""
	1274	+
	1275	+
	1276	+	class SigHup(SignalException):
	1277	+	"""raised on SIGHUP signal"""
	1278	+
	1279	+
	1280	+	class SigTerm(SignalException):
	1281	+	"""raised on SIGTERM signal"""
	1282	+
	1283	+
	1284	+	@contextlib.contextmanager
	1285	+	def signal_handler(sig, handler):
	1286	+	"""
	1287	+	when entering context, set up signal handler <handler> for signal <sig>.
	1288	+	when leaving context, restore original signal handler.
	1289	+
	1290	+	<sig> can bei either a str when giving a signal.SIGXXX attribute name (it
	1291	+	won't crash if the attribute name does not exist as some names are platform
	1292	+	specific) or a int, when giving a signal number.
	1293	+
	1294	+	<handler> is any handler value as accepted by the signal.signal(sig, handler).
	1295	+	"""
	1296	+	if isinstance(sig, str):
	1297	+	sig = getattr(signal, sig, None)
	1298	+	if sig is not None:
	1299	+	orig_handler = signal.signal(sig, handler)
	1300	+	try:
	1301	+	yield
	1302	+	finally:
	1303	+	if sig is not None:
	1304	+	signal.signal(sig, orig_handler)
	1305	+
	1306	+
	1307	+	def raising_signal_handler(exc_cls):
	1308	+	def handler(sig_no, frame):
	1309	+	# setting SIG_IGN avoids that an incoming second signal of this
	1310	+	# kind would raise a 2nd exception while we still process the
	1311	+	# exception handler for exc_cls for the 1st signal.
	1312	+	signal.signal(sig_no, signal.SIG_IGN)
	1313	+	raise exc_cls
	1314	+
	1315	+	return handler

@@ -1159 +1316 @@

@@ -3,14 +3,14 @@

from collections import namedtuple
import os
import stat
from binascii import hexlify, unhexlify
from binascii import unhexlify
import shutil

from .key import PlaintextKey
from .logger import create_logger
logger = create_logger()
from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \
    bigint_to_int, format_file_size, yes
    bigint_to_int, format_file_size, yes, bin_to_hex, Location
from .locking import Lock
from .hashindex import ChunkIndex


@@ -20,8 +20,11 @@

class Cache:
    """Client Side cache
    """
    class RepositoryIDNotUnique(Error):
        """Cache is newer than repository - do you have multiple, independently updated repos with same ID?"""

    class RepositoryReplay(Error):
        """Cache is newer than repository, refusing to continue"""
        """Cache is newer than repository - this is either an attack or unsafe (multiple repos with same ID)"""

    class CacheInitAbortedError(Error):
        """Cache initialization aborted"""

@@ -34,13 +37,13 @@


    @staticmethod
    def break_lock(repository, path=None):
        path = path or os.path.join(get_cache_dir(), hexlify(repository.id).decode('ascii'))
        path = path or os.path.join(get_cache_dir(), bin_to_hex(repository.id))
        Lock(os.path.join(path, 'lock'), exclusive=True).break_lock()

    @staticmethod
    def destroy(repository, path=None):
        """destroy the cache for ``repository`` or at ``path``"""
        path = path or os.path.join(get_cache_dir(), hexlify(repository.id).decode('ascii'))
        path = path or os.path.join(get_cache_dir(), bin_to_hex(repository.id))
        config = os.path.join(path, 'config')
        if os.path.exists(config):
            os.remove(config)  # kill config first

@@ -55,15 +58,16 @@

        self.repository = repository
        self.key = key
        self.manifest = manifest
        self.path = path or os.path.join(get_cache_dir(), hexlify(repository.id).decode('ascii'))
        self.path = path or os.path.join(get_cache_dir(), bin_to_hex(repository.id))
        self.do_files = do_files
        # Warn user before sending data to a never seen before unencrypted repository
        if not os.path.exists(self.path):
            if warn_if_unencrypted and isinstance(key, PlaintextKey):
                msg = ("Warning: Attempting to access a previously unknown unencrypted repository!" +
                       "\n" +
                       "Do you want to continue? [yN] ")
                if not yes(msg, false_msg="Aborting.", env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'):
                if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.",
                           retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'):
                    raise self.CacheInitAbortedError()
            self.create()
        self.open(lock_wait=lock_wait)

@@ -73,13 +77,20 @@

                msg = ("Warning: The repository at location {} was previously located at {}".format(repository._location.canonical_path(), self.previous_location) +
                       "\n" +
                       "Do you want to continue? [yN] ")
                if not yes(msg, false_msg="Aborting.", env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'):
                if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.",
                           retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'):
                    raise self.RepositoryAccessAborted()
                # adapt on-disk config immediately if the new location was accepted
                self.begin_txn()
                self.commit()

            if sync and self.manifest.id != self.manifest_id:
                # If repository is older than the cache something fishy is going on
                if self.timestamp and self.timestamp > manifest.timestamp:
                    raise self.RepositoryReplay()
                    if isinstance(key, PlaintextKey):
                        raise self.RepositoryIDNotUnique()
                    else:
                        raise self.RepositoryReplay()
                # Make sure an encrypted repository has not been swapped for an unencrypted repository
                if self.key_type is not None and self.key_type != str(key.TYPE):
                    raise self.EncryptionMethodMismatch()

@@ -120,7 +131,7 @@

        config = configparser.ConfigParser(interpolation=None)
        config.add_section('cache')
        config.set('cache', 'version', '1')
        config.set('cache', 'repository', hexlify(self.repository.id).decode('ascii'))
        config.set('cache', 'repository', bin_to_hex(self.repository.id))
        config.set('cache', 'manifest', '')
        with open(os.path.join(self.path, 'config'), 'w') as fd:
            config.write(fd)

@@ -129,10 +140,7 @@

        with open(os.path.join(self.path, 'files'), 'wb') as fd:
            pass  # empty file

    def _do_open(self):
        self.config = configparser.ConfigParser(interpolation=None)
        config_path = os.path.join(self.path, 'config')
        self.config.read(config_path)
    def _check_upgrade(self, config_path):
        try:
            cache_version = self.config.getint('cache', 'version')
            wanted_version = 1

@@ -143,6 +151,25 @@

        except configparser.NoSectionError:
            self.close()
            raise Exception('%s does not look like a Borg cache.' % config_path) from None
        # borg < 1.0.8rc1 had different canonicalization for the repo location (see #1655 and #1741).
        cache_loc = self.config.get('cache', 'previous_location', fallback=None)
        if cache_loc:
            repo_loc = self.repository._location.canonical_path()
            rl = Location(repo_loc)
            cl = Location(cache_loc)
            if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \
                    and \
                    cl.path and rl.path and \
                    cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]:
                # everything is same except the expected change in relative path canonicalization,
                # update previous_location to avoid warning / user query about changed location:
                self.config.set('cache', 'previous_location', repo_loc)

    def _do_open(self):
        self.config = configparser.ConfigParser(interpolation=None)
        config_path = os.path.join(self.path, 'config')
        self.config.read(config_path)
        self._check_upgrade(config_path)
        self.id = self.config.get('cache', 'repository')
        self.manifest_id = unhexlify(self.config.get('cache', 'manifest'))
        self.timestamp = self.config.get('cache', 'timestamp', fallback=None)

@@ -164,7 +191,7 @@


    def _read_files(self):
        self.files = {}
        self._newest_mtime = 0
        self._newest_mtime = None
        logger.debug('Reading files cache ...')
        with open(os.path.join(self.path, 'files'), 'rb') as fd:
            u = msgpack.Unpacker(use_list=True)

@@ -195,6 +222,9 @@

        if not self.txn_active:
            return
        if self.files is not None:
            if self._newest_mtime is None:
                # was never set because no files were modified/added
                self._newest_mtime = 2 ** 63 - 1  # nanoseconds, good until y2262
            ttl = int(os.environ.get('BORG_FILES_CACHE_TTL', 20))
            with open(os.path.join(self.path, 'files'), 'wb') as fd:
                for path_hash, item in self.files.items():

@@ -206,7 +236,7 @@

                    if age == 0 and bigint_to_int(item[3]) < self._newest_mtime or \
                       age > 0 and age < ttl:
                        msgpack.pack((path_hash, item), fd)
        self.config.set('cache', 'manifest', hexlify(self.manifest.id).decode('ascii'))
        self.config.set('cache', 'manifest', bin_to_hex(self.manifest.id))
        self.config.set('cache', 'timestamp', self.manifest.timestamp)
        self.config.set('cache', 'key_type', str(self.key.TYPE))
        self.config.set('cache', 'previous_location', self.repository._location.canonical_path())

@@ -249,7 +279,7 @@

        archive_path = os.path.join(self.path, 'chunks.archive.d')

        def mkpath(id, suffix=''):
            id_hex = hexlify(id).decode('ascii')
            id_hex = bin_to_hex(id)
            path = os.path.join(archive_path, id_hex + suffix)
            return path.encode('utf-8')


@@ -424,4 +454,4 @@

424	454		# Entry: Age, inode, size, mtime, chunk ids
425	455		mtime_ns = st.st_mtime_ns
426	456		self.files[path_hash] = msgpack.packb((0, st.st_ino, st.st_size, int_to_bigint(mtime_ns), ids))
427		-	self._newest_mtime = max(self._newest_mtime, mtime_ns)
	457	+	self._newest_mtime = max(self._newest_mtime or 0, mtime_ns)

@@ -428 +458 @@

@@ -4,22 +4,47 @@

import os
import select
import shlex
from subprocess import Popen, PIPE
import sys
import tempfile
import textwrap
import time
from subprocess import Popen, PIPE

from . import __version__

from .helpers import Error, IntegrityError, sysinfo
from .helpers import replace_placeholders
from .helpers import bin_to_hex
from .repository import Repository
from .logger import create_logger

import msgpack

logger = create_logger(__name__)

RPC_PROTOCOL_VERSION = 2

BUFSIZE = 10 * 1024 * 1024

MAX_INFLIGHT = 100


def os_write(fd, data):
    """os.write wrapper so we do not lose data for partial writes."""
    # This is happening frequently on cygwin due to its small pipe buffer size of only 64kiB
    # and also due to its different blocking pipe behaviour compared to Linux/*BSD.
    # Neither Linux nor *BSD ever do partial writes on blocking pipes, unless interrupted by a
    # signal, in which case serve() would terminate.
    amount = remaining = len(data)
    while remaining:
        count = os.write(fd, data)
        remaining -= count
        if not remaining:
            break
        data = data[count:]
        time.sleep(count * 1e-09)
    return amount


class ConnectionClosed(Error):
    """Connection closed by remote host"""

@@ -37,6 +62,23 @@

    """RPC method {} is not valid"""


class UnexpectedRPCDataFormatFromClient(Error):
    """Borg {}: Got unexpected RPC data format from client."""


class UnexpectedRPCDataFormatFromServer(Error):
    """Got unexpected RPC data format from server:\n{}"""

    def __init__(self, data):
        try:
            data = data.decode()[:128]
        except UnicodeDecodeError:
            data = data[:128]
            data = ['%02X' % byte for byte in data]
            data = textwrap.fill(' '.join(data), 16 * 3)
        super().__init__(data)


class RepositoryServer:  # pragma: no cover
    rpc_methods = (
        '__len__',

@@ -79,13 +121,18 @@

            if r:
                data = os.read(stdin_fd, BUFSIZE)
                if not data:
                    self.repository.close()
                    if self.repository is not None:
                        self.repository.close()
                    else:
                        os_write(stderr_fd, "Borg {}: Got connection close before repository was opened.\n"
                                 .format(__version__).encode())
                    return
                unpacker.feed(data)
                for unpacked in unpacker:
                    if not (isinstance(unpacked, tuple) and len(unpacked) == 4):
                        self.repository.close()
                        raise Exception("Unexpected RPC data format.")
                        if self.repository is not None:
                            self.repository.close()
                        raise UnexpectedRPCDataFormatFromClient(__version__)
                    type, msgid, method, args = unpacked
                    method = method.decode('ascii')
                    try:

@@ -104,9 +151,9 @@

                            logging.exception('Borg %s: exception in RPC call:', __version__)
                            logging.error(sysinfo())
                        exc = "Remote Exception (see remote log for the traceback)"
                        os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc)))
                        os_write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc)))
                    else:
                        os.write(stdout_fd, msgpack.packb((1, msgid, None, res)))
                        os_write(stdout_fd, msgpack.packb((1, msgid, None, res)))
            if es:
                self.repository.close()
                return

@@ -116,8 +163,10 @@


    def open(self, path, create=False, lock_wait=None, lock=True, exclusive=None, append_only=False):
        path = os.fsdecode(path)
        if path.startswith('/~'):
        if path.startswith('/~'):  # /~/x = path x relative to home dir, /~username/x = relative to "user" home dir
            path = path[1:]
        elif path.startswith('/./'):  # /./x = path x relative to cwd
            path = path[3:]
        path = os.path.realpath(os.path.expanduser(path))
        if self.restrict_to_paths:
            # if --restrict-to-path P is given, we make sure that we only operate in/below path P.

@@ -168,6 +217,7 @@

            env.pop('LD_LIBRARY_PATH', None)
        env.pop('BORG_PASSPHRASE', None)  # security: do not give secrets to subprocess
        env['BORG_VERSION'] = __version__
        logger.debug('SSH command line: %s', borg_cmd)
        self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
        self.stdin_fd = self.p.stdin.fileno()
        self.stdout_fd = self.p.stdout.fileno()

@@ -301,7 +351,6 @@


        calls = list(calls)
        waiting_for = []
        w_fds = [self.stdin_fd]
        while wait or calls:
            while waiting_for:
                try:

@@ -315,6 +364,10 @@

                            return
                except KeyError:
                    break
            if self.to_send or ((calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT):
                w_fds = [self.stdin_fd]
            else:
                w_fds = []
            r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1)
            if x:
                raise Exception('FD exception occurred')

@@ -326,7 +379,7 @@

                    self.unpacker.feed(data)
                    for unpacked in self.unpacker:
                        if not (isinstance(unpacked, tuple) and len(unpacked) == 4):
                            raise Exception("Unexpected RPC data format.")
                            raise UnexpectedRPCDataFormatFromServer(data)
                        type, msgid, error, res = unpacked
                        if msgid in self.ignore_responses:
                            self.ignore_responses.remove(msgid)

@@ -347,7 +400,7 @@

                        else:
                            sys.stderr.write("Remote: " + line)
            if w:
                while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < 100:
                while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT:
                    if calls:
                        if is_preloaded:
                            if calls[0] in self.cache:

@@ -374,8 +427,6 @@

                        # that the fd should be writable
                        if e.errno != errno.EAGAIN:
                            raise
                if not self.to_send and not (calls or self.preload_ids):
                    w_fds = []
        self.ignore_responses |= set(waiting_for)

    def check(self, repair=False, save_space=False):

@@ -382 +433 @@

@@ -1,5 +1,6 @@

import errno
import os
import subprocess
import sys



@@ -17,14 +18,19 @@

        os.close(fd)


# most POSIX platforms (but not Linux), see also borg 1.1 platform.base
def umount(mountpoint):
    return subprocess.call(['umount', mountpoint])


if sys.platform.startswith('linux'):  # pragma: linux only
    from .platform_linux import acl_get, acl_set, API_VERSION
    from .platform_linux import acl_get, acl_set, umount, API_VERSION
elif sys.platform.startswith('freebsd'):  # pragma: freebsd only
    from .platform_freebsd import acl_get, acl_set, API_VERSION
elif sys.platform == 'darwin':  # pragma: darwin only
    from .platform_darwin import acl_get, acl_set, API_VERSION
else:  # pragma: unknown platform only
    API_VERSION = 2
    API_VERSION = '1.0_01'

    def acl_get(path, item, st, numeric_owner=False):
        pass

@@ -31 +37 @@

@@ -1,37 +1,41 @@

from binascii import hexlify, unhexlify
from binascii import unhexlify
from datetime import datetime
from hashlib import sha256
from operator import attrgetter
import argparse
import faulthandler
import functools
import inspect
import io
import os
import re
import shlex
import signal
import stat
import sys
import textwrap
import time
import traceback
import collections

from . import __version__
from .helpers import Error, location_validator, archivename_validator, format_line, format_time, format_file_size, \
    parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \
    parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, bin_to_hex, \
    get_cache_dir, prune_within, prune_split, \
    Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
    dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
    EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher, ErrorIgnoringTextIOWrapper
from .helpers import signal_handler, raising_signal_handler, SigHup, SigTerm
from .logger import create_logger, setup_logging
logger = create_logger()
from .compress import Compressor
from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
from .repository import Repository
from .cache import Cache
from .key import key_creator, RepoKey, PassphraseKey
from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
from .keymanager import KeyManager
from .archive import backup_io, BackupOSError, Archive, ArchiveChecker, CHUNKER_PARAMS, is_special
from .remote import RepositoryServer, RemoteRepository, cache_if_remote
from .platform import umount

has_lchflags = hasattr(os, 'lchflags')


@@ -45,10 +49,12 @@

    """If bool is passed, return it. If str is passed, retrieve named attribute from args."""
    if isinstance(str_or_bool, str):
        return getattr(args, str_or_bool)
    if isinstance(str_or_bool, (list, tuple)):
        return any(getattr(args, item) for item in str_or_bool)
    return str_or_bool


def with_repository(fake=False, create=False, lock=True, exclusive=False, manifest=True, cache=False):
def with_repository(fake=False, invert_fake=False, create=False, lock=True, exclusive=False, manifest=True, cache=False):
    """
    Method decorator for subcommand-handling methods: do_XYZ(self, args, repository, …)


@@ -65,7 +71,7 @@

        def wrapper(self, args, **kwargs):
            location = args.location  # note: 'location' must be always present in args
            append_only = getattr(args, 'append_only', False)
            if argument(args, fake):
            if argument(args, fake) ^ invert_fake:
                return method(self, args, repository=None, **kwargs)
            elif location.proto == 'ssh':
                repository = RemoteRepository(location, create=create, exclusive=argument(args, exclusive),

@@ -124,14 +130,28 @@

    @with_repository(create=True, exclusive=True, manifest=False)
    def do_init(self, args, repository):
        """Initialize an empty repository"""
        logger.info('Initializing repository at "%s"' % args.location.canonical_path())
        path = args.location.canonical_path()
        logger.info('Initializing repository at "%s"' % path)
        key = key_creator(repository, args)
        manifest = Manifest(key, repository)
        manifest.key = key
        manifest.write()
        repository.commit()
        with Cache(repository, key, manifest, warn_if_unencrypted=False):
            pass
        if key.tam_required:
            tam_file = tam_required_file(repository)
            open(tam_file, 'w').close()
            logger.warning(
                '\n'
                'By default repositories initialized with this version will produce security\n'
                'errors if written to with an older version (up to and including Borg 1.0.8).\n'
                '\n'
                'If you want to use these older versions, you can disable the check by runnning:\n'
                'borg upgrade --disable-tam \'%s\'\n'
                '\n'
                'See https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability '
                'for details about the security implications.', path)
        return self.exit_code

    @with_repository(exclusive=True, manifest=False)

@@ -141,7 +161,8 @@

            msg = ("'check --repair' is an experimental feature that might result in data loss." +
                   "\n" +
                   "Type 'YES' if you understand this and want to continue: ")
            if not yes(msg, false_msg="Aborting.", truish=('YES', ),
            if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.",
                       truish=('YES', ), retry=False,
                       env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'):
                return EXIT_ERROR
        if not args.archives_only:

@@ -156,7 +177,50 @@

156	177		@with_repository()
157	178		def do_change_passphrase(self, args, repository, manifest, key):
158	179		"""Change repository key file passphrase"""
	180	+	if not hasattr(key, 'change_passphrase'):
	181	+	print('This repository is not encrypted, cannot change the passphrase.')
	182	+	return EXIT_ERROR
159	183		key.change_passphrase()
	184	+	logger.info('Key updated')
	185	+	if hasattr(key, 'find_key'):
	186	+	# print key location to make backing it up easier
	187	+	logger.info('Key location: %s', key.find_key())
	188	+	return EXIT_SUCCESS
	189	+
	190	+	@with_repository(lock=False, exclusive=False, manifest=False, cache=False)
	191	+	def do_key_export(self, args, repository):
	192	+	"""Export the repository key for backup"""
	193	+	manager = KeyManager(repository)
	194	+	manager.load_keyblob()
	195	+	if args.paper:
	196	+	manager.export_paperkey(args.path)
	197	+	else:
	198	+	if not args.path:
	199	+	self.print_error("output file to export key to expected")
	200	+	return EXIT_ERROR
	201	+	if args.qr:
	202	+	manager.export_qr(args.path)
	203	+	else:
	204	+	manager.export(args.path)
	205	+	return EXIT_SUCCESS
	206	+
	207	+	@with_repository(lock=False, exclusive=False, manifest=False, cache=False)
	208	+	def do_key_import(self, args, repository):
	209	+	"""Import the repository key from backup"""
	210	+	manager = KeyManager(repository)
	211	+	if args.paper:
	212	+	if args.path:
	213	+	self.print_error("with --paper import from file is not supported")
	214	+	return EXIT_ERROR
	215	+	manager.import_paperkey(args)
	216	+	else:
	217	+	if not args.path:
	218	+	self.print_error("input file to import key from expected")
	219	+	return EXIT_ERROR
	220	+	if not os.path.exists(args.path):
	221	+	self.print_error("input file does not exist: " + args.path)
	222	+	return EXIT_ERROR
	223	+	manager.import_keyfile(args)
160	224		return EXIT_SUCCESS
161	225
162	226		@with_repository(manifest=False)

@@ -172,6 +236,7 @@

        key_new.id_key = key_old.id_key
        key_new.chunk_seed = key_old.chunk_seed
        key_new.change_passphrase()  # option to change key protection passphrase, save
        logger.info('Key updated')
        return EXIT_SUCCESS

    @with_repository(fake='dry_run', exclusive=True)

@@ -226,7 +291,6 @@

                if args.progress:
                    archive.stats.show_progress(final=True)
                if args.stats:
                    archive.end = datetime.utcnow()
                    log_multi(DASHES,
                              str(archive),
                              DASHES,

@@ -239,13 +303,15 @@

        self.ignore_inode = args.ignore_inode
        dry_run = args.dry_run
        t0 = datetime.utcnow()
        t0_monotonic = time.monotonic()
        if not dry_run:
            key.compressor = Compressor(**args.compression)
            with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache:
                archive = Archive(repository, key, manifest, args.location.archive, cache=cache,
                                  create=True, checkpoint_interval=args.checkpoint_interval,
                                  numeric_owner=args.numeric_owner, progress=args.progress,
                                  chunker_params=args.chunker_params, start=t0)
                                  numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime,
                                  progress=args.progress,
                                  chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic)
                create_inner(archive, cache)
        else:
            create_inner(None, None)

@@ -306,8 +372,13 @@

                if not read_special:
                    status = archive.process_symlink(path, st)
                else:
                    st_target = os.stat(path)
                    if is_special(st_target.st_mode):
                    try:
                        st_target = os.stat(path)
                    except OSError:
                        special = False
                    else:
                        special = is_special(st_target.st_mode)
                    if special:
                        status = archive.process_file(path, st_target, cache)
                    else:
                        status = archive.process_symlink(path, st)

@@ -461,8 +532,8 @@

                        msg.append(format_archive(archive_info))
                msg.append("Type 'YES' if you understand this and want to continue: ")
                msg = '\n'.join(msg)
                if not yes(msg, false_msg="Aborting.", truish=('YES', ),
                           env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'):
                if not yes(msg, false_msg="Aborting.", invalid_msg='Invalid answer, aborting.', truish=('YES', ),
                           retry=False, env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'):
                    self.exit_code = EXIT_ERROR
                    return self.exit_code
                repository.destroy()

@@ -473,7 +544,7 @@


    @with_repository()
    def do_mount(self, args, repository, manifest, key):
        """Mount archive or an entire repository as a FUSE fileystem"""
        """Mount archive or an entire repository as a FUSE filesystem"""
        try:
            from .fuse import FuseOperations
        except ImportError as e:

@@ -498,6 +569,10 @@

                self.exit_code = EXIT_ERROR
        return self.exit_code

    def do_umount(self, args):
        """un-mount the FUSE filesystem"""
        return umount(args.mountpoint)

    @with_repository()
    def do_list(self, args, repository, manifest, key):
        """List archive or repository contents"""

@@ -590,7 +665,7 @@

        """Show archive details such as disk space used"""
        stats = archive.calc_stats(cache)
        print('Name:', archive.name)
        print('Fingerprint: %s' % hexlify(archive.id).decode('ascii'))
        print('Fingerprint: %s' % bin_to_hex(archive.id))
        print('Hostname:', archive.metadata[b'hostname'])
        print('Username:', archive.metadata[b'username'])
        print('Time (start): %s' % format_time(to_localtime(archive.ts)))

@@ -658,21 +733,61 @@

                          DASHES)
        return self.exit_code

    def do_upgrade(self, args):
    @with_repository(fake=('tam', 'disable_tam'), invert_fake=True, manifest=False, exclusive=True)
    def do_upgrade(self, args, repository, manifest=None, key=None):
        """upgrade a repository from a previous version"""
        # mainly for upgrades from Attic repositories,
        # but also supports borg 0.xx -> 1.0 upgrade.
        if args.tam:
            manifest, key = Manifest.load(repository, force_tam_not_required=args.force)

        repo = AtticRepositoryUpgrader(args.location.path, create=False)
        try:
            repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
        except NotImplementedError as e:
            print("warning: %s" % e)
        repo = BorgRepositoryUpgrader(args.location.path, create=False)
        try:
            repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
        except NotImplementedError as e:
            print("warning: %s" % e)
            if not hasattr(key, 'change_passphrase'):
                print('This repository is not encrypted, cannot enable TAM.')
                return EXIT_ERROR

            if not manifest.tam_verified or not manifest.config.get(b'tam_required', False):
                # The standard archive listing doesn't include the archive ID like in borg 1.1.x
                print('Manifest contents:')
                for archive_info in manifest.list_archive_infos(sort_by='ts'):
                    print(format_archive(archive_info), '[%s]' % bin_to_hex(archive_info.id))
                manifest.config[b'tam_required'] = True
                manifest.write()
                repository.commit()
            if not key.tam_required:
                key.tam_required = True
                key.change_passphrase(key._passphrase)
                print('Key updated')
                if hasattr(key, 'find_key'):
                    print('Key location:', key.find_key())
            if not tam_required(repository):
                tam_file = tam_required_file(repository)
                open(tam_file, 'w').close()
                print('Updated security database')
        elif args.disable_tam:
            manifest, key = Manifest.load(repository, force_tam_not_required=True)
            if tam_required(repository):
                os.unlink(tam_required_file(repository))
            if key.tam_required:
                key.tam_required = False
                key.change_passphrase(key._passphrase)
                print('Key updated')
                if hasattr(key, 'find_key'):
                    print('Key location:', key.find_key())
            manifest.config[b'tam_required'] = False
            manifest.write()
            repository.commit()
        else:
            # mainly for upgrades from Attic repositories,
            # but also supports borg 0.xx -> 1.0 upgrade.

            repo = AtticRepositoryUpgrader(args.location.path, create=False)
            try:
                repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
            except NotImplementedError as e:
                print("warning: %s" % e)
            repo = BorgRepositoryUpgrader(args.location.path, create=False)
            try:
                repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
            except NotImplementedError as e:
                print("warning: %s" % e)
        return self.exit_code

    def do_debug_info(self, args):

@@ -686,7 +801,7 @@

        archive = Archive(repository, key, manifest, args.location.archive)
        for i, item_id in enumerate(archive.metadata[b'items']):
            data = key.decrypt(item_id, repository.get(item_id))
            filename = '%06d_%s.items' % (i, hexlify(item_id).decode('ascii'))
            filename = '%06d_%s.items' % (i, bin_to_hex(item_id))
            print('Dumping', filename)
            with open(filename, 'wb') as fd:
                fd.write(data)

@@ -707,7 +822,7 @@

                cdata = repository.get(id)
                give_id = id if id != Manifest.MANIFEST_ID else None
                data = key.decrypt(give_id, cdata)
                filename = '%06d_%s.obj' % (i, hexlify(id).decode('ascii'))
                filename = '%06d_%s.obj' % (i, bin_to_hex(id))
                print('Dumping', filename)
                with open(filename, 'wb') as fd:
                    fd.write(data)

@@ -726,7 +841,7 @@

        else:
            try:
                data = repository.get(id)
            except repository.ObjectNotFound:
            except Repository.ObjectNotFound:
                print("object %s not found." % hex_id)
            else:
                with open(args.path, "wb") as f:

@@ -760,13 +875,29 @@

                    repository.delete(id)
                    modified = True
                    print("object %s deleted." % hex_id)
                except repository.ObjectNotFound:
                except Repository.ObjectNotFound:
                    print("object %s not found." % hex_id)
        if modified:
            repository.commit()
        print('Done.')
        return EXIT_SUCCESS

    @with_repository(manifest=False, exclusive=True, cache=True)
    def do_debug_refcount_obj(self, args, repository, manifest, key, cache):
        """display refcounts for the objects with the given IDs"""
        for hex_id in args.ids:
            try:
                id = unhexlify(hex_id)
            except ValueError:
                print("object id %s is invalid." % hex_id)
            else:
                try:
                    refcount = cache.chunks[id][0]
                    print("object %s has %d referrers [info from chunks cache]." % (hex_id, refcount))
                except KeyError:
                    print("object %s not found [info from chunks cache]." % hex_id)
        return EXIT_SUCCESS

    @with_repository(lock=False, manifest=False)
    def do_break_lock(self, args, repository):
        """Break the repository lock (e.g. in case it was left by a dead borg."""

@@ -892,7 +1023,19 @@


        {borgversion}

            The version of borg.
            The version of borg, e.g.: 1.0.8rc1

        {borgmajor}

            The version of borg, only the major version, e.g.: 1

        {borgminor}

            The version of borg, only major and minor version, e.g.: 1.0

        {borgpatch}

            The version of borg, only major, minor and patch version, e.g.: 1.0.8

       Examples::


@@ -917,6 +1060,11 @@

            parser.error('No help available on %s' % (args.topic,))
        return self.exit_code

    def do_subcommand_help(self, parser, args):
        """display infos about subcommand"""
        parser.print_help()
        return EXIT_SUCCESS

    def preprocess_args(self, args):
        deprecations = [
            # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'),

@@ -970,7 +1118,9 @@

                                          help='start repository server process')
        subparser.set_defaults(func=self.do_serve)
        subparser.add_argument('--restrict-to-path', dest='restrict_to_paths', action='append',
                               metavar='PATH', help='restrict repository access to PATH')
                               metavar='PATH', help='restrict repository access to PATH. '
                                                    'Can be specified multiple times to allow the client access to several directories. '
                                                    'Access to all sub-directories is granted implicitly; PATH doesn\'t need to directly point to a repository.')
        subparser.add_argument('--append-only', dest='append_only', action='store_true',
                               help='only allow appending to repository segment files')
        init_epilog = textwrap.dedent("""

@@ -1071,6 +1221,73 @@

1071	1221		subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
1072	1222		type=location_validator(archive=False))
1073	1223
	1224	+	subparser = subparsers.add_parser('key', parents=[common_parser],
	1225	+	description="Manage a keyfile or repokey of a repository",
	1226	+	epilog="",
	1227	+	formatter_class=argparse.RawDescriptionHelpFormatter,
	1228	+	help='manage repository key')
	1229	+
	1230	+	key_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
	1231	+	subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser))
	1232	+
	1233	+	key_export_epilog = textwrap.dedent("""
	1234	+	If repository encryption is used, the repository is inaccessible
	1235	+	without the key. This command allows to backup this essential key.
	1236	+
	1237	+	There are two backup formats. The normal backup format is suitable for
	1238	+	digital storage as a file. The ``--paper`` backup format is optimized
	1239	+	for printing and typing in while importing, with per line checks to
	1240	+	reduce problems with manual input.
	1241	+
	1242	+	For repositories using keyfile encryption the key is saved locally
	1243	+	on the system that is capable of doing backups. To guard against loss
	1244	+	of this key, the key needs to be backed up independently of the main
	1245	+	data backup.
	1246	+
	1247	+	For repositories using the repokey encryption the key is saved in the
	1248	+	repository in the config file. A backup is thus not strictly needed,
	1249	+	but guards against the repository becoming inaccessible if the file
	1250	+	is damaged for some reason.
	1251	+	""")
	1252	+	subparser = key_parsers.add_parser('export', parents=[common_parser],
	1253	+	description=self.do_key_export.__doc__,
	1254	+	epilog=key_export_epilog,
	1255	+	formatter_class=argparse.RawDescriptionHelpFormatter,
	1256	+	help='export repository key for backup')
	1257	+	subparser.set_defaults(func=self.do_key_export)
	1258	+	subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
	1259	+	type=location_validator(archive=False))
	1260	+	subparser.add_argument('path', metavar='PATH', nargs='?', type=str,
	1261	+	help='where to store the backup')
	1262	+	subparser.add_argument('--paper', dest='paper', action='store_true',
	1263	+	default=False,
	1264	+	help='Create an export suitable for printing and later type-in')
	1265	+	subparser.add_argument('--qr-html', dest='qr', action='store_true',
	1266	+	default=False,
	1267	+	help='Create an html file suitable for printing and later type-in or qr scan')
	1268	+
	1269	+	key_import_epilog = textwrap.dedent("""
	1270	+	This command allows to restore a key previously backed up with the
	1271	+	export command.
	1272	+
	1273	+	If the ``--paper`` option is given, the import will be an interactive
	1274	+	process in which each line is checked for plausibility before
	1275	+	proceeding to the next line. For this format PATH must not be given.
	1276	+	""")
	1277	+	subparser = key_parsers.add_parser('import', parents=[common_parser],
	1278	+	description=self.do_key_import.__doc__,
	1279	+	epilog=key_import_epilog,
	1280	+	formatter_class=argparse.RawDescriptionHelpFormatter,
	1281	+	help='import repository key from backup')
	1282	+	subparser.set_defaults(func=self.do_key_import)
	1283	+	subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
	1284	+	type=location_validator(archive=False))
	1285	+	subparser.add_argument('path', metavar='PATH', nargs='?', type=str,
	1286	+	help='path to the backup')
	1287	+	subparser.add_argument('--paper', dest='paper', action='store_true',
	1288	+	default=False,
	1289	+	help='interactively import from a backup done with --paper')
	1290	+
1074	1291		migrate_to_repokey_epilog = textwrap.dedent("""
1075	1292		This command migrates a repository from passphrase mode (not supported any
1076	1293		more) to repokey mode.

@@ -1100,15 +1317,19 @@


        create_epilog = textwrap.dedent("""
        This command creates a backup archive containing all files found while recursively
        traversing all paths specified. The archive will consume almost no disk space for
        files or parts of files that have already been stored in other archives.
        traversing all paths specified. When giving '-' as path, borg will read data
        from standard input and create a file 'stdin' in the created archive from that
        data.

        The archive will consume almost no disk space for files or parts of files that
        have already been stored in other archives.

        The archive name needs to be unique. It must not end in '.checkpoint' or
        '.checkpoint.N' (with N being a number), because these names are used for
        checkpoints and treated in special ways.

        In the archive name, you may use the following format tags:
        {now}, {utcnow}, {fqdn}, {hostname}, {user}, {pid}, {borgversion}
        In the archive name, you may use the following placeholders:
        {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others.

        To speed up pulling backups over sshfs and similar network file systems which do
        not provide correct inode information the --ignore-inode flag can be used. This

@@ -1158,10 +1379,16 @@

                               help='write checkpoint every SECONDS seconds (Default: 300)')
        subparser.add_argument('-x', '--one-file-system', dest='one_file_system',
                               action='store_true', default=False,
                               help='stay in same file system, do not cross mount points')
                               help='stay in same file system')
        subparser.add_argument('--numeric-owner', dest='numeric_owner',
                               action='store_true', default=False,
                               help='only store numeric user and group identifiers')
        subparser.add_argument('--noatime', dest='noatime',
                               action='store_true', default=False,
                               help='do not store atime into archive')
        subparser.add_argument('--noctime', dest='noctime',
                               action='store_true', default=False,
                               help='do not store ctime into archive')
        subparser.add_argument('--timestamp', dest='timestamp',
                               type=timestamp, default=None,
                               metavar='yyyy-mm-ddThh:mm:ss',

@@ -1325,6 +1552,13 @@

        - allow_damaged_files: by default damaged files (where missing chunks were
          replaced with runs of zeros by borg check --repair) are not readable and
          return EIO (I/O error). Set this option to read such files.

        When the daemonized process receives a signal or crashes, it does not unmount.
        Unmounting in these cases could cause an active rsync or similar process
        to unintentionally delete data.

        When running in the foreground ^C/SIGINT unmounts cleanly, but other
        signals or crashes do not.
        """)
        subparser = subparsers.add_parser('mount', parents=[common_parser],
                                          description=self.do_mount.__doc__,

@@ -1342,8 +1576,32 @@

        subparser.add_argument('-o', dest='options', type=str,
                               help='Extra mount options')

        umount_epilog = textwrap.dedent("""
        This command un-mounts a FUSE filesystem that was mounted with ``borg mount``.

        This is a convenience wrapper that just calls the platform-specific shell
        command - usually this is either umount or fusermount -u.
        """)
        subparser = subparsers.add_parser('umount', parents=[common_parser],
                                          description=self.do_umount.__doc__,
                                          epilog=umount_epilog,
                                          formatter_class=argparse.RawDescriptionHelpFormatter,
                                          help='umount repository')
        subparser.set_defaults(func=self.do_umount)
        subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str,
                               help='mountpoint of the filesystem to umount')

        info_epilog = textwrap.dedent("""
        This command displays some detailed information about the specified archive.

        Please note that the deduplicated sizes of the individual archives do not add
        up to the deduplicated size of the repository ("all archives"), because the two
        are meaning different things:

        This archive / deduplicated size = amount of data stored ONLY for this archive
                                         = unique chunks of this archive.
        All archives / deduplicated size = amount of data stored in the repo
                                         = all chunks in the repository.
        """)
        subparser = subparsers.add_parser('info', parents=[common_parser],
                                          description=self.do_info.__doc__,

@@ -1393,6 +1651,8 @@

        considered for deletion and only those archives count towards the totals
        specified by the rules.
        Otherwise, *all* archives in the repository are candidates for deletion!
        There is no automatic distinction between archives representing different
        contents. These need to be distinguished by specifying matching prefixes.
        """)
        subparser = subparsers.add_parser('prune', parents=[common_parser],
                                          description=self.do_prune.__doc__,

@@ -1435,6 +1695,32 @@

1435	1695
1436	1696		upgrade_epilog = textwrap.dedent("""
1437	1697		Upgrade an existing Borg repository.
	1698	+
	1699	+	Borg 1.x.y upgrades
	1700	+	+++++++++++++++++++
	1701	+
	1702	+	Use ``borg upgrade --tam REPO`` to require manifest authentication
	1703	+	introduced with Borg 1.0.9 to address security issues. This means
	1704	+	that modifying the repository after doing this with a version prior
	1705	+	to 1.0.9 will raise a validation error, so only perform this upgrade
	1706	+	after updating all clients using the repository to 1.0.9 or newer.
	1707	+
	1708	+	This upgrade should be done on each client for safety reasons.
	1709	+
	1710	+	If a repository is accidentally modified with a pre-1.0.9 client after
	1711	+	this upgrade, use ``borg upgrade --tam --force REPO`` to remedy it.
	1712	+
	1713	+	If you routinely do this you might not want to enable this upgrade
	1714	+	(which will leave you exposed to the security issue). You can
	1715	+	reverse the upgrade by issuing ``borg upgrade --disable-tam REPO``.
	1716	+
	1717	+	See
	1718	+	https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability
	1719	+	for details.
	1720	+
	1721	+	Attic and Borg 0.xx to Borg 1.x
	1722	+	+++++++++++++++++++++++++++++++
	1723	+
1438	1724		This currently supports converting an Attic repository to Borg and also
1439	1725		helps with converting Borg 0.xx to 1.0.
1440	1726

@@ -1487,6 +1773,12 @@

                               default=False, action='store_true',
                               help="""rewrite repository in place, with no chance of going back to older
                               versions of the repository.""")
        subparser.add_argument('--force', dest='force', action='store_true',
                               help="""Force upgrade""")
        subparser.add_argument('--tam', dest='tam', action='store_true',
                               help="""Enable manifest authentication (in key and cache) (Borg 1.0.9 and later)""")
        subparser.add_argument('--disable-tam', dest='disable_tam', action='store_true',
                               help="""Disable manifest authentication (in key and cache)""")
        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                               type=location_validator(archive=False),
                               help='path to the repository to be upgraded')

@@ -1501,6 +1793,23 @@

        subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?',
                               help='additional help on TOPIC')

        debug_epilog = textwrap.dedent("""
        These commands are not intended for normal use and potentially very
        dangerous if used incorrectly.

        They exist to improve debugging capabilities without direct system access, e.g.
        in case you ever run into some severe malfunction. Use them only if you know
        what you are doing or if a trusted developer tells you what to do.""")

        subparser = subparsers.add_parser('debug', parents=[common_parser],
                                          description='debugging command (not intended for normal use)',
                                          epilog=debug_epilog,
                                          formatter_class=argparse.RawDescriptionHelpFormatter,
                                          help='debugging command (not intended for normal use)')

        debug_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
        subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser))

        debug_info_epilog = textwrap.dedent("""
        This command displays some system information that might be useful for bug
        reports and debugging problems. If a traceback happens, this information is

@@ -1513,6 +1822,13 @@

                                          help='show system infos for debugging / bug reports (debug)')
        subparser.set_defaults(func=self.do_debug_info)

        subparser = debug_parsers.add_parser('info', parents=[common_parser],
                                          description=self.do_debug_info.__doc__,
                                          epilog=debug_info_epilog,
                                          formatter_class=argparse.RawDescriptionHelpFormatter,
                                          help='show system infos for debugging / bug reports (debug)')
        subparser.set_defaults(func=self.do_debug_info)

        debug_dump_archive_items_epilog = textwrap.dedent("""
        This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files.
        """)

@@ -1526,6 +1842,16 @@

                               type=location_validator(archive=True),
                               help='archive to dump')

        subparser = debug_parsers.add_parser('dump-archive-items', parents=[common_parser],
                                          description=self.do_debug_dump_archive_items.__doc__,
                                          epilog=debug_dump_archive_items_epilog,
                                          formatter_class=argparse.RawDescriptionHelpFormatter,
                                          help='dump archive items (metadata) (debug)')
        subparser.set_defaults(func=self.do_debug_dump_archive_items)
        subparser.add_argument('location', metavar='ARCHIVE',
                               type=location_validator(archive=True),
                               help='archive to dump')

        debug_dump_repo_objs_epilog = textwrap.dedent("""
        This command dumps raw (but decrypted and decompressed) repo objects to files.
        """)

@@ -1539,6 +1865,16 @@

                               type=location_validator(archive=False),
                               help='repo to dump')

        subparser = debug_parsers.add_parser('dump-repo-objs', parents=[common_parser],
                                          description=self.do_debug_dump_repo_objs.__doc__,
                                          epilog=debug_dump_repo_objs_epilog,
                                          formatter_class=argparse.RawDescriptionHelpFormatter,
                                          help='dump repo objects (debug)')
        subparser.set_defaults(func=self.do_debug_dump_repo_objs)
        subparser.add_argument('location', metavar='REPOSITORY',
                               type=location_validator(archive=False),
                               help='repo to dump')

        debug_get_obj_epilog = textwrap.dedent("""
        This command gets an object from the repository.
        """)

@@ -1556,6 +1892,20 @@

        subparser.add_argument('path', metavar='PATH', type=str,
                               help='file to write object data into')

        subparser = debug_parsers.add_parser('get-obj', parents=[common_parser],
                                          description=self.do_debug_get_obj.__doc__,
                                          epilog=debug_get_obj_epilog,
                                          formatter_class=argparse.RawDescriptionHelpFormatter,
                                          help='get object from repository (debug)')
        subparser.set_defaults(func=self.do_debug_get_obj)
        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                               type=location_validator(archive=False),
                               help='repository to use')
        subparser.add_argument('id', metavar='ID', type=str,
                               help='hex object ID to get from the repo')
        subparser.add_argument('path', metavar='PATH', type=str,
                               help='file to write object data into')

        debug_put_obj_epilog = textwrap.dedent("""
        This command puts objects into the repository.
        """)

@@ -1571,6 +1921,18 @@

        subparser.add_argument('paths', metavar='PATH', nargs='+', type=str,
                               help='file(s) to read and create object(s) from')

        subparser = debug_parsers.add_parser('put-obj', parents=[common_parser],
                                          description=self.do_debug_put_obj.__doc__,
                                          epilog=debug_put_obj_epilog,
                                          formatter_class=argparse.RawDescriptionHelpFormatter,
                                          help='put object to repository (debug)')
        subparser.set_defaults(func=self.do_debug_put_obj)
        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                               type=location_validator(archive=False),
                               help='repository to use')
        subparser.add_argument('paths', metavar='PATH', nargs='+', type=str,
                               help='file(s) to read and create object(s) from')

        debug_delete_obj_epilog = textwrap.dedent("""
        This command deletes objects from the repository.
        """)

@@ -1585,6 +1947,46 @@

1585	1947		help='repository to use')
1586	1948		subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
1587	1949		help='hex object ID(s) to delete from the repo')
	1950	+
	1951	+	subparser = debug_parsers.add_parser('delete-obj', parents=[common_parser],
	1952	+	description=self.do_debug_delete_obj.__doc__,
	1953	+	epilog=debug_delete_obj_epilog,
	1954	+	formatter_class=argparse.RawDescriptionHelpFormatter,
	1955	+	help='delete object from repository (debug)')
	1956	+	subparser.set_defaults(func=self.do_debug_delete_obj)
	1957	+	subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
	1958	+	type=location_validator(archive=False),
	1959	+	help='repository to use')
	1960	+	subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
	1961	+	help='hex object ID(s) to delete from the repo')
	1962	+
	1963	+	debug_refcount_obj_epilog = textwrap.dedent("""
	1964	+	This command displays the reference count for objects from the repository.
	1965	+	""")
	1966	+	subparser = subparsers.add_parser('debug-refcount-obj', parents=[common_parser],
	1967	+	description=self.do_debug_refcount_obj.__doc__,
	1968	+	epilog=debug_refcount_obj_epilog,
	1969	+	formatter_class=argparse.RawDescriptionHelpFormatter,
	1970	+	help='show refcount for object from repository (debug)')
	1971	+	subparser.set_defaults(func=self.do_debug_refcount_obj)
	1972	+	subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
	1973	+	type=location_validator(archive=False),
	1974	+	help='repository to use')
	1975	+	subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
	1976	+	help='hex object ID(s) to show refcounts for')
	1977	+
	1978	+	subparser = debug_parsers.add_parser('refcount-obj', parents=[common_parser],
	1979	+	description=self.do_debug_refcount_obj.__doc__,
	1980	+	epilog=debug_refcount_obj_epilog,
	1981	+	formatter_class=argparse.RawDescriptionHelpFormatter,
	1982	+	help='show refcount for object from repository (debug)')
	1983	+	subparser.set_defaults(func=self.do_debug_refcount_obj)
	1984	+	subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
	1985	+	type=location_validator(archive=False),
	1986	+	help='repository to use')
	1987	+	subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
	1988	+	help='hex object ID(s) to show refcounts for')
	1989	+
1588	1990		return parser
1589	1991
1590	1992		def get_args(self, argv, cmd):

@@ -1614,105 +2016,111 @@

    def run(self, args):
        os.umask(args.umask)  # early, before opening files
        self.lock_wait = args.lock_wait
        setup_logging(level=args.log_level, is_serve=args.func == self.do_serve)  # do not use loggers before this!
        # This works around http://bugs.python.org/issue9351
        func = getattr(args, 'func', None) or getattr(args, 'fallback_func')
        setup_logging(level=args.log_level, is_serve=func == self.do_serve)  # do not use loggers before this!
        check_extension_modules()
        if is_slow_msgpack():
            logger.warning("Using a pure-python msgpack! This will result in lower performance.")
        return args.func(args)
        return func(args)


def sig_info_handler(signum, stack):  # pragma: no cover
def sig_info_handler(sig_no, stack):  # pragma: no cover
    """search the stack for infos about the currently processed file and print them"""
    for frame in inspect.getouterframes(stack):
        func, loc = frame[3], frame[0].f_locals
        if func in ('process_file', '_process', ):  # create op
            path = loc['path']
            try:
                pos = loc['fd'].tell()
                total = loc['st'].st_size
            except Exception:
                pos, total = 0, 0
            logger.info("{0} {1}/{2}".format(path, format_file_size(pos), format_file_size(total)))
            break
        if func in ('extract_item', ):  # extract op
            path = loc['item'][b'path']
            try:
                pos = loc['fd'].tell()
            except Exception:
                pos = 0
            logger.info("{0} {1}/???".format(path, format_file_size(pos)))
            break


class SIGTERMReceived(BaseException):
    pass


def sig_term_handler(signum, stack):
    raise SIGTERMReceived
    with signal_handler(sig_no, signal.SIG_IGN):
        for frame in inspect.getouterframes(stack):
            func, loc = frame[3], frame[0].f_locals
            if func in ('process_file', '_process', ):  # create op
                path = loc['path']
                try:
                    pos = loc['fd'].tell()
                    total = loc['st'].st_size
                except Exception:
                    pos, total = 0, 0
                logger.info("{0} {1}/{2}".format(path, format_file_size(pos), format_file_size(total)))
                break
            if func in ('extract_item', ):  # extract op
                path = loc['item'][b'path']
                try:
                    pos = loc['fd'].tell()
                except Exception:
                    pos = 0
                logger.info("{0} {1}/???".format(path, format_file_size(pos)))
                break


def setup_signal_handlers():  # pragma: no cover
    sigs = []
    if hasattr(signal, 'SIGUSR1'):
        sigs.append(signal.SIGUSR1)  # kill -USR1 pid
    if hasattr(signal, 'SIGINFO'):
        sigs.append(signal.SIGINFO)  # kill -INFO pid (or ctrl-t)
    for sig in sigs:
        signal.signal(sig, sig_info_handler)
    signal.signal(signal.SIGTERM, sig_term_handler)
def sig_trace_handler(sig_no, stack):  # pragma: no cover
    print('\nReceived SIGUSR2 at %s, dumping trace...' % datetime.now().replace(microsecond=0), file=sys.stderr)
    faulthandler.dump_traceback()


def main():  # pragma: no cover
    # Make sure stdout and stderr have errors='replace') to avoid unicode
    # issues when print()-ing unicode file names
    sys.stdout = ErrorIgnoringTextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True)
    sys.stderr = ErrorIgnoringTextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True)
    setup_signal_handlers()
    archiver = Archiver()
    msg = None
    try:
        args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
    except Error as e:
        msg = e.get_message()
        if e.traceback:
            msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo())
        # we might not have logging setup yet, so get out quickly
        print(msg, file=sys.stderr)
        sys.exit(e.exit_code)
    try:
        exit_code = archiver.run(args)
    except Error as e:
        msg = e.get_message()
        if e.traceback:
            msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo())
        exit_code = e.exit_code
    except RemoteRepository.RPCError as e:
        msg = '%s\n%s' % (str(e), sysinfo())
        exit_code = EXIT_ERROR
    except Exception:
        msg = 'Local Exception.\n%s\n%s' % (traceback.format_exc(), sysinfo())
        exit_code = EXIT_ERROR
    except KeyboardInterrupt:
        msg = 'Keyboard interrupt.\n%s\n%s' % (traceback.format_exc(), sysinfo())
        exit_code = EXIT_ERROR
    except SIGTERMReceived:
        msg = 'Received SIGTERM.'
        exit_code = EXIT_ERROR
    if msg:
        logger.error(msg)
    if args.show_rc:
        exit_msg = 'terminating with %s status, rc %d'
        if exit_code == EXIT_SUCCESS:
            logger.info(exit_msg % ('success', exit_code))
        elif exit_code == EXIT_WARNING:
            logger.warning(exit_msg % ('warning', exit_code))
        elif exit_code == EXIT_ERROR:
            logger.error(exit_msg % ('error', exit_code))
        else:
            # if you see 666 in output, it usually means exit_code was None
            logger.error(exit_msg % ('abnormal', exit_code or 666))
    sys.exit(exit_code)
    # If we receive SIGINT (ctrl-c), SIGTERM (kill) or SIGHUP (kill -HUP),
    # catch them and raise a proper exception that can be handled for an
    # orderly exit.
    # SIGHUP is important especially for systemd systems, where logind
    # sends it when a session exits, in addition to any traditional use.
    # Output some info if we receive SIGUSR1 or SIGINFO (ctrl-t).

    # Register fault handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL.
    faulthandler.enable()
    with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \
         signal_handler('SIGHUP', raising_signal_handler(SigHup)), \
         signal_handler('SIGTERM', raising_signal_handler(SigTerm)), \
         signal_handler('SIGUSR1', sig_info_handler), \
         signal_handler('SIGUSR2', sig_trace_handler), \
         signal_handler('SIGINFO', sig_info_handler):
        archiver = Archiver()
        msg = None
        try:
            args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
        except Error as e:
            msg = e.get_message()
            if e.traceback:
                msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo())
            # we might not have logging setup yet, so get out quickly
            print(msg, file=sys.stderr)
            sys.exit(e.exit_code)
        try:
            exit_code = archiver.run(args)
        except Error as e:
            msg = e.get_message()
            if e.traceback:
                msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo())
            exit_code = e.exit_code
        except RemoteRepository.RPCError as e:
            msg = '%s\n%s' % (str(e), sysinfo())
            exit_code = EXIT_ERROR
        except Exception:
            msg = 'Local Exception.\n%s\n%s' % (traceback.format_exc(), sysinfo())
            exit_code = EXIT_ERROR
        except KeyboardInterrupt:
            msg = 'Keyboard interrupt.\n%s\n%s' % (traceback.format_exc(), sysinfo())
            exit_code = EXIT_ERROR
        except SigTerm:
            msg = 'Received SIGTERM.'
            exit_code = EXIT_ERROR
        except SigHup:
            msg = 'Received SIGHUP.'
            exit_code = EXIT_ERROR
        if msg:
            logger.error(msg)
        if args.show_rc:
            exit_msg = 'terminating with %s status, rc %d'
            if exit_code == EXIT_SUCCESS:
                logger.info(exit_msg % ('success', exit_code))
            elif exit_code == EXIT_WARNING:
                logger.warning(exit_msg % ('warning', exit_code))
            elif exit_code == EXIT_ERROR:
                logger.error(exit_msg % ('error', exit_code))
            else:
                # if you see 666 in output, it usually means exit_code was None
                logger.error(exit_msg % ('abnormal', exit_code or 666))
        sys.exit(exit_code)


if __name__ == '__main__':

@@ -1719 +2127 @@

@@ -1,18 +1,19 @@

from configparser import ConfigParser
from binascii import hexlify, unhexlify
from binascii import unhexlify
from datetime import datetime
from itertools import islice
import errno
import logging
logger = logging.getLogger(__name__)

import os
import shutil
import struct
from zlib import crc32

import msgpack
from .helpers import Error, ErrorWithTraceback, IntegrityError, Location, ProgressIndicatorPercent

from .logger import create_logger
logger = create_logger()

from .helpers import Error, ErrorWithTraceback, IntegrityError, Location, ProgressIndicatorPercent, bin_to_hex
from .hashindex import NSIndex
from .locking import Lock, LockError, LockErrorT
from .lrucache import LRUCache

@@ -109,7 +110,7 @@

        config.set('repository', 'segments_per_dir', str(self.DEFAULT_SEGMENTS_PER_DIR))
        config.set('repository', 'max_segment_size', str(self.DEFAULT_MAX_SEGMENT_SIZE))
        config.set('repository', 'append_only', str(int(self.append_only)))
        config.set('repository', 'id', hexlify(os.urandom(32)).decode('ascii'))
        config.set('repository', 'id', bin_to_hex(os.urandom(32)))
        self.save_config(path, config)

    def save_config(self, path, config):

@@ -438,6 +439,9 @@

            transaction_id = self.get_index_transaction_id()
        if transaction_id is None:
            transaction_id = self.io.get_latest_segment()
        if transaction_id is None:
            report_error('This repository contains no valid data.')
            return False
        if repair:
            self.io.cleanup(transaction_id)
        segments_transaction_id = self.io.get_segments_transaction_id()

@@ -666,7 +670,8 @@

                if not os.path.exists(dirname):
                    os.mkdir(dirname)
                    sync_dir(os.path.join(self.path, 'data'))
            self._write_fd = open(self.segment_filename(self.segment), 'ab')
            # play safe: fail if file exists (do not overwrite existing contents, do not append)
            self._write_fd = open(self.segment_filename(self.segment), 'xb')
            self._write_fd.write(MAGIC)
            self.offset = MAGIC_LEN
        return self._write_fd

@@ -705,6 +710,13 @@

            else:
                yield tag, key, offset
            offset += size
            # we must get the fd via get_fd() here again as we yielded to our caller and it might
            # have triggered closing of the fd we had before (e.g. by calling io.read() for
            # different segment(s)).
            # by calling get_fd() here again we also make our fd "recently used" so it likely
            # does not get kicked out of self.fds LRUcache.
            fd = self.get_fd(segment)
            fd.seek(offset)
            header = fd.read(self.header_fmt.size)

    def recover_segment(self, segment, filename):

@@ -809,18 +821,35 @@

        self.close_segment()  # after-commit fsync()

    def close_segment(self):
        if self._write_fd:
            self.segment += 1
            self.offset = 0
            self._write_fd.flush()
            os.fsync(self._write_fd.fileno())
            if hasattr(os, 'posix_fadvise'):  # only on UNIX
                # tell the OS that it does not need to cache what we just wrote,
                # avoids spoiling the cache for the OS and other processes.
                os.posix_fadvise(self._write_fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
            self._write_fd.close()
            sync_dir(os.path.dirname(self._write_fd.name))
            self._write_fd = None
        # set self._write_fd to None early to guard against reentry from error handling code pathes:
        fd, self._write_fd = self._write_fd, None
        if fd is not None:
            dirname = None
            try:
                self.segment += 1
                self.offset = 0
                dirname = os.path.dirname(fd.name)
                fd.flush()
                fileno = fd.fileno()
                os.fsync(fileno)
                if hasattr(os, 'posix_fadvise'):  # only on UNIX
                    try:
                        # tell the OS that it does not need to cache what we just wrote,
                        # avoids spoiling the cache for the OS and other processes.
                        os.posix_fadvise(fileno, 0, 0, os.POSIX_FADV_DONTNEED)
                    except OSError:
                        # usually, posix_fadvise can't fail for us, but there seem to
                        # be failures when running borg under docker on ARM, likely due
                        # to a bug outside of borg.
                        # also, there is a python wrapper bug, always giving errno = 0.
                        # https://github.com/borgbackup/borg/issues/2095
                        # as this call is not critical for correct function (just to
                        # optimize cache usage), we ignore these errors.
                        pass
            finally:
                fd.close()
                if dirname:
                    sync_dir(dirname)


MAX_DATA_SIZE = MAX_OBJECT_SIZE - LoggedIO.put_header_fmt.size

@@ -827 +856 @@

@@ -0,0 +1,225 @@

1	+	from binascii import unhexlify, a2b_base64, b2a_base64
2	+	import binascii
3	+	import textwrap
4	+	from hashlib import sha256
5	+	import pkgutil
6	+
7	+	from .key import KeyfileKey, RepoKey, PassphraseKey, KeyfileNotFoundError, PlaintextKey
8	+	from .helpers import Manifest, NoManifestError, Error, yes, bin_to_hex
9	+	from .repository import Repository
10	+
11	+
12	+	class UnencryptedRepo(Error):
13	+	"""Keymanagement not available for unencrypted repositories."""
14	+
15	+
16	+	class UnknownKeyType(Error):
17	+	"""Keytype {0} is unknown."""
18	+
19	+
20	+	class RepoIdMismatch(Error):
21	+	"""This key backup seems to be for a different backup repository, aborting."""
22	+
23	+
24	+	class NotABorgKeyFile(Error):
25	+	"""This file is not a borg key backup, aborting."""
26	+
27	+
28	+	def sha256_truncated(data, num):
29	+	h = sha256()
30	+	h.update(data)
31	+	return h.hexdigest()[:num]
32	+
33	+
34	+	KEYBLOB_LOCAL = 'local'
35	+	KEYBLOB_REPO = 'repo'
36	+
37	+
38	+	class KeyManager:
39	+	def __init__(self, repository):
40	+	self.repository = repository
41	+	self.keyblob = None
42	+	self.keyblob_storage = None
43	+
44	+	try:
45	+	cdata = self.repository.get(Manifest.MANIFEST_ID)
46	+	except Repository.ObjectNotFound:
47	+	raise NoManifestError
48	+
49	+	key_type = cdata[0]
50	+	if key_type == KeyfileKey.TYPE:
51	+	self.keyblob_storage = KEYBLOB_LOCAL
52	+	elif key_type == RepoKey.TYPE or key_type == PassphraseKey.TYPE:
53	+	self.keyblob_storage = KEYBLOB_REPO
54	+	elif key_type == PlaintextKey.TYPE:
55	+	raise UnencryptedRepo()
56	+	else:
57	+	raise UnknownKeyType(key_type)
58	+
59	+	def load_keyblob(self):
60	+	if self.keyblob_storage == KEYBLOB_LOCAL:
61	+	k = KeyfileKey(self.repository)
62	+	target = k.find_key()
63	+	with open(target, 'r') as fd:
64	+	self.keyblob = ''.join(fd.readlines()[1:])
65	+
66	+	elif self.keyblob_storage == KEYBLOB_REPO:
67	+	self.keyblob = self.repository.load_key().decode()
68	+
69	+	def store_keyblob(self, args):
70	+	if self.keyblob_storage == KEYBLOB_LOCAL:
71	+	k = KeyfileKey(self.repository)
72	+	try:
73	+	target = k.find_key()
74	+	except KeyfileNotFoundError:
75	+	target = k.get_new_target(args)
76	+
77	+	self.store_keyfile(target)
78	+	elif self.keyblob_storage == KEYBLOB_REPO:
79	+	self.repository.save_key(self.keyblob.encode('utf-8'))
80	+
81	+	def get_keyfile_data(self):
82	+	data = '%s %s\n' % (KeyfileKey.FILE_ID, bin_to_hex(self.repository.id))
83	+	data += self.keyblob
84	+	if not self.keyblob.endswith('\n'):
85	+	data += '\n'
86	+	return data
87	+
88	+	def store_keyfile(self, target):
89	+	with open(target, 'w') as fd:
90	+	fd.write(self.get_keyfile_data())
91	+
92	+	def export(self, path):
93	+	self.store_keyfile(path)
94	+
95	+	def export_qr(self, path):
96	+	with open(path, 'wb') as fd:
97	+	key_data = self.get_keyfile_data()
98	+	html = pkgutil.get_data('borg', 'paperkey.html')
99	+	html = html.replace(b'</textarea>', key_data.encode() + b'</textarea>')
100	+	fd.write(html)
101	+
102	+	def export_paperkey(self, path):
103	+	def grouped(s):
104	+	ret = ''
105	+	i = 0
106	+	for ch in s:
107	+	if i and i % 6 == 0:
108	+	ret += ' '
109	+	ret += ch
110	+	i += 1
111	+	return ret
112	+
113	+	export = 'To restore key use borg key import --paper /path/to/repo\n\n'
114	+
115	+	binary = a2b_base64(self.keyblob)
116	+	export += 'BORG PAPER KEY v1\n'
117	+	lines = (len(binary) + 17) // 18
118	+	repoid = bin_to_hex(self.repository.id)[:18]
119	+	complete_checksum = sha256_truncated(binary, 12)
120	+	export += 'id: {0:d} / {1} / {2} - {3}\n'.format(lines,
121	+	grouped(repoid),
122	+	grouped(complete_checksum),
123	+	sha256_truncated((str(lines) + '/' + repoid + '/' + complete_checksum).encode('ascii'), 2))
124	+	idx = 0
125	+	while len(binary):
126	+	idx += 1
127	+	binline = binary[:18]
128	+	checksum = sha256_truncated(idx.to_bytes(2, byteorder='big') + binline, 2)
129	+	export += '{0:2d}: {1} - {2}\n'.format(idx, grouped(bin_to_hex(binline)), checksum)
130	+	binary = binary[18:]
131	+
132	+	if path:
133	+	with open(path, 'w') as fd:
134	+	fd.write(export)
135	+	else:
136	+	print(export)
137	+
138	+	def import_keyfile(self, args):
139	+	file_id = KeyfileKey.FILE_ID
140	+	first_line = file_id + ' ' + bin_to_hex(self.repository.id) + '\n'
141	+	with open(args.path, 'r') as fd:
142	+	file_first_line = fd.read(len(first_line))
143	+	if file_first_line != first_line:
144	+	if not file_first_line.startswith(file_id):
145	+	raise NotABorgKeyFile()
146	+	else:
147	+	raise RepoIdMismatch()
148	+	self.keyblob = fd.read()
149	+
150	+	self.store_keyblob(args)
151	+
152	+	def import_paperkey(self, args):
153	+	# imported here because it has global side effects
154	+	import readline
155	+
156	+	repoid = bin_to_hex(self.repository.id)[:18]
157	+	try:
158	+	while True: # used for repeating on overall checksum mismatch
159	+	# id line input
160	+	while True:
161	+	idline = input('id: ').replace(' ', '')
162	+	if idline == "":
163	+	if yes("Abort import? [yN]:"):
164	+	raise EOFError()
165	+
166	+	try:
167	+	(data, checksum) = idline.split('-')
168	+	except ValueError:
169	+	print("each line must contain exactly one '-', try again")
170	+	continue
171	+	try:
172	+	(id_lines, id_repoid, id_complete_checksum) = data.split('/')
173	+	except ValueError:
174	+	print("the id line must contain exactly three '/', try again")
175	+	if sha256_truncated(data.lower().encode('ascii'), 2) != checksum:
176	+	print('line checksum did not match, try same line again')
177	+	continue
178	+	try:
179	+	lines = int(id_lines)
180	+	except ValueError:
181	+	print('internal error while parsing length')
182	+
183	+	break
184	+
185	+	if repoid != id_repoid:
186	+	raise RepoIdMismatch()
187	+
188	+	result = b''
189	+	idx = 1
190	+	# body line input
191	+	while True:
192	+	inline = input('{0:2d}: '.format(idx))
193	+	inline = inline.replace(' ', '')
194	+	if inline == "":
195	+	if yes("Abort import? [yN]:"):
196	+	raise EOFError()
197	+	try:
198	+	(data, checksum) = inline.split('-')
199	+	except ValueError:
200	+	print("each line must contain exactly one '-', try again")
201	+	continue
202	+	try:
203	+	part = unhexlify(data)
204	+	except binascii.Error:
205	+	print("only characters 0-9 and a-f and '-' are valid, try again")
206	+	continue
207	+	if sha256_truncated(idx.to_bytes(2, byteorder='big') + part, 2) != checksum:
208	+	print('line checksum did not match, try line {0} again'.format(idx))
209	+	continue
210	+	result += part
211	+	if idx == lines:
212	+	break
213	+	idx += 1
214	+
215	+	if sha256_truncated(result, 12) != id_complete_checksum:
216	+	print('The overall checksum did not match, retry or enter a blank line to abort.')
217	+	continue
218	+
219	+	self.keyblob = '\n'.join(textwrap.wrap(b2a_base64(result).decode('ascii'))) + '\n'
220	+	self.store_keyblob(args)
221	+	break
222	+
223	+	except EOFError:
224	+	print('\n - aborted')
225	+	return

@@ -0 +226 @@

@@ -1,12 +1,12 @@

from binascii import hexlify
import datetime
import logging
logger = logging.getLogger(__name__)
import os
import shutil
import time

from .helpers import get_keys_dir, get_cache_dir, ProgressIndicatorPercent
from .logger import create_logger
logger = create_logger()

from .helpers import get_keys_dir, get_cache_dir, ProgressIndicatorPercent, bin_to_hex
from .locking import Lock
from .repository import Repository, MAGIC
from .key import KeyfileKey, KeyfileNotFoundError

@@ -188,8 +188,8 @@

        attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR',
                                         os.path.join(os.path.expanduser('~'),
                                                      '.cache', 'attic'))
        attic_cache_dir = os.path.join(attic_cache_dir, hexlify(self.id).decode('ascii'))
        borg_cache_dir = os.path.join(get_cache_dir(), hexlify(self.id).decode('ascii'))
        attic_cache_dir = os.path.join(attic_cache_dir, bin_to_hex(self.id))
        borg_cache_dir = os.path.join(get_cache_dir(), bin_to_hex(self.id))

        def copy_cache_file(path):
            """copy the given attic cache path into the borg directory

@@ -263,7 +263,7 @@

           assume the repository has been opened by the archiver yet
        """
        get_keys_dir = cls.get_keys_dir
        id = hexlify(repository.id).decode('ascii')
        id = bin_to_hex(repository.id)
        keys_dir = get_keys_dir()
        if not os.path.exists(keys_dir):
            raise KeyfileNotFoundError(repository.path, keys_dir)

@@ -313,7 +313,7 @@

    @classmethod
    def find_key_file(cls, repository):
        get_keys_dir = cls.get_keys_dir
        id = hexlify(repository.id).decode('ascii')
        id = bin_to_hex(repository.id)
        keys_dir = get_keys_dir()
        if not os.path.exists(keys_dir):
            raise KeyfileNotFoundError(repository.path, keys_dir)

@@ -320 +320 @@

@@ -12,9 +12,6 @@


from .helpers import Buffer

from .logger import create_logger
logger = create_logger()


try:
    ENOATTR = errno.ENOATTR

@@ -68,7 +65,7 @@

        libc_name = 'libc.dylib'
    else:
        msg = "Can't find C library. No fallback known. Try installing ldconfig, gcc/cc or objdump."
        logger.error(msg)
        print(msg, file=sys.stderr)  # logger isn't initialized at this stage
        raise Exception(msg)

# If we are running with fakeroot on Linux, then use the xattr functions of fakeroot. This is needed by

@@ -117,7 +114,7 @@



class BufferTooSmallError(Exception):
    """the buffer given to an xattr function was too small for the result"""
    """the buffer given to an xattr function was too small for the result."""


def _check(rv, path=None, detect_buffer_too_small=False):

@@ -208,7 +205,7 @@


        n, buf = _listxattr_inner(func, path)
        return [os.fsdecode(name) for name in split_string0(buf[:n])
                if not name.startswith(b'system.posix_acl_')]
                if name and not name.startswith(b'system.posix_acl_')]

    def getxattr(path, name, *, follow_symlinks=True):
        def func(path, name, buf, size):

@@ -264,7 +261,7 @@

                    return libc.listxattr(path, buf, size, XATTR_NOFOLLOW)

        n, buf = _listxattr_inner(func, path)
        return [os.fsdecode(name) for name in split_string0(buf[:n])]
        return [os.fsdecode(name) for name in split_string0(buf[:n]) if name]

    def getxattr(path, name, *, follow_symlinks=True):
        def func(path, name, buf, size):

@@ -323,7 +320,7 @@

                    return libc.extattr_list_link(path, ns, buf, size)

        n, buf = _listxattr_inner(func, path)
        return [os.fsdecode(name) for name in split_lstring(buf[:n])]
        return [os.fsdecode(name) for name in split_lstring(buf[:n]) if name]

    def getxattr(path, name, *, follow_symlinks=True):
        def func(path, name, buf, size):

@@ -330 +327 @@

Learn more Showing 5 files with coverage changes found.

New file borg/keymanager.py

New

Loading file...

Changes in borg/remote.py

-1

→

Loading file...

Changes in borg/key.py

-1

→

Loading file...

Changes in borg/helpers.py

-14

→

+234

Loading file...

Changes in borg/archiver.py

-32

→

+228

Loading file...

83.88% ø ø

paperkey.html - decode as utf-8, fixes #2150

ThomasWaldmann

e5f7121

CI Passed

83.88% ø -0.04%

add paperkey.html to pyinstaller spec

bb62603

-2

33aeb22

+663

+670

-16

81.69% ø +1.31%

setup.py: build_api: sort file list for determinism

enkore

e208d11

CI Passed

Hiding 3 contexual commits

+67

-6

-61

80.38% ø -1.25%

key export: center QR code on the page

enkore

0d20a0d

CI Passed

Hiding 2 contexual commits

-58

+54

81.51% ø ø

Merge pull request #2146 from bebehei/samefs-mountpoints-doc-maint

enkore

c688b1b

CI Passed

81.51% ø -0.10%

clearify doc for same filesystems

bebehei

d3a2f36

CI Passed

81.51% 61.11% ø

Merge pull request #2135 from textshell/feature/paperkey-doc-qr

enkore

4f9dd98

CI Passed

81.51% 61.11% -0.10%

Add qr html export mode to `key export` command

textshell

257e55f

CI Passed

Hiding 2 contexual commits

+13

81.60% ø ø

Merge pull request #2136 from textshell/update-authors

ThomasWaldmann

455a9c0

CI Passed

Hiding 1 contexual commits

81.60% 100.00% ø

Merge pull request #2115 from textshell/fix/manifest-timestamp

textshell

5479ef4

CI Passed

81.62% ø +0.03%

Merge pull request #2122 from ThomasWaldmann/pytest2x-compat

enkore

2209b9c

CI Passed

-1

81.58% ø ø

pytest: use [pytest] section in setup.cfg

ThomasWaldmann

acc44a1

CI Passed

-5

81.60% ø +0.01%

Manifest: Make sure manifest timestamp is strictly monotonically increasing.

textshell

6b8cf0a

CI Passed

81.58% ø ø

Merge pull request #2110 from ThomasWaldmann/release-1.0.10rc1

d23dbb4

81.58% ø ø

e6c1931

Merge pull request #2090 from ThomasWaldmann/update-1.0-changes

ThomasWaldmann

6252614

CI Passed

+66

-5

-61

80.29% ø -1.34%

update CHANGES (1.0-maint)

ThomasWaldmann

dc34926

CI Passed

-68

+62

81.62% ø +<.01%

Merge pull request #2104 from ThomasWaldmann/fix-fadvise-fail

ThomasWaldmann

2c1751c

CI Passed

-1

81.58% ø ø

ignore posix_fadvise errors in repository.py, work around #2095

ThomasWaldmann

add38e8

CI Passed

Hiding 1 contexual commits

81.62% ø ø

Merge pull request #2109 from ThomasWaldmann/vagrantfile-improvements

ThomasWaldmann

213e7b7

CI Passed

Hiding 1 contexual commits

81.62% ø -0.04%

Merge pull request #2102 from enkore/issue/2082

ThomasWaldmann

127250c

CI Passed

81.62% ø -0.04%

mount: umount on SIGINT/^C when in foreground

enkore

2cfaf03

CI Passed

-2

81.66% ø +0.03%

Merge pull request #2100 from ThomasWaldmann/fix-double-magic

ThomasWaldmann

528891a

CI Passed

-1

81.62% ø ø

Merge pull request #2107 from enkore/issue/2106

ThomasWaldmann

4e21715

CI Passed

81.62% ø ø

docs: add CVE numbers for issues fixed in 1.0.9

enkore

fbaefc9

CI Passed

-2

81.66% ø +<.01%

creating a new segment: use "xb" mode, fixes #2099

ThomasWaldmann

6996fa6

CI Passed

-1

81.62% 100.00% -0.04%

Merge pull request #2094 from enkore/issue/2092

ThomasWaldmann

cf0192c

CI Passed

81.62% ø ø

mount: handle invalid hard link refs

enkore

8fe047e

CI Passed

-1

81.65% ø +0.04%

Merge pull request #2096 from ThomasWaldmann/fix-use-after-close

ThomasWaldmann

b6191ec

CI Passed

81.65% ø +<.01%

SyncFile: fix use of fd object after close

ThomasWaldmann

fc8be58

CI Passed

Hiding 1 contexual commits

-1

81.61% ø -0.04%

Merge pull request #2091 from enkore/issue/2073

enkore

739578e

CI Passed

-2

81.65% ø ø

Merge pull request #2089 from ThomasWaldmann/fix-manifest

enkore

5f5b1b2

CI Passed

-1

81.61% ø ø

hashindex: separate endian-dependent defs from endian detection

enkore

fafd5e0

CI Passed

Hiding 1 contexual commits

-2

81.65% ø +<.01%

Merge pull request #2015 from ThomasWaldmann/fix-location-regex

ThomasWaldmann

ca0c1da

CI Passed

-90

-94

82.05% ø +0.39%

Manifest.in: simplify, exclude *.{so,dll,orig}, fixes #2066

ThomasWaldmann

ddd9d77

CI Passed

+91

+95

-4

81.65% ø +0.03%

Merge pull request #2085 from ThomasWaldmann/use-freebsd-release

ThomasWaldmann

0b2321a

CI Passed

Hiding 1 contexual commits

-1

81.61% ø ø

Merge pull request #2084 from ThomasWaldmann/binaries-with-py353

ThomasWaldmann

bdab5de

CI Passed

+65

-7

-58

80.33% ø ø

use osxfuse 3.5.4 for tests / to build binaries

ThomasWaldmann

7b9ff75

CI Passed

Hiding 1 contexual commits

-65

+58

81.61% ø ø

Merge pull request #2076 from ThomasWaldmann/fix-pyinstaller-bootloader

ThomasWaldmann

c44d9ad

CI Passed

+65

-7

-58

80.33% ø -1.28%

pyinstaller: use fixed AND freshly compiled bootloader, fixes #2002

ThomasWaldmann

cdffd93

CI Passed

-65

+58

81.61% ø ø

Merge pull request #2064 from ThomasWaldmann/update-1.0-changes

ThomasWaldmann

e022cf1

CI Passed

Hiding 1 contexual commits

81.61% ø +1.27%

Merge pull request #2063 from ThomasWaldmann/travis-osx-pythons

ThomasWaldmann

f266299

CI Passed

+65

-5

-60

80.33% ø ø

Merge pull request #2061 from ThomasWaldmann/fix-vagrant-freebsd

ThomasWaldmann

3b8beb9

CI Passed

-2

80.33% ø ø

travis: use latest pythons for OS X based testing

ThomasWaldmann

94e35fc

CI Passed

Hiding 2 contexual commits

-2

80.33% ø -1.28%

Merge pull request #2060 from ThomasWaldmann/require-current-pip-setuptools

ThomasWaldmann

c716df4

CI Passed

-65

+60

81.61% ø ø

Merge pull request #2059 from ThomasWaldmann/fix-xattr-test-race

ThomasWaldmann

52c2784

CI Passed

Hiding 1 contexual commits

-2

81.65% ø +0.03%

add pip and setuptools to requirements file, fixes #2030

ThomasWaldmann

555c6a9

CI Passed

-1

81.61% ø ø

fix xattr test race condition, fixes #2047

ThomasWaldmann

c0fb8da

CI Passed

81.61% ø ø

Merge pull request #2057 from ThomasWaldmann/fix-pytest-deprecation

ThomasWaldmann

e0094f7

CI Passed

81.61% ø ø

Merge pull request #2056 from ThomasWaldmann/remove-dot-github

ThomasWaldmann

c3e4d7a

CI Passed

81.61% ø ø

Merge pull request #2058 from ThomasWaldmann/adjust-xdist-parallelism

ThomasWaldmann

8e9cf5d

CI Passed

Hiding 1 contexual commits

+65

-5

-60

80.33% ø -1.28%

setup.cfg: fix pytest deprecation warning, fixes #2050

ThomasWaldmann

dedc4c0

CI Passed

Hiding 1 contexual commits

-65

+60

81.61% ø ø

Merge pull request #2038 from ThomasWaldmann/update-1.0-changes

ThomasWaldmann

32e58e8

CI Passed

Hiding 1 contexual commits

81.61% ø ø

Merge pull request #2044 from ThomasWaldmann/fix-openbsd-repo

enkore

aea1305

CI Passed

Hiding 1 contexual commits

81.61% ø ø

Merge pull request #2035 from ThomasWaldmann/docs-backup-from-stdin

enkore

82fd84e

CI Passed

81.61% ø ø

Merge pull request #2034 from ThomasWaldmann/api-version-number-spacing

enkore

a488d39

CI Passed

81.61% ø ø

Merge pull request #2033 from ThomasWaldmann/require-succeeding-osx-tests

enkore

68dd61f

CI Passed

81.61% ø +0.02%

Merge pull request #2032 from ThomasWaldmann/fix-pipe-write

ThomasWaldmann

986740b

CI Passed

+65

-7

-58

80.33% ø -1.26%

borg create: document how to backup stdin, fixes #2013

ThomasWaldmann

022c128

CI Passed

-97

-163

+11

+55

81.97% ø +0.38%

API_VERSION: use numberspaces, fixes #2023

ThomasWaldmann

1c854b9

CI Passed

-3

82.03% ø +0.44%

travis: require succeeding OS X tests, fixes #2028

ThomasWaldmann

93d7d3c

CI Passed

+97

+101

-5

81.61% ø +0.02%

borg serve: fix transmission data loss of pipe writes, fixes #1268

ThomasWaldmann

941b8d7

CI Passed

Hiding 4 contexual commits

+11

+10

81.59% ø -0.10%

Merge pull request #2018 from enkore/f/debug-remote

enkore

207a211

CI Passed

Hiding 2 contexual commits

+13

81.68% ø +0.05%

Merge pull request #2016 from ThomasWaldmann/fix-typos

ThomasWaldmann

b206aa7

CI Passed

Hiding 1 contexual commits

-1

-2

81.62% ø -0.19%

Merge pull request #2000 from enkore/issue/1997

enkore

53aaee3

CI Passed

-3

81.81% ø ø

Merge pull request #1991 from ThomasWaldmann/fix-pytest-xdist

enkore

6075d54

CI Passed

Hiding 2 contexual commits

-191

-203

82.70% ø +0.85%

fix bad parsing of wrong syntax

ThomasWaldmann

1667926

CI Passed

+191

+201

-6

-4

81.85% ø +0.03%

Merge pull request #2006 from sherbang/patch-1

ThomasWaldmann

c517264

CI Passed

Hiding 1 contexual commits

-1

81.81% ø ø

update CHANGES (1.0-maint)

ThomasWaldmann

69b816f

CI Passed

81.81% ø +1.34%

Merge pull request #2005 from ThomasWaldmann/fix-travis-osx-builds

ThomasWaldmann

6cb6984

CI Passed

81.81% ø ø

travis: install py36 on OS X

ThomasWaldmann

e119042

CI Passed

Hiding 1 contexual commits

+68

-8

-60

80.46% ø ø

work around spurious log level related test fail when using pytest-xdist

ThomasWaldmann

2938a5f

CI Passed

Hiding 3 contexual commits

80.46% ø ø

Merge pull request #1996 from ThomasWaldmann/py36

enkore

e61de12

CI Passed

80.46% ø +0.03%

Merge pull request #1995 from enkore/f/buffer-exc

ThomasWaldmann

a84466d

CI Passed

80.46% ø +0.03%

helpers.Buffer: raise OSError subclass if too much memory is allocd

enkore

320a561

CI Passed

-1

-5

80.37% ø -0.07%

check: fail if single archive does not exist

enkore

be8e0c8

CI Passed

-2

80.47% ø ø

vagrant: add Python 3.6.0

ThomasWaldmann

c412b86

CI Passed

Hiding 1 contexual commits

-1

80.43% ø -0.14%

tox / travis: also test on Python 3.6

ThomasWaldmann

9533493

CI Passed

80.43% ø -0.13%

Merge pull request #1994 from enkore/issue/1981

enkore

73795f5

CI Passed

80.43% ø ø

change-passphrase: print key location

enkore

4b9a9f9

CI Passed

Hiding 2 contexual commits

80.56% ø -0.02%

Merge pull request #1993 from enkore/issue/1992

enkore

9d7ec9a

CI Passed

80.56% ø ø

xattr: only skip file on BufferTooSmallError

enkore

3e04fa9

CI Passed

Hiding 1 contexual commits

80.57% ø +<.01%

Merge pull request #1983 from enkore/f/sigtrace

60d33b8

80.57% ø ø

c2c31aa

Hiding 1 contexual commits

80.57% ø +0.35%

Merge pull request #1968 from ThomasWaldmann/release-1.0.9

ThomasWaldmann

d5bc486

CI Passed

80.57% ø ø

CHANGES: fix 1.0.9 release date

ThomasWaldmann

c54a912

CI Passed

Hiding 7 contexual commits

+156

+143

+10

80.21% ø +<.01%

Merge pull request #1966 from enkore/1.0-340

enkore

343387b

CI Passed

Hiding 1 contexual commits

80.21% ø ø

Merge pull request #1965 from borgbackup/pytest2

enkore

0b32524

CI Passed

-2

80.25% ø +0.04%

conftest: pytest 2 compat

enkore

af923e2

CI Passed

-1

80.21% ø +<.01%

Merge pull request #1941 from enkore/issue/1936

enkore

49c0571

CI Passed

-2

80.25% ø +0.02%

create: fix duration if --timestamp is given

enkore

baa8baa

CI Passed

-1

80.22% ø ø

update changes

enkore

445365b

CI Passed

Hiding 4 contexual commits

80.20% ø ø

Merge pull request #1962 from ThomasWaldmann/win10-linux-subsys

enkore

9396aac

CI Passed

80.20% ø ø

Merge pull request #1963 from ThomasWaldmann/fix-win10-lxsys-tests

enkore

20d3eff

CI Passed

80.20% ø ø

catch errno.ENOSYS for mknod (win 10 lxsys)

ThomasWaldmann

6137008

CI Passed

80.20% ø ø

document windows 10 linux subsystem install

ThomasWaldmann

6b6ddec

CI Passed

80.20% ø ø

Merge pull request #1959 from ThomasWaldmann/pretty-test-fails2

ThomasWaldmann

4f0c2ab

CI Passed

80.20% ø ø

pytest: only rewrite the testsuite, fixes #1938

ThomasWaldmann

04340ae

CI Passed

80.20% ø -0.03%

Merge pull request #1953 from ThomasWaldmann/fix-1932

ThomasWaldmann

7f63ca8

CI Passed

80.20% ø -0.03%

add a borg debug/key dummy command, fixes #1932

ThomasWaldmann

5a40870

CI Passed

80.22% ø ø

update CHANGES (1.0-maint) (#1954)

ThomasWaldmann

60bbd7a

CI Passed

80.22% ø +<.01%

Merge pull request #1937 from ThomasWaldmann/location-parser

ThomasWaldmann

d138548

CI Passed

80.22% ø ø

refactor common regex part into optional_user_re

ThomasWaldmann

292ff42

CI Passed

80.22% ø ø

Merge pull request #1951 from ThomasWaldmann/pretty-test-fails

enkore

60b6f5a

CI Passed

80.22% ø ø

get back pytest's pretty assertion failures, fixes #1938

ThomasWaldmann

c6017ab

CI Passed

80.22% ø ø

Merge pull request #1939 from ThomasWaldmann/update-1.0-changes

enkore

796ea9e

CI Passed

80.22% ø ø

update CHANGES (1.0-maint)

ThomasWaldmann

bd8b4a4

CI Passed

Hiding 1 contexual commits

80.22% ø ø

Merge pull request #1924 from ThomasWaldmann/pr-template

enkore

4affad7

CI Passed

80.22% ø ø

add a PR template pointing to guidelines

ThomasWaldmann

dfd37d0

CI Passed

80.22% ø ø

Merge pull request #1922 from enkore/f/buildusage

ThomasWaldmann

7eb2dff

CI Passed

Hiding 1 contexual commits

-2

80.26% ø +0.04%

setup.py: fix build_usage not processing all commands

enkore

1c261f6

CI Passed

-1

Processing...

Merge pull request #1906 from enkore/f/check-corrupted-manifest

ThomasWaldmann

8ddbc45

+11

+14

-1

-2

80.11% ø ø

Merge pull request #1907 from ThomasWaldmann/cygwin-docs

ThomasWaldmann

2a340bf

CI Passed

Hiding 1 contexual commits

-11

-14

80.22% ø ø

check: skip corrupted chunks during manifest rebuild

enkore

146d586

CI Passed

Hiding 1 contexual commits

+11

+14

-1

-2

Processing...

Merge pull request #1905 from ThomasWaldmann/fix-1903

enkore

cdd9891

Hiding 1 contexual commits

80.15% ø -0.02%

Merge pull request #1895 from ThomasWaldmann/fix-1894

ThomasWaldmann

b25de0a

CI Passed

80.15% ø -0.02%

fix traceback in Error handler if id is None, fixes #1894

ThomasWaldmann

cd50e28

CI Passed

80.16% ø ø

Merge pull request #1898 from ThomasWaldmann/fix-1897

ThomasWaldmann

baab519

CI Passed

80.16% ø ø

test_get_(cache|keys)_dir: clean env state, fixes #1897

ThomasWaldmann

9e760a6

CI Passed

80.16% ø ø

1.0 maint AUTHORS +me

enkore

619cb12

CI Passed

80.16% ø ø

Merge pull request #1885 from jcrben/patch-1

ThomasWaldmann

2780a83

CI Passed

80.16% ø -0.05%

Clarify extract is relative to current directory

jcrben

a49fc6f

CI Passed

80.16% ø -0.05%

Merge pull request #1889 from enkore/f/109docfixes

enkore

104e9f3

CI Passed

80.16% ø ø

docs/usage: fix literal/emph without end-string (two instances)

enkore

bf3a1f0

CI Passed

Hiding 2 contexual commits

-2

5de31f5

-1

80.16% ø ø

Merge pull request #1865 from ThomasWaldmann/update-1.0-changes

ThomasWaldmann

c920a16

CI Passed

Hiding 3 contexual commits

80.16% ø ø

Merge pull request #1884 from ThomasWaldmann/upgrade-osxfuse

ThomasWaldmann

373d8e8

CI Passed

Hiding 1 contexual commits

80.16% ø ø

Merge pull request #1882 from ThomasWaldmann/docs-rsrc-usage

ThomasWaldmann

8465979

CI Passed

80.16% ø -0.01%

Merge pull request #1883 from ThomasWaldmann/noatime-noctime

ThomasWaldmann

b808d95

CI Passed

80.16% ø -0.01%

implement noatime / noctime, fixes #1853

ThomasWaldmann

9451ab6

CI Passed

80.17% ø ø

fixup: fixes, clarify

ThomasWaldmann

30d1e21

CI Passed

80.17% ø ø

Merge pull request #1876 from ThomasWaldmann/vagrantfile-cleanup

enkore

20798df

CI Passed

80.17% ø ø

add more details about resource usage

ThomasWaldmann

76c5f1a

CI Passed

80.17% ø ø

Merge pull request #1875 from ThomasWaldmann/improve-files-cache-docs

ThomasWaldmann

cc20c98

CI Passed

80.17% ø ø

Merge pull request #1877 from anarcat/pypi

ThomasWaldmann

d56683f

CI Passed

80.17% ø -1.59%

display README correctly on PyPI

anarcat

fbada18

CI Passed

Hiding 2 contexual commits

80.17% ø ø

improve cache / index docs, esp. files cache docs, fixes #1825

ThomasWaldmann

c8b58e0

CI Passed

80.17% ø -0.04%

Merge pull request #1864 from ThomasWaldmann/fix-newest-mtime

ThomasWaldmann

df5482d

CI Passed

80.17% ø -0.04%

fix determination of newest mtime, fixes #1860

ThomasWaldmann

cabcbc5

CI Passed

80.21% ø ø

Merge pull request #1863 from ThomasWaldmann/doc-updates

enkore

1fc368c

CI Passed

Hiding 4 contexual commits

80.21% ø -0.01%

Merge pull request #1856 from ThomasWaldmann/umount

ThomasWaldmann

ea6515d

CI Passed

80.21% ø -0.01%

borg umount, fixes #1855

ThomasWaldmann

e17fe2b

CI Passed

+10

80.21% ø ø

Merge pull request #1851 from Abogical/patch-1

d871efc

80.21% ø ø

9cf4846

80.21% ø ø

Merge pull request #1848 from ThomasWaldmann/clarify-pr-targets

enkore

c055dd2

CI Passed

80.21% ø ø

developers docs: clarify how to choose PR target branch

ThomasWaldmann

c81f861

CI Passed

80.21% ø ø

Merge pull request #1847 from ThomasWaldmann/travis-osx

ThomasWaldmann

cda88e1

CI Passed

80.21% ø ø

travis: allow OS X failures until the brew cask osxfuse issue is fixed

ThomasWaldmann

372ebb4

CI Passed

80.21% ø -0.08%

Merge pull request #1841 from enkore/issue/1837

enkore

4b5a17c

CI Passed

Hiding 2 contexual commits

80.28% ø -0.04%

Merge pull request #1844 from enkore/issue/1815

ThomasWaldmann

9b1241e

CI Passed

80.28% ø ø

check: handle repo w/o objects gracefully

enkore

2261709

CI Passed

Hiding 1 contexual commits

80.31% ø ø

Merge pull request #1843 from enkore/issue/1801

enkore

3789c78

CI Passed

80.31% ø ø

fix tox build for environment-python != containing-python in yet-another instance

enkore

960c421

CI Passed

80.31% ø -0.01%

Merge pull request #1840 from ThomasWaldmann/fix-logger

enkore

1a0218b

CI Passed

80.31% ø -0.01%

at xattr module import time, loggers are not initialized yet

ThomasWaldmann

3237db4

CI Passed

-2

80.32% ø -1.39%

Merge pull request #1839 from ThomasWaldmann/osxfuse-non-beta

ThomasWaldmann

5c98b85

CI Passed

-1

80.32% ø -1.39%

caskroom osxfuse-beta gone, it's osxfuse now (3.5.3)

ThomasWaldmann

cd8dfda

CI Passed

-67

+60

81.71% ø ø

Merge pull request #1830 from languitar/1.0-maint

ThomasWaldmann

19f6f7f

CI Passed

+67

-7

-60

80.32% ø -1.39%

Clarify prune behavior for different archive contents

languitar

3ee0197

CI Passed

-67

+60

81.71% ø ø

testsuite/archiver: fix missing newline before RemoteArchiverTestCase

enkore

5fecac6

CI Passed

Hiding 4 contexual commits

-1

-2

Processing...

Merge pull request #1808 from anarcat/restore-size

ThomasWaldmann

94ff495

+64

-7

-57

adfd66d

-67

+59

81.71% ø -0.05%

Merge pull request #1807 from anarcat/docs-misc

ThomasWaldmann

365ec85

CI Passed

+67

-8

-59

80.32% ø ø

fix a heading that was incorrectly set

anarcat

057ad07

CI Passed

Hiding 2 contexual commits

-69

+60

81.75% ø +0.04%

Merge pull request #1804 from anarcat/security-notes

ThomasWaldmann

45c76c1

CI Passed

-1

81.71% ø ø

fix links in standalone README

anarcat

319ecd8

CI Passed

Hiding 3 contexual commits

81.71% ø ø

Merge pull request #1805 from anarcat/pdf-docs

ThomasWaldmann

c5f5d17

CI Passed

+1 Files

+300

+223

+12

+65

Processing...

update CHANGES with description of issue #1428

ThomasWaldmann

f32c885

Files		•	•	•	Coverage
borg	+1,252	+1,126	+42	+84	^+1.67% 83.88%
Project Totals (15 files)	5,776	4,845	265	666	83.88%

1008	1047		return _state
1009	1048
1010	1049		def report(msg, chunk_id, chunk_no):
1011		-	cid = hexlify(chunk_id).decode('ascii')
	1050	+	cid = bin_to_hex(chunk_id)
1012	1051		msg += ' [chunk: %06d_%s]' % (chunk_no, cid) # see debug-dump-archive-items
1013	1052		self.error_found = True
1014	1053		logger.error(msg)
1015	1054
	1055	+	def list_keys_safe(keys):
	1056	+	return ', '.join((k.decode() if isinstance(k, bytes) else str(k) for k in keys))
	1057	+
1016	1058		def valid_item(obj):
1017	1059		if not isinstance(obj, StableDict):
1018		-	return False
	1060	+	return False, 'not a dictionary'
	1061	+	# A bug in Attic up to and including release 0.13 added a (meaningless) b'acl' key to every item.
	1062	+	# We ignore it here, should it exist. See test_attic013_acl_bug for details.
	1063	+	obj.pop(b'acl', None)
1019	1064		keys = set(obj)
1020		-	return REQUIRED_ITEM_KEYS.issubset(keys) and keys.issubset(item_keys)
	1065	+	if not REQUIRED_ITEM_KEYS.issubset(keys):
	1066	+	return False, 'missing required keys: ' + list_keys_safe(REQUIRED_ITEM_KEYS - keys)
	1067	+	if not keys.issubset(item_keys):
	1068	+	return False, 'invalid keys: ' + list_keys_safe(keys - item_keys)
	1069	+	return True, ''
1021	1070
1022	1071		i = 0
1023	1072		for state, items in groupby(archive[b'items'], missing_chunk_detector):

1033	1082		unpacker.feed(self.key.decrypt(chunk_id, cdata))
1034	1083		try:
1035	1084		for item in unpacker:
1036		-	if valid_item(item):
	1085	+	valid, reason = valid_item(item)
	1086	+	if valid:
1037	1087		yield item
1038	1088		else:
1039		-	report('Did not get expected metadata dict when unpacking item metadata', chunk_id, i)
	1089	+	report('Did not get expected metadata dict when unpacking item metadata (%s)' % reason, chunk_id, i)
1040	1090		except RobustUnpacker.UnpackerCrashed as err:
1041	1091		report('Unpacker crashed while unpacking item metadata, trying to resync...', chunk_id, i)
1042	1092		unpacker.resync()

1051	1101		key=lambda name_info: name_info[1][b'time'])
1052	1102		if prefix is not None:
1053	1103		archive_items = [item for item in archive_items if item[0].startswith(prefix)]
	1104	+	if not archive_items:
	1105	+	logger.warning('--prefix %s does not match any archives', prefix)
1054	1106		num_archives = len(archive_items)
1055	1107		end = None if last is None else min(num_archives, last)
	1108	+	if last is not None and end < last:
	1109	+	logger.warning('--last %d archives: only found %d archives', last, end)
1056	1110		else:
1057	1111		# we only want one specific archive
1058	1112		archive_items = [item for item in self.manifest.archives.items() if item[0] == archive]
1059	1113		num_archives = 1
1060	1114		end = 1
	1115	+	if not archive_items:
	1116	+	logger.error('Archive %s does not exist', archive)
	1117	+	self.error_found = True
	1118	+	return
1061	1119
1062	1120		with cache_if_remote(self.repository) as repository:
1063	1121		for i, (name, info) in enumerate(archive_items[:end]):

5	5		import sys
6	6		import textwrap
7	7		from hmac import HMAC, compare_digest
8		-	from hashlib import sha256, pbkdf2_hmac
	8	+	from hashlib import sha256, sha512, pbkdf2_hmac
9	9
10		-	from .helpers import IntegrityError, get_keys_dir, Error, yes
	10	+	import msgpack
	11	+
	12	+	from .helpers import StableDict, IntegrityError, get_keys_dir, get_security_dir, Error, yes, bin_to_hex
11	13		from .logger import create_logger
12	14		logger = create_logger()
13	15
14	16		from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
15		-	from .compress import Compressor
16		-	import msgpack
	17	+	from .crypto import hkdf_hmac_sha512
	18	+	from .compress import Compressor, CNONE
17	19
18	20		PREFIX = b'\0' * 8
19	21

30	32		"""Unsupported payload type {}. A newer version is required to access this repository."""
31	33
32	34
	35	+	class UnsupportedManifestError(Error):
	36	+	"""Unsupported manifest envelope. A newer version is required to access this repository."""
	37	+
	38	+
33	39		class KeyfileNotFoundError(Error):
34	40		"""No key file for repository {} found in {}."""
35	41

63	95		raise UnsupportedPayloadError(key_type)
64	96
65	97
	98	+	def tam_required_file(repository):
	99	+	security_dir = get_security_dir(bin_to_hex(repository.id))
	100	+	return os.path.join(security_dir, 'tam_required')
	101	+
	102	+
	103	+	def tam_required(repository):
	104	+	file = tam_required_file(repository)
	105	+	return os.path.isfile(file)
	106	+
	107	+
66	108		class KeyBase:
67	109		TYPE = None # override in subclasses
68	110

71	113		self.repository = repository
72	114		self.target = None # key location file path / repo obj
73	115		self.compressor = Compressor('none')
	116	+	self.tam_required = True
74	117
75	118		def id_hash(self, data):
76	119		"""Return HMAC hash using the "id" HMAC key
77	120		"""
78	121
79		-	def encrypt(self, data):
	122	+	def encrypt(self, data, none_compression=False):
80	123		pass
81	124
82	125		def decrypt(self, id, data):
83	126		pass
84	127
	128	+	def _tam_key(self, salt, context):
	129	+	return hkdf_hmac_sha512(
	130	+	ikm=self.id_key + self.enc_key + self.enc_hmac_key,
	131	+	salt=salt,
	132	+	info=b'borg-metadata-authentication-' + context,
	133	+	output_length=64
	134	+	)
	135	+
	136	+	def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest'):
	137	+	metadata_dict = StableDict(metadata_dict)
	138	+	tam = metadata_dict['tam'] = StableDict({
	139	+	'type': 'HKDF_HMAC_SHA512',
	140	+	'hmac': bytes(64),
	141	+	'salt': os.urandom(64),
	142	+	})
	143	+	packed = msgpack.packb(metadata_dict, unicode_errors='surrogateescape')
	144	+	tam_key = self._tam_key(tam['salt'], context)
	145	+	tam['hmac'] = HMAC(tam_key, packed, sha512).digest()
	146	+	return msgpack.packb(metadata_dict, unicode_errors='surrogateescape')
	147	+
	148	+	def unpack_and_verify_manifest(self, data, force_tam_not_required=False):
	149	+	"""Unpack msgpacked data and return (object, did_verify)."""
	150	+	if data.startswith(b'\xc1' * 4):
	151	+	# This is a manifest from the future, we can't read it.
	152	+	raise UnsupportedManifestError()
	153	+	tam_required = self.tam_required
	154	+	if force_tam_not_required and tam_required:
	155	+	logger.warning('Manifest authentication DISABLED.')
	156	+	tam_required = False
	157	+	data = bytearray(data)
	158	+	# Since we don't trust these bytes we use the slower Python unpacker,
	159	+	# which is assumed to have a lower probability of security issues.
	160	+	unpacked = msgpack.fallback.unpackb(data, object_hook=StableDict, unicode_errors='surrogateescape')
	161	+	if b'tam' not in unpacked:
	162	+	if tam_required:
	163	+	raise TAMRequiredError(self.repository._location.canonical_path())
	164	+	else:
	165	+	logger.debug('TAM not found and not required')
	166	+	return unpacked, False
	167	+	tam = unpacked.pop(b'tam', None)
	168	+	if not isinstance(tam, dict):
	169	+	raise TAMInvalid()
	170	+	tam_type = tam.get(b'type', b'<none>').decode('ascii', 'replace')
	171	+	if tam_type != 'HKDF_HMAC_SHA512':
	172	+	if tam_required:
	173	+	raise TAMUnsupportedSuiteError(repr(tam_type))
	174	+	else:
	175	+	logger.debug('Ignoring TAM made with unsupported suite, since TAM is not required: %r', tam_type)
	176	+	return unpacked, False
	177	+	tam_hmac = tam.get(b'hmac')
	178	+	tam_salt = tam.get(b'salt')
	179	+	if not isinstance(tam_salt, bytes) or not isinstance(tam_hmac, bytes):
	180	+	raise TAMInvalid()
	181	+	offset = data.index(tam_hmac)
	182	+	data[offset:offset + 64] = bytes(64)
	183	+	tam_key = self._tam_key(tam_salt, context=b'manifest')
	184	+	calculated_hmac = HMAC(tam_key, data, sha512).digest()
	185	+	if not compare_digest(calculated_hmac, tam_hmac):
	186	+	raise TAMInvalid()
	187	+	logger.debug('TAM-verified manifest')
	188	+	return unpacked, True
	189	+
85	190
86	191		class PlaintextKey(KeyBase):
87	192		TYPE = 0x02
88	193
89	194		chunk_seed = 0
90	195
	196	+	def __init__(self, repository):
	197	+	super().__init__(repository)
	198	+	self.tam_required = False
	199	+
91	200		@classmethod
92	201		def create(cls, repository, args):
93	202		logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey\|keyfile" to enable encryption.')

100	209		def id_hash(self, data):
101	210		return sha256(data).digest()
102	211
103		-	def encrypt(self, data):
104		-	return b''.join([self.TYPE_STR, self.compressor.compress(data)])
	212	+	def encrypt(self, data, none_compression=False):
	213	+	if none_compression:
	214	+	compressed = CNONE().compress(data)
	215	+	else:
	216	+	compressed = self.compressor.compress(data)
	217	+	return b''.join([self.TYPE_STR, compressed])
105	218
106	219		def decrypt(self, id, data):
107	220		if data[0] != self.TYPE:
108		-	raise IntegrityError('Invalid encryption envelope')
	221	+	id_str = bin_to_hex(id) if id is not None else '(unknown)'
	222	+	raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str)
109	223		data = self.compressor.decompress(memoryview(data)[1:])
110	224		if id and sha256(data).digest() != id:
111		-	raise IntegrityError('Chunk id verification failed')
	225	+	raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id))
112	226		return data
113	227
	228	+	def _tam_key(self, salt, context):
	229	+	return salt + context
	230	+
114	231
115	232		class AESKeyBase(KeyBase):
116	233		"""Common base class shared by KeyfileKey and PassphraseKey

132	249		"""
133	250		return HMAC(self.id_key, data, sha256).digest()
134	251
135		-	def encrypt(self, data):
136		-	data = self.compressor.compress(data)
	252	+	def encrypt(self, data, none_compression=False):
	253	+	if none_compression:
	254	+	data = CNONE().compress(data)
	255	+	else:
	256	+	data = self.compressor.compress(data)
137	257		self.enc_cipher.reset()
138	258		data = b''.join((self.enc_cipher.iv[8:], self.enc_cipher.encrypt(data)))
139	259		hmac = HMAC(self.enc_hmac_key, data, sha256).digest()

142	262		def decrypt(self, id, data):
143	263		if not (data[0] == self.TYPE or
144	264		data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
145		-	raise IntegrityError('Invalid encryption envelope')
	265	+	id_str = bin_to_hex(id) if id is not None else '(unknown)'
	266	+	raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str)
146	267		hmac_given = memoryview(data)[1:33]
147	268		hmac_computed = memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest())
148	269		if not compare_digest(hmac_computed, hmac_given):
149		-	raise IntegrityError('Encryption envelope checksum mismatch')
	270	+	id_str = bin_to_hex(id) if id is not None else '(unknown)'
	271	+	raise IntegrityError('Chunk %s: Encryption envelope checksum mismatch' % id_str)
150	272		self.dec_cipher.reset(iv=PREFIX + data[33:41])
151	273		data = self.compressor.decompress(self.dec_cipher.decrypt(data[41:]))
152	274		if id:
153	275		hmac_given = id
154	276		hmac_computed = HMAC(self.id_key, data, sha256).digest()
155	277		if not compare_digest(hmac_computed, hmac_given):
156		-	raise IntegrityError('Chunk id verification failed')
	278	+	raise IntegrityError('Chunk %s: Chunk id verification failed' % bin_to_hex(id))
157	279		return data
158	280
159	281		def extract_nonce(self, payload):
160	282		if not (payload[0] == self.TYPE or
161	283		payload[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
162		-	raise IntegrityError('Invalid encryption envelope')
	284	+	raise IntegrityError('Manifest: Invalid encryption envelope')
163	285		nonce = bytes_to_long(payload[33:41])
164	286		return nonce
165	287

190	312
191	313		@classmethod
192	314		def verification(cls, passphrase):
193		-	if yes('Do you want your passphrase to be displayed for verification? [yN]: ',
194		-	env_var_override='BORG_DISPLAY_PASSPHRASE'):
	315	+	msg = 'Do you want your passphrase to be displayed for verification? [yN]: '
	316	+	if yes(msg, retry_msg=msg, invalid_msg='Invalid answer, try again.',
	317	+	retry=True, env_var_override='BORG_DISPLAY_PASSPHRASE'):
195	318		print('Your passphrase (between double-quotes): "%s"' % passphrase,
196	319		file=sys.stderr)
197	320		print('Make sure the passphrase displayed above is exactly what you wanted.',

200	323		passphrase.encode('ascii')
201	324		except UnicodeEncodeError:
202	325		print('Your passphrase (UTF-8 encoding in hex): %s' %
203		-	hexlify(passphrase.encode('utf-8')).decode('ascii'),
	326	+	bin_to_hex(passphrase.encode('utf-8')),
204	327		file=sys.stderr)
205	328		print('As you have a non-ASCII passphrase, it is recommended to keep the UTF-8 encoding in hex together with the passphrase at a safe place.',
206	329		file=sys.stderr)

265	388		key.decrypt(None, manifest_data)
266	389		num_blocks = num_aes_blocks(len(manifest_data) - 41)
267	390		key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
	391	+	key._passphrase = passphrase
268	392		return key
269	393		except IntegrityError:
270	394		passphrase = Passphrase.getpass(prompt)

280	404		def init(self, repository, passphrase):
281	405		self.init_from_random_data(passphrase.kdf(repository.id, self.iterations, 100))
282	406		self.init_ciphers()
	407	+	self.tam_required = False
283	408
284	409
285	410		class KeyfileKeyBase(AESKeyBase):

303	428		raise PassphraseWrong
304	429		num_blocks = num_aes_blocks(len(manifest_data) - 41)
305	430		key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
	431	+	key._passphrase = passphrase
306	432		return key
307	433
308	434		def find_key(self):

323	449		self.enc_hmac_key = key[b'enc_hmac_key']
324	450		self.id_key = key[b'id_key']
325	451		self.chunk_seed = key[b'chunk_seed']
	452	+	self.tam_required = key.get(b'tam_required', tam_required(self.repository))
326	453		return True
327	454		return False
328	455

359	486		'enc_hmac_key': self.enc_hmac_key,
360	487		'id_key': self.id_key,
361	488		'chunk_seed': self.chunk_seed,
	489	+	'tam_required': self.tam_required,
362	490		}
363	491		data = self.encrypt_key_file(msgpack.packb(key), passphrase)
364	492		key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))
365	493		return key_data
366	494
367		-	def change_passphrase(self):
368		-	passphrase = Passphrase.new(allow_empty=True)
	495	+	def change_passphrase(self, passphrase=None):
	496	+	if passphrase is None:
	497	+	passphrase = Passphrase.new(allow_empty=True)
369	498		self.save(self.target, passphrase)
370		-	logger.info('Key updated')
371	499
372	500		@classmethod
373	501		def create(cls, repository, args):

426	554		def save(self, target, passphrase):
427	555		key_data = self._save(passphrase)
428	556		with open(target, 'w') as fd:
429		-	fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii')))
	557	+	fd.write('%s %s\n' % (self.FILE_ID, bin_to_hex(self.repository_id)))
430	558		fd.write(key_data)
431	559		fd.write('\n')
432	560		self.target = target

1		-	from binascii import hexlify
2	1		from contextlib import contextmanager
3		-	from datetime import datetime, timezone
	2	+	from datetime import datetime, timezone, timedelta
4	3		from getpass import getuser
5	4		from itertools import groupby
6	5		import errno

17	16		import time
18	17		from io import BytesIO
19	18		from . import xattr
20		-	from .helpers import Error, uid2user, user2uid, gid2group, group2gid, \
	19	+	from .helpers import Error, uid2user, user2uid, gid2group, group2gid, bin_to_hex, \
21	20		parse_timestamp, to_localtime, format_time, format_timedelta, remove_surrogates, \
22	21		Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \
23		-	ProgressIndicatorPercent
	22	+	ProgressIndicatorPercent, IntegrityError
24	23		from .platform import acl_get, acl_set
25	24		from .chunker import Chunker
26	25		from .hashindex import ChunkIndex

186	185		"""Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable."""
187	186
188	187		def __init__(self, repository, key, manifest, name, cache=None, create=False,
189		-	checkpoint_interval=300, numeric_owner=False, progress=False,
190		-	chunker_params=CHUNKER_PARAMS, start=None, end=None):
	188	+	checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, progress=False,
	189	+	chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None):
191	190		self.cwd = os.getcwd()
192	191		self.key = key
193	192		self.repository = repository

199	198		self.name = name
200	199		self.checkpoint_interval = checkpoint_interval
201	200		self.numeric_owner = numeric_owner
	201	+	self.noatime = noatime
	202	+	self.noctime = noctime
	203	+	assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.'
202	204		if start is None:
203	205		start = datetime.utcnow()
	206	+	start_monotonic = time.monotonic()
204	207		self.start = start
	208	+	self.start_monotonic = start_monotonic
205	209		if end is None:
206	210		end = datetime.utcnow()
207	211		self.end = end

211	215		self.chunker = Chunker(self.key.chunk_seed, *chunker_params)
212	216		if name in manifest.archives:
213	217		raise self.AlreadyExists(name)
214		-	self.last_checkpoint = time.time()
	218	+	self.last_checkpoint = time.monotonic()
215	219		i = 0
216	220		while True:
217	221		self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '')

Compare f32c885 ... +249 ... e5f7121

No flags found

Show failed CI and contexual commits c 251 Commits

Compare `f32c885 ... +249 ... e5f7121`

Show failed CI and contexual commits `c`

251 Commits

227	231
228	232		def _load_meta(self, id):
229	233		data = self.key.decrypt(id, self.repository.get(id))
230		-	metadata = msgpack.unpackb(data)
	234	+	metadata = msgpack.unpackb(data, unicode_errors='surrogateescape')
231	235		if metadata[b'version'] != 1:
232	236		raise Exception('Unknown archive metadata version')
233	237		return metadata

254	258
255	259		@property
256	260		def fpr(self):
257		-	return hexlify(self.id).decode('ascii')
	261	+	return bin_to_hex(self.id)
258	262
259	263		@property
260	264		def duration(self):

286	290		if self.show_progress:
287	291		self.stats.show_progress(item=item, dt=0.2)
288	292		self.items_buffer.add(item)
289		-	if time.time() - self.last_checkpoint > self.checkpoint_interval:
	293	+	if time.monotonic() - self.last_checkpoint > self.checkpoint_interval:
290	294		self.write_checkpoint()
291		-	self.last_checkpoint = time.time()
	295	+	self.last_checkpoint = time.monotonic()
292	296
293	297		def write_checkpoint(self):
294	298		self.save(self.checkpoint_name)

300	304		if name in self.manifest.archives:
301	305		raise self.AlreadyExists(name)
302	306		self.items_buffer.flush(flush=True)
	307	+	duration = timedelta(seconds=time.monotonic() - self.start_monotonic)
303	308		if timestamp is None:
304	309		self.end = datetime.utcnow()
	310	+	self.start = self.end - duration
305	311		start = self.start
306	312		end = self.end
307	313		else:
308	314		self.end = timestamp
309		-	start = timestamp
310		-	end = timestamp # we only have 1 value
	315	+	self.start = timestamp - duration
	316	+	end = timestamp
	317	+	start = self.start
311	318		metadata = StableDict({
312	319		'version': 1,
313	320		'name': name,

318	325		'time': start.isoformat(),
319	326		'time_end': end.isoformat(),
320	327		})
321		-	data = msgpack.packb(metadata, unicode_errors='surrogateescape')
	328	+	data = self.key.pack_and_authenticate_metadata(metadata, context=b'archive')
322	329		self.id = self.key.id_hash(data)
323	330		self.cache.add_chunk(self.id, data, self.stats)
324	331		self.manifest.archives[name] = {'id': self.id, 'time': metadata['time']}

522	529		try:
523	530		self.cache.chunk_decref(id, stats)
524	531		except KeyError:
525		-	cid = hexlify(id).decode('ascii')
	532	+	cid = bin_to_hex(id)
526	533		raise ChunksIndexError(cid)
527	534		except Repository.ObjectNotFound as e:
528	535		# object not in repo - strange, but we wanted to delete it anyway.

572	579		b'mode': st.st_mode,
573	580		b'uid': st.st_uid, b'user': uid2user(st.st_uid),
574	581		b'gid': st.st_gid, b'group': gid2group(st.st_gid),
575		-	b'atime': int_to_bigint(st.st_atime_ns),
576		-	b'ctime': int_to_bigint(st.st_ctime_ns),
577	582		b'mtime': int_to_bigint(st.st_mtime_ns),
578	583		}
	584	+	# borg can work with archives only having mtime (older attic archives do not have
	585	+	# atime/ctime). it can be useful to omit atime/ctime, if they change without the
	586	+	# file content changing - e.g. to get better metadata deduplication.
	587	+	if not self.noatime:
	588	+	item[b'atime'] = int_to_bigint(st.st_atime_ns)
	589	+	if not self.noctime:
	590	+	item[b'ctime'] = int_to_bigint(st.st_ctime_ns)
579	591		if self.numeric_owner:
580	592		item[b'user'] = item[b'group'] = None
581	593		with backup_io():

610	622		return 'b' # block device
611	623
612	624		def process_symlink(self, path, st):
613		-	source = os.readlink(path)
	625	+	with backup_io():
	626	+	source = os.readlink(path)
614	627		item = {b'path': make_path_safe(path), b'source': source}
615	628		item.update(self.stat_attrs(st, path))
616	629		self.add_item(item)

641	654		# Is it a hard link?
642	655		if st.st_nlink > 1:
643	656		source = self.hard_links.get((st.st_ino, st.st_dev))
644		-	if (st.st_ino, st.st_dev) in self.hard_links:
	657	+	if source is not None:
645	658		item = self.stat_attrs(st, path)
646	659		item.update({b'path': safe_path, b'source': source})
647	660		self.add_item(item)
648	661		status = 'h' # regular file, hardlink (to already seen inodes)
649	662		return status
650		-	else:
651		-	self.hard_links[st.st_ino, st.st_dev] = safe_path
652	663		is_special_file = is_special(st.st_mode)
653	664		if not is_special_file:
654	665		path_hash = self.key.id_hash(os.path.join(self.cwd, path).encode('utf-8', 'surrogateescape'))

696	707		item[b'mode'] = stat.S_IFREG \| stat.S_IMODE(item[b'mode'])
697	708		self.stats.nfiles += 1
698	709		self.add_item(item)
	710	+	if st.st_nlink > 1 and source is None:
	711	+	# Add the hard link reference after the file has been added to the archive.
	712	+	self.hard_links[st.st_ino, st.st_dev] = safe_path
699	713		return status
700	714
701	715		@staticmethod