datalad / datalad

Compare 5cb174b ... +18 ... c02ad49

Coverage Reach
support/tests/test_annexrepo.py support/tests/test_gitrepo.py support/tests/test_network.py support/tests/test_sshconnector.py support/tests/test_external_versions.py support/tests/test_fileinfo.py support/tests/test_stats.py support/tests/test_github_.py support/tests/test_ansi_colors.py support/tests/test_repodates.py support/tests/test_globbedpaths.py support/tests/test_sshrun.py support/tests/test_locking.py support/tests/test_due_utils.py support/tests/test_versions.py support/tests/test_path.py support/tests/test_repo_save.py support/tests/test_json_py.py support/tests/utils.py support/tests/test_vcr_.py support/tests/test_status.py support/tests/test_cookies.py support/tests/test_cache.py support/tests/test_digests.py support/tests/__init__.py support/gitrepo.py support/annexrepo.py support/network.py support/third/noseclasses.py support/third/nosetester.py support/third/nda_aws_token_generator.py support/third/loris_token_generator.py support/s3.py support/sshconnector.py support/constraints.py support/archives.py support/github_.py support/exceptions.py support/external_versions.py support/protocol.py support/_lru_cache2.py support/repodates.py support/json_py.py support/path.py support/globbedpaths.py support/archive_utils_patool.py support/stats.py support/versions.py support/cookies.py support/vcr_.py support/param.py support/repo.py support/keyring_.py support/configparserinc.py support/due_utils.py support/sshrun.py support/locking.py support/status.py support/ansi_colors.py support/due.py support/archive_utils_7z.py support/digests.py support/strings.py support/nda_.py support/cache.py support/__init__.py interface/tests/test_rerun.py interface/tests/test_rerun_merges.py interface/tests/test_save.py interface/tests/test_add_archive_content.py interface/tests/test_utils.py interface/tests/test_annotate_paths.py interface/tests/test_ls_webui.py interface/tests/test_run_procedure.py interface/tests/test_diff.py interface/tests/test_download_url.py interface/tests/test_unlock.py interface/tests/test_ls.py interface/tests/test_base.py interface/tests/test_docs.py interface/tests/test_clean.py interface/tests/__init__.py interface/ls.py interface/base.py interface/rerun.py interface/annotate_paths.py interface/utils.py interface/add_archive_content.py interface/ls_webui.py interface/run_procedure.py interface/diff.py interface/save.py interface/results.py interface/download_url.py interface/unlock.py interface/clean.py interface/common_opts.py interface/test.py interface/common_cfg.py interface/__init__.py interface/run.py distribution/tests/test_install.py distribution/tests/test_publish.py distribution/tests/test_uninstall.py distribution/tests/test_create_sibling.py distribution/tests/test_dataset.py distribution/tests/test_get.py distribution/tests/test_add.py distribution/tests/test_update.py distribution/tests/test_siblings.py distribution/tests/test_create_github.py distribution/tests/test_create_test_dataset.py distribution/tests/test_utils.py distribution/tests/test_dataset_binding.py distribution/tests/test_dataset_api.py distribution/tests/__init__.py distribution/siblings.py distribution/create_sibling.py distribution/publish.py distribution/dataset.py distribution/get.py distribution/add.py distribution/remove.py distribution/update.py distribution/install.py distribution/create_test_dataset.py distribution/drop.py distribution/uninstall.py distribution/create_sibling_github.py distribution/utils.py distribution/clone.py distribution/__init__.py distribution/subdatasets.py tests/utils.py tests/test_utils.py tests/test_tests_utils.py tests/test_config.py tests/test_cmd.py tests/test_auto.py tests/test_constraints.py tests/test_archives.py tests/utils_testrepos.py tests/test_protocols.py tests/test_log.py tests/test_dochelpers.py tests/test_interface.py tests/test_direct_mode.py tests/test_s3.py tests/test_version.py tests/test_api.py tests/utils_testdatasets.py tests/test_base.py tests/test_utils_testrepos.py tests/test_testrepos.py tests/test__main__.py tests/test_misc.py tests/test_installed.py tests/test_strings.py tests/heavyoutput.py tests/__init__.py core/local/tests/test_save.py core/local/tests/test_run.py core/local/tests/test_create.py core/local/tests/test_diff.py core/local/tests/test_status.py core/local/tests/test_resulthooks.py core/local/run.py core/local/create.py core/local/status.py core/local/diff.py core/local/save.py core/local/resulthooks.py core/local/__init__.py core/distributed/tests/test_clone.py core/distributed/clone.py core/__init__.py metadata/extractors/tests/test_base.py metadata/extractors/tests/test_audio.py metadata/extractors/tests/test_xmp.py metadata/extractors/tests/test_image.py metadata/extractors/tests/test_exif.py metadata/extractors/tests/test_datacite_xml.py metadata/extractors/tests/test_frictionless_datapackage.py metadata/extractors/tests/test_rfc822.py metadata/extractors/tests/__init__.py metadata/extractors/datalad_rfc822.py metadata/extractors/xmp.py metadata/extractors/frictionless_datapackage.py metadata/extractors/datacite.py metadata/extractors/datalad_core.py metadata/extractors/audio.py metadata/extractors/exif.py metadata/extractors/annex.py metadata/extractors/image.py metadata/extractors/base.py metadata/extractors/__init__.py metadata/tests/test_aggregation.py metadata/tests/test_base.py metadata/tests/test_search.py metadata/tests/test_extract_metadata.py metadata/search.py metadata/metadata.py metadata/aggregate.py metadata/extract_metadata.py metadata/definitions.py metadata/__init__.py downloaders/tests/test_http.py downloaders/tests/test_providers.py downloaders/tests/test_credentials.py downloaders/tests/test_s3.py downloaders/tests/utils.py downloaders/tests/test_base.py downloaders/base.py downloaders/http.py downloaders/providers.py downloaders/credentials.py downloaders/s3.py downloaders/__init__.py plugin/tests/test_addurls.py plugin/tests/test_plugins.py plugin/tests/test_export_archive.py plugin/tests/test_check_dates.py plugin/tests/__init__.py plugin/addurls.py plugin/wtf.py plugin/export_to_figshare.py plugin/export_archive.py plugin/check_dates.py plugin/add_readme.py plugin/no_annex.py plugin/__init__.py utils.py customremotes/base.py customremotes/tests/test_archives.py customremotes/tests/test_base.py customremotes/tests/test_datalad.py customremotes/tests/__init__.py customremotes/archives.py customremotes/datalad.py customremotes/main.py customremotes/__init__.py cmdline/main.py cmdline/tests/test_main.py cmdline/tests/test_helpers.py cmdline/tests/test_formatters.py cmdline/tests/__init__.py cmdline/helpers.py cmdline/common_args.py cmdline/__init__.py ui/dialog.py ui/progressbars.py ui/tests/test_dialog.py ui/tests/test_base.py ui/tests/__init__.py ui/__init__.py ui/utils.py ui/base.py cmd.py config.py distributed/create_sibling_gitlab.py distributed/tests/test_create_sibling_gitlab.py distributed/__init__.py local/subdatasets.py local/tests/test_subdataset.py local/__init__.py log.py auto.py dochelpers.py __init__.py api.py __main__.py consts.py version.py coreapi.py

No flags found

Use flags to group coverage reports by test type, project and/or folders.
Then setup custom commit statuses and notifications for each flag.

e.g., #unittest #integration

#production #enterprise

#frontend #backend

Learn more about Codecov Flags here.


@@ -271,7 +271,9 @@
Loading
271 271
    """
272 272
    d = dict(
273 273
        {'dlm_progress_{}'.format(n): v for n, v in kwargs.items()
274 -
         if v},
274 +
         # initial progress might be zero, but not sending it further
275 +
         # would signal to destroy the progress bar, hence test for 'not None'
276 +
         if v is not None},
275 277
        dlm_progress=pid)
276 278
    lgrcall(*args, extra=d)
277 279

@@ -98,7 +98,185 @@
Loading
98 98
        std.close()
99 99
100 100
101 -
class Runner(object):
101 +
class LeanRunner(object):
102 +
    """Minimal Runner with support for online command output processing
103 +
104 +
    It aims to be as simple as possible, providing only essential
105 +
    functionality. Derived classes should be used for additional
106 +
    specializations and convenience features.
107 +
    """
108 +
    __slots__ = ['cwd', 'env', '_poll_period']
109 +
110 +
    def __init__(self, cwd=None, env=None, poll_period=0.1):
111 +
        """
112 +
        Parameters
113 +
        ----------
114 +
        cwd : path-like, optional
115 +
          If given, commands are executed with this path as PWD,
116 +
          the PWD of the parent process is used otherwise.
117 +
        env : dict, optional
118 +
          Environment to be pass to subprocess.Popen(). If `cwd`
119 +
          was given, 'PWD' in the environment is set to its value.
120 +
          This must be a complete environment definition, no values
121 +
          from the current environment will be inherited.
122 +
        poll_period : float, optional
123 +
          Interval at which the running process is queried for
124 +
          output.
125 +
        """
126 +
        self._poll_period = poll_period
127 +
        self.env = env.copy() if env else None
128 +
        # stringify to support Path instances on PY35
129 +
        self.cwd = str(cwd) if cwd is not None else None
130 +
        if cwd and env is not None:
131 +
            # if CWD was provided, we must not make it conflict with
132 +
            # a potential PWD setting
133 +
            self.env['PWD'] = cwd
134 +
135 +
    def run(self, cmd, proc_stdout=None, proc_stderr=None, stdin=None):
136 +
        """Execute a command and communicate with it.
137 +
138 +
        Parameters
139 +
        ----------
140 +
        cmd : list
141 +
          Sequence of program arguments. Passing a single string means
142 +
          that it is simply the name of the program, no complex shell
143 +
          commands are supported.
144 +
        proc_stdout : callable, optional
145 +
          If given, all stdout is passed as byte-string to this callable,
146 +
          in the chunks it was received by polling the processing. The
147 +
          callable may transform it in any way, its output (byte-string)
148 +
          is concatenated and provided as stdout return value.
149 +
        proc_stderr : callable, optional
150 +
          Like proc_stdout, but for stderr.
151 +
        stdin : byte stream, optional
152 +
          File descriptor like, used as stdin for the process. Passed
153 +
          verbatim to subprocess.Popen().
154 +
155 +
        Returns
156 +
        -------
157 +
        stdout, stderr
158 +
          Unicode string with the cumulative standard output and error
159 +
          of the process.
160 +
161 +
        Raises
162 +
        ------
163 +
        CommandError
164 +
          On executation failure (non-zero exit code) this exception is
165 +
          raised which provides the command (cmd), stdout, stderr,
166 +
          exit code (status), and a message identifying the failed
167 +
          command, as properties.
168 +
        """
169 +
        proc_out = (proc_stdout, proc_stderr)
170 +
        if all(p is None for p in proc_out):
171 +
            proc_out = None
172 +
        try:
173 +
            lgr.log(8, "Start running %r", cmd)
174 +
            process = subprocess.Popen(
175 +
                cmd,
176 +
                # from PY37 onwards
177 +
                #capture_output=proc_output,
178 +
                stdout=subprocess.PIPE if proc_out else None,
179 +
                stderr=subprocess.PIPE if proc_out else None,
180 +
                shell=False,
181 +
                cwd=self.cwd,
182 +
                env=self.env,
183 +
                stdin=stdin,
184 +
                # intermediate reports are never decoded anyways
185 +
                # from PY37 onwards
186 +
                #text=False,
187 +
                universal_newlines=False,
188 +
            )
189 +
        except Exception as e:
190 +
            lgr.log(11, "Failed to start %r%r: %s" %
191 +
                    (cmd,
192 +
                     (" under %r" % self.cwd) if self.cwd else '',
193 +
                     exc_str(e)))
194 +
            raise
195 +
196 +
        try:
197 +
            out = [b'', b'']
198 +
            last_idx = [0, 0]
199 +
            pout = None
200 +
            # make sure to run this loop at least once, even if the
201 +
            # process is already dead
202 +
            while process.poll() is None or pout is None:
203 +
                try:
204 +
                    # get a chunk of output for the specific period
205 +
                    # of time
206 +
                    pout = process.communicate(
207 +
                        timeout=self._poll_period,
208 +
                    )
209 +
                except subprocess.TimeoutExpired as poll:
210 +
                    # this will always be the full report so far,
211 +
                    # not just an output increment
212 +
                    pout = (poll.stdout, poll.stderr)
213 +
214 +
                if proc_out is not None:
215 +
                    for i, (o, proc) in enumerate(zip(pout, proc_out)):
216 +
                        if proc is None:
217 +
                            if o is not None:
218 +
                                out[i] += o
219 +
                            # no index update needed, can never change
220 +
                            continue
221 +
                        # current length of output
222 +
                        len_o = len(o) if o else 0
223 +
                        # anything new in the output?
224 +
                        if len_o - last_idx[i] > 0:
225 +
                            # engage output processed
226 +
                            processed, unprocessed_len = proc(o[last_idx[i]:])
227 +
                            # only keep the processed output
228 +
                            out[i] += processed
229 +
                            # record point to which we processed the output
230 +
                            # subtract number of bytes reported as unprocessed
231 +
                            last_idx[i] = len_o - unprocessed_len
232 +
233 +
            # obtain exit code
234 +
            status = process.poll()
235 +
236 +
            # decode bytes to string
237 +
            out = tuple(o.decode('utf-8') if o else '' for o in out)
238 +
239 +
            if status not in [0, None]:
240 +
                msg = "Failed to run %r%s." % (
241 +
                    cmd,
242 +
                    (" under %r" % self.cwd) if self.cwd else '',
243 +
                )
244 +
                raise CommandError(
245 +
                    cmd=str(cmd),
246 +
                    msg=msg,
247 +
                    code=status,
248 +
                    stdout=out[0],
249 +
                    stderr=out[1],
250 +
                )
251 +
            else:
252 +
                lgr.log(8, "Finished running %r with status %s", cmd, status)
253 +
254 +
        except CommandError:
255 +
            # do not bother with reacting to "regular" CommandError
256 +
            # exceptions.  Somehow if we also terminate here for them
257 +
            # some processes elsewhere might stall:
258 +
            # see https://github.com/datalad/datalad/pull/3794
259 +
            raise
260 +
261 +
        except BaseException as exc:
262 +
            exc_info = sys.exc_info()
263 +
            # KeyboardInterrupt is subclass of BaseException
264 +
            lgr.debug("Terminating process for %s upon exception: %s",
265 +
                      cmd, exc_str(exc))
266 +
            try:
267 +
                # there are still possible (although unlikely) cases when
268 +
                # we fail to interrupt but we
269 +
                # should not crash if we fail to terminate the process
270 +
                process.terminate()
271 +
            except BaseException as exc2:
272 +
                lgr.warning("Failed to terminate process for %s: %s",
273 +
                            cmd, exc_str(exc2))
274 +
            raise exc_info[1]
275 +
276 +
        return out
277 +
278 +
279 +
class Runner(LeanRunner):
102 280
    """Provides a wrapper for calling functions and commands.
103 281
104 282
    An object of this class provides a methods that calls shell commands or
@@ -131,9 +309,8 @@
Loading
131 309
             Switch to instruct whether outputs should be logged or not.  If not
132 310
             set (default), config 'datalad.log.outputs' would be consulted
133 311
        """
312 +
        super(Runner, self).__init__(cwd=cwd, env=env)
134 313
135 -
        self.cwd = cwd
136 -
        self.env = env
137 314
        if protocol is None:
138 315
            # TODO: config cmd.protocol = null
139 316
            protocol_str = os.environ.get('DATALAD_CMD_PROTOCOL', 'null')

@@ -53,10 +53,12 @@
Loading
53 53
    NoSuchPathError,
54 54
    InvalidGitRepositoryError
55 55
)
56 +
from datalad.log import log_progress
56 57
from datalad.support.due import due, Doi
57 58
58 59
from datalad import ssh_manager
59 60
from datalad.cmd import (
61 +
    LeanRunner,
60 62
    GitRunner,
61 63
    BatchedCommand
62 64
)
@@ -74,14 +76,14 @@
Loading
74 76
from datalad.utils import (
75 77
    Path,
76 78
    PurePosixPath,
77 -
    assure_list,
79 +
    ensure_list,
78 80
    optional_args,
79 81
    on_windows,
80 82
    getpwd,
81 83
    posix_relpath,
82 -
    assure_dir,
84 +
    ensure_dir,
83 85
    generate_file_chunks,
84 -
    assure_unicode,
86 +
    ensure_unicode,
85 87
    split_cmdline,
86 88
)
87 89
@@ -416,6 +418,208 @@
Loading
416 418
    return wrapped
417 419
418 420
421 +
class GitProgress(object):
422 +
    """Reduced variant of GitPython's RemoteProgress class
423 +
    """
424 +
    _num_op_codes = 10
425 +
    BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING, FINDING_SOURCES, CHECKING_OUT, ENUMERATING = \
426 +
        [1 << x for x in range(_num_op_codes)]
427 +
    STAGE_MASK = BEGIN | END
428 +
    OP_MASK = ~STAGE_MASK
429 +
430 +
    DONE_TOKEN = 'done.'
431 +
    TOKEN_SEPARATOR = ', '
432 +
433 +
    _known_ops = {
434 +
        COUNTING: ("Counting", "Objects"),
435 +
        ENUMERATING: ("Enumerating", "Objects"),
436 +
        COMPRESSING: ("Compressing", "Objects"),
437 +
        WRITING: ("Writing", "Objects"),
438 +
        RECEIVING: ("Receiving", "Objects"),
439 +
        RESOLVING: ("Resolving", "Deltas"),
440 +
        FINDING_SOURCES: ("Finding", "Sources"),
441 +
        CHECKING_OUT: ("Check out", "Things"),
442 +
    }
443 +
444 +
    __slots__ = ('_seen_ops', '_pbars')
445 +
446 +
    re_op_absolute = re.compile(r"(remote: )?([\w\s]+):\s+()(\d+)()(.*)")
447 +
    re_op_relative = re.compile(r"(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)")
448 +
449 +
    def __init__(self):
450 +
        self.__enter__()
451 +
452 +
    def __enter__(self):
453 +
        self._seen_ops = []
454 +
        self._pbars = set()
455 +
        return self
456 +
457 +
    def __exit__(self, exc_type, exc_value, traceback):
458 +
        # take down any progress bars that were not closed orderly
459 +
        for pbar_id in self._pbars:
460 +
            log_progress(
461 +
                lgr.info,
462 +
                pbar_id,
463 +
                'Finished',
464 +
            )
465 +
466 +
    def __call__(self, byts):
467 +
        """Callable interface compatible with LeanRunner()
468 +
469 +
        Parameters
470 +
        ----------
471 +
        byts : bytes
472 +
          One or more lines of command output.
473 +
474 +
        Returns
475 +
        -------
476 +
        bytes
477 +
          All input in its orignal form, excepted for lines that
478 +
          were identified as recognized progress reports.
479 +
        """
480 +
        keep_lines = []
481 +
        for line in byts.splitlines(keepends=True):
482 +
            if not self._parse_progress_line(line):
483 +
                keep_lines.append(line)
484 +
        # the zero indicated that no data remained unprocessed at the
485 +
        # end of the input
486 +
        # TODO reevaluate whether it is useful to keep this feature
487 +
        return b''. join(keep_lines), 0
488 +
489 +
    def _parse_progress_line(self, line):
490 +
        """Process a single line
491 +
492 +
        Parameters
493 +
        ----------
494 +
        line : bytes
495 +
496 +
        Returns
497 +
        -------
498 +
        bool
499 +
          Flag whether the line was recognized as a Git progress report.
500 +
        """
501 +
        # handle
502 +
        # Counting objects: 4, done.
503 +
        # Compressing objects:  50% (1/2)
504 +
        # Compressing objects: 100% (2/2)
505 +
        # Compressing objects: 100% (2/2), done.
506 +
        line = line.decode('utf-8') if isinstance(line, bytes) else line
507 +
        if line.startswith(('error:', 'fatal:')):
508 +
            return False
509 +
510 +
        # find escape characters and cut them away - regex will not work with
511 +
        # them as they are non-ascii. As git might expect a tty, it will send them
512 +
        last_valid_index = None
513 +
        for i, c in enumerate(reversed(line)):
514 +
            if ord(c) < 32:
515 +
                # its a slice index
516 +
                last_valid_index = -i - 1
517 +
            # END character was non-ascii
518 +
        # END for each character in line
519 +
        if last_valid_index is not None:
520 +
            line = line[:last_valid_index]
521 +
        # END cut away invalid part
522 +
        line = line.rstrip()
523 +
524 +
        cur_count, max_count = None, None
525 +
        match = self.re_op_relative.match(line)
526 +
        if match is None:
527 +
            match = self.re_op_absolute.match(line)
528 +
529 +
        if not match:
530 +
            return False
531 +
        # END could not get match
532 +
533 +
        op_code = 0
534 +
        _remote, op_name, _percent, cur_count, max_count, message = match.groups()
535 +
536 +
        # get operation id
537 +
        if op_name == "Counting objects":
538 +
            op_code |= self.COUNTING
539 +
        elif op_name == "Compressing objects":
540 +
            op_code |= self.COMPRESSING
541 +
        elif op_name == "Writing objects":
542 +
            op_code |= self.WRITING
543 +
        elif op_name == 'Receiving objects':
544 +
            op_code |= self.RECEIVING
545 +
        elif op_name == 'Resolving deltas':
546 +
            op_code |= self.RESOLVING
547 +
        elif op_name == 'Finding sources':
548 +
            op_code |= self.FINDING_SOURCES
549 +
        elif op_name == 'Checking out files':
550 +
            op_code |= self.CHECKING_OUT
551 +
        elif op_name == 'Enumerating objects':
552 +
            op_code |= self.ENUMERATING
553 +
        else:
554 +
            # Note: On windows it can happen that partial lines are sent
555 +
            # Hence we get something like "CompreReceiving objects", which is
556 +
            # a blend of "Compressing objects" and "Receiving objects".
557 +
            # This can't really be prevented.
558 +
            lgr.debug(
559 +
                'Output line matched a progress report of an unknown type: %s',
560 +
                line)
561 +
            # TODO investigate if there is any chance that we might swallow
562 +
            # important info -- until them do not flag this line
563 +
            # as progress
564 +
            return False
565 +
        # END handle op code
566 +
567 +
        pbar_id = 'gitprogress-{}-{}'.format(id(self), op_code)
568 +
569 +
        op_props = self._known_ops[op_code]
570 +
571 +
        # figure out stage
572 +
        if op_code not in self._seen_ops:
573 +
            self._seen_ops.append(op_code)
574 +
            op_code |= self.BEGIN
575 +
            log_progress(
576 +
                lgr.info,
577 +
                pbar_id,
578 +
                'Start {} {}'.format(
579 +
                    op_props[0].lower(),
580 +
                    op_props[1].lower(),
581 +
                ),
582 +
                label=op_props[0],
583 +
                unit=' {}'.format(op_props[1]),
584 +
                total=float(max_count) if max_count else None,
585 +
            )
586 +
            self._pbars.add(pbar_id)
587 +
        # END begin opcode
588 +
589 +
        if message is None:
590 +
            message = ''
591 +
        # END message handling
592 +
593 +
        done_progress = False
594 +
        message = message.strip()
595 +
        if message.endswith(self.DONE_TOKEN):
596 +
            op_code |= self.END
597 +
            message = message[:-len(self.DONE_TOKEN)]
598 +
            done_progress = True
599 +
        # END end message handling
600 +
        message = message.strip(self.TOKEN_SEPARATOR)
601 +
602 +
        if cur_count and max_count:
603 +
            log_progress(
604 +
                lgr.info,
605 +
                pbar_id,
606 +
                message,
607 +
                update=float(cur_count),
608 +
            )
609 +
610 +
        if done_progress:
611 +
            log_progress(
612 +
                lgr.info,
613 +
                pbar_id,
614 +
                'Finished {} {}'.format(
615 +
                    op_props[0].lower(),
616 +
                    op_props[1].lower(),
617 +
                ),
618 +
            )
619 +
            self._pbars.discard(pbar_id)
620 +
        return True
621 +
622 +
419 623
class GitPythonProgressBar(RemoteProgress):
420 624
    """A handler for Git commands interfaced by GitPython which report progress
421 625
    """
@@ -865,13 +1069,14 @@
Loading
865 1069
            ssh_manager.get_connection(url).open()
866 1070
            # TODO: with git <= 2.3 keep old mechanism:
867 1071
            #       with rm.repo.git.custom_environment(GIT_SSH="wrapper_script"):
868 -
            env = GitRepo.GIT_SSH_ENV
1072 +
            env = os.environ.copy()
1073 +
            env.update(GitRepo.GIT_SSH_ENV)
869 1074
        else:
870 1075
            if isinstance(url_ri, PathRI):
1076 +
                # expand user, because execution not going through a shell
1077 +
                # doesn't work well otherwise
871 1078
                new_url = os.path.expanduser(url)
872 1079
                if url != new_url:
873 -
                    # TODO: remove whenever GitPython is fixed:
874 -
                    # https://github.com/gitpython-developers/GitPython/issues/731
875 1080
                    lgr.info("Expanded source path to %s from %s", new_url, url)
876 1081
                    url = new_url
877 1082
            env = None
@@ -881,24 +1086,17 @@
Loading
881 1086
        for trial in range(ntries):
882 1087
            try:
883 1088
                lgr.debug("Git clone from {0} to {1}".format(url, path))
884 -
                with GitPythonProgressBar("Cloning") as git_progress:
885 -
                    repo = gitpy.Repo.clone_from(
886 -
                        url, path,
887 -
                        env=env,
888 -
                        # we accept a plain dict with options, and not a gitpy
889 -
                        # tailored list of "multi options" to make a future
890 -
                        # non-GitPy based implementation easier. Do conversion
891 -
                        # here
892 -
                        multi_options=to_options(**clone_options) if clone_options else None,
893 -
                        odbt=default_git_odbt,
894 -
                        progress=git_progress
895 -
                    )
896 -
                # Note/TODO: signature for clone from:
897 -
                # (url, to_path, progress=None, env=None, **kwargs)
898 1089
1090 +
                # TODO bring back progress reporting
1091 +
                with GitProgress() as progress:
1092 +
                    LeanRunner(env=env).run(
1093 +
                        ['git', 'clone', '--progress', url, path] \
1094 +
                        + (to_options(**clone_options) if clone_options else []),
1095 +
                        proc_stderr=progress,
1096 +
                    )
899 1097
                lgr.debug("Git clone completed")
900 1098
                break
901 -
            except GitCommandError as e:
1099 +
            except CommandError as e:
902 1100
                # log here but let caller decide what to do
903 1101
                e_str = exc_str(e)
904 1102
                # see https://github.com/datalad/datalad/issues/785
@@ -918,18 +1116,8 @@
Loading
918 1116
919 1117
                raise
920 1118
921 -
            except ValueError as e:
922 -
                if gitpy.__version__ == '1.0.2' \
923 -
                        and "I/O operation on closed file" in str(e):
924 -
                    # bug https://github.com/gitpython-developers/GitPython
925 -
                    # /issues/383
926 -
                    raise GitCommandError(
927 -
                        "clone has failed, telling ya",
928 -
                        999,  # good number
929 -
                        stdout="%s already exists" if exists(path) else "")
930 -
                raise  # reraise original
931 -
932 -
        gr = cls(path, *args, repo=repo, **kwargs)
1119 +
        # get ourselves a repository instance
1120 +
        gr = cls(path, *args, **kwargs)
933 1121
        if fix_annex:
934 1122
            # cheap check whether we deal with an AnnexRepo - we can't check the class of `gr` itself, since we then
935 1123
            # would need to import our own subclass
@@ -1174,7 +1362,7 @@
Loading
1174 1362
        # there is no other way then to collect all files into a list
1175 1363
        # at this point, because we need to pass them at once to a single
1176 1364
        # `git add` call
1177 -
        files = [_normalize_path(self.path, f) for f in assure_list(files) if f]
1365 +
        files = [_normalize_path(self.path, f) for f in ensure_list(files) if f]
1178 1366
1179 1367
        if not (files or git_options or update):
1180 1368
            # wondering why just a warning? in cmdline this is also not an error
@@ -1188,7 +1376,7 @@
Loading
1188 1376
                # Set annex.largefiles to prevent storing files in annex when
1189 1377
                # GitRepo() is instantiated with a v6+ annex repo.
1190 1378
                ['git', '-c', 'annex.largefiles=nothing', 'add'] +
1191 -
                assure_list(git_options) +
1379 +
                ensure_list(git_options) +
1192 1380
                to_options(update=update) + ['--verbose']
1193 1381
            )
1194 1382
            # get all the entries
@@ -1230,7 +1418,7 @@
Loading
1230 1418
        modes when ran through proxy
1231 1419
        """
1232 1420
        return [{u'file': f, u'success': True}
1233 -
                for f in re.findall("'(.*)'[\n$]", assure_unicode(stdout))]
1421 +
                for f in re.findall("'(.*)'[\n$]", ensure_unicode(stdout))]
1234 1422
1235 1423
    @normalize_paths(match_return_type=False)
1236 1424
    def remove(self, files, recursive=False, **kwargs):
@@ -1331,7 +1519,7 @@
Loading
1331 1519
        """
1332 1520
        if not fields:
1333 1521
            raise ValueError('no `fields` provided, refuse to proceed')
1334 -
        fields = assure_list(fields)
1522 +
        fields = ensure_list(fields)
1335 1523
        cmd = [
1336 1524
            "git",
1337 1525
            "for-each-ref",
@@ -1344,10 +1532,10 @@
Loading
1344 1532
        if contains:
1345 1533
            cmd.append('--contains={}'.format(contains))
1346 1534
        if sort:
1347 -
            for k in assure_list(sort):
1535 +
            for k in ensure_list(sort):
1348 1536
                cmd.append('--sort={}'.format(k))
1349 1537
        if pattern:
1350 -
            cmd += assure_list(pattern)
1538 +
            cmd += ensure_list(pattern)
1351 1539
        if count:
1352 1540
            cmd.append('--count={:d}'.format(count))
1353 1541
@@ -3028,7 +3216,7 @@
Loading
3028 3216
          name and value items. Attribute values are either True or False,
3029 3217
          for set and unset attributes, or are the literal attribute value.
3030 3218
        """
3031 -
        path = assure_list(path)
3219 +
        path = ensure_list(path)
3032 3220
        cmd = ["git", "check-attr", "-z", "--all"]
3033 3221
        if index_only:
3034 3222
            cmd.append('--cached')
@@ -3926,7 +4114,7 @@
Loading
3926 4114
            # without --verbose git 2.9.3  add does not return anything
3927 4115
            add_out = self._git_custom_command(
3928 4116
                list(files.keys()),
3929 -
                ['git', 'add'] + assure_list(git_opts) + ['--verbose']
4117 +
                ['git', 'add'] + ensure_list(git_opts) + ['--verbose']
3930 4118
            )
3931 4119
            # get all the entries
3932 4120
            for r in self._process_git_get_output(*add_out):
@@ -3983,7 +4171,7 @@
Loading
3983 4171
    src_dotgit = str(repo.dot_git)
3984 4172
    # move .git
3985 4173
    from os import rename, listdir, rmdir
3986 -
    assure_dir(subds_dotgit)
4174 +
    ensure_dir(subds_dotgit)
3987 4175
    for dot_git_entry in listdir(src_dotgit):
3988 4176
        rename(opj(src_dotgit, dot_git_entry),
3989 4177
               opj(subds_dotgit, dot_git_entry))

@@ -13,7 +13,6 @@
Loading
13 13
from os import linesep
14 14
15 15
16 -
17 16
class CommandError(RuntimeError):
18 17
    """Thrown if a command call fails.
19 18
    """
@@ -27,13 +26,17 @@
Loading
27 26
        self.stderr = stderr
28 27
29 28
    def __str__(self):
30 -
        from datalad.utils import assure_unicode
29 +
        from datalad.utils import ensure_unicode
31 30
        to_str = "%s: " % self.__class__.__name__
32 31
        if self.cmd:
33 32
            to_str += "command '%s'" % (self.cmd,)
34 33
        if self.code:
35 34
            to_str += " failed with exitcode %d" % self.code
36 -
        to_str += "\n%s" % assure_unicode(self.msg)
35 +
        to_str += "\n{}\nstdout={}\nstderr={}".format(
36 +
            ensure_unicode(self.msg),
37 +
            ensure_unicode(self.stdout),
38 +
            ensure_unicode(self.stderr),
39 +
        )
37 40
        return to_str
38 41
39 42

@@ -16,6 +16,7 @@
Loading
16 16
from functools import partial
17 17
from glob import glob
18 18
import os
19 +
import re
19 20
from os import mkdir
20 21
from os.path import join as opj
21 22
from os.path import basename
@@ -258,7 +259,7 @@
Loading
258 259
        ar.get("file")
259 260
    exc = cme.exception
260 261
    eq_(exc.sizemore_msg, '905.6 MB')
261 -
    assert_re_in(".*annex (find|get). needs 905.6 MB more", str(exc))
262 +
    assert_re_in(".*annex (find|get).*needs 905.6 MB more", str(exc), re.DOTALL)
262 263
263 264
264 265
# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789014#step:8:405

Click to load this diff.
Loading diff...

Learn more Showing 9 files with coverage changes found.

Changes in datalad/core/distributed/clone.py
-1
+2
Loading file...
Changes in datalad/distribution/tests/test_create_github.py
-6
Loading file...
Changes in datalad/support/gitrepo.py
-2
+2
Loading file...
Changes in datalad/support/external_versions.py
+1
Loading file...
Changes in datalad/config.py
+7
Loading file...
Changes in datalad/tests/test_config.py
+43
Loading file...
Changes in datalad/core/distributed/tests/test_clone.py
+19
Loading file...
Changes in datalad/distribution/get.py
+12
Loading file...
Changes in datalad/support/github_.py
-2
+2
Loading file...
Files Coverage
datalad 0.01% 89.66%
Project Totals (274 files) 89.66%
Loading