1
# emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*-
2
# vi: set ft=python sts=4 ts=4 sw=4 et:
3
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
4
#
5
#   See COPYING file distributed along with the NiBabel package for the
6
#   copyright and license terms.
7
#
8
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
9 33
""" Array proxy base class
10

11
The proxy API is - at minimum:
12

13
* The object has a read-only attribute ``shape``
14
* read only ``is_proxy`` attribute / property set to True
15
* the object returns the data array from ``np.asarray(prox)``
16
* returns array slice from ``prox[<slice_spec>]`` where ``<slice_spec>`` is any
17
  ndarray slice specification that does not use numpy 'advanced indexing'.
18
* modifying no object outside ``obj`` will affect the result of
19
  ``np.asarray(obj)``.  Specifically:
20

21
  * Changes in position (``obj.tell()``) of passed file-like objects will
22
    not affect the output of from ``np.asarray(proxy)``.
23
  * if you pass a header into the __init__, then modifying the original
24
    header will not affect the result of the array return.
25

26
See :mod:`nibabel.tests.test_proxy_api` for proxy API conformance checks.
27
"""
28 33
from contextlib import contextmanager
29 33
from threading import RLock
30

31 33
import numpy as np
32

33 33
from .deprecated import deprecate_with_version
34 33
from .volumeutils import array_from_file, apply_read_scaling
35 33
from .fileslice import fileslice, canonical_slicers
36 33
from . import openers
37

38

39
"""This flag controls whether a new file handle is created every time an image
40
is accessed through an ``ArrayProxy``, or a single file handle is created and
41
used for the lifetime of the ``ArrayProxy``. It should be set to one of
42
``True`` or ``False``.
43

44
Management of file handles will be performed either by ``ArrayProxy`` objects,
45
or by the ``indexed_gzip`` package if it is used.
46

47
If this flag is set to ``True``, a single file handle is created and used. If
48
``False``, a new file handle is created every time the image is accessed.
49

50
If this is set to any other value, attempts to create an ``ArrayProxy`` without
51
specifying the ``keep_file_open`` flag will result in a ``ValueError`` being
52
raised.
53

54
.. warning:: Setting this flag to a value of ``'auto'`` became deprecated
55
             behaviour in version 2.4.1. Support for ``'auto'`` was removed
56
             in version 3.0.0.
57
"""
58 33
KEEP_FILE_OPEN_DEFAULT = False
59

60

61 33
class ArrayProxy(object):
62
    """ Class to act as proxy for the array that can be read from a file
63

64
    The array proxy allows us to freeze the passed fileobj and header such that
65
    it returns the expected data array.
66

67
    This implementation assumes a contiguous array in the file object, with one
68
    of the numpy dtypes, starting at a given file position ``offset`` with
69
    single ``slope`` and ``intercept`` scaling to produce output values.
70

71
    The class ``__init__`` requires a spec which defines how the data will be
72
    read and rescaled. The spec may be a tuple of length 2 - 5, containing the
73
    shape, storage dtype, offset, slope and intercept, or a ``header`` object
74
    with methods:
75

76
    * get_data_shape
77
    * get_data_dtype
78
    * get_data_offset
79
    * get_slope_inter
80

81
    A header should also have a 'copy' method.  This requirement will go away
82
    when the deprecated 'header' property goes away.
83

84
    This implementation allows us to deal with Analyze and its variants,
85
    including Nifti1, and with the MGH format.
86

87
    Other image types might need more specific classes to implement the API.
88
    See :mod:`nibabel.minc1`, :mod:`nibabel.ecat` and :mod:`nibabel.parrec` for
89
    examples.
90
    """
91
    # Assume Fortran array memory layout
92 33
    order = 'F'
93 33
    _header = None
94

95 33
    def __init__(self, file_like, spec, *, mmap=True, keep_file_open=None):
96
        """Initialize array proxy instance
97

98
        .. deprecated:: 2.4.1
99
            ``keep_file_open='auto'`` is redundant with `False` and has
100
            been deprecated. It raises an error as of nibabel 3.0.
101

102
        Parameters
103
        ----------
104
        file_like : object
105
            File-like object or filename. If file-like object, should implement
106
            at least ``read`` and ``seek``.
107
        spec : object or tuple
108
            Tuple must have length 2-5, with the following values:
109

110
            #. shape: tuple - tuple of ints describing shape of data;
111
            #. storage_dtype: dtype specifier - dtype of array inside proxied
112
               file, or input to ``numpy.dtype`` to specify array dtype;
113
            #. offset: int - offset, in bytes, of data array from start of file
114
               (default: 0);
115
            #. slope: float - scaling factor for resulting data (default: 1.0);
116
            #. inter: float - intercept for rescaled data (default: 0.0).
117

118
            OR
119

120
            Header object implementing ``get_data_shape``, ``get_data_dtype``,
121
            ``get_data_offset``, ``get_slope_inter``
122
        mmap : {True, False, 'c', 'r'}, optional, keyword only
123
            `mmap` controls the use of numpy memory mapping for reading data.
124
            If False, do not try numpy ``memmap`` for data array.  If one of
125
            {'c', 'r'}, try numpy memmap with ``mode=mmap``.  A `mmap` value of
126
            True gives the same behavior as ``mmap='c'``.  If `file_like`
127
            cannot be memory-mapped, ignore `mmap` value and read array from
128
            file.
129
        keep_file_open : { None, True, False }, optional, keyword only
130
            `keep_file_open` controls whether a new file handle is created
131
            every time the image is accessed, or a single file handle is
132
            created and used for the lifetime of this ``ArrayProxy``. If
133
            ``True``, a single file handle is created and used. If ``False``,
134
            a new file handle is created every time the image is accessed.
135
            If ``file_like`` is an open file handle, this setting has no
136
            effect. The default value (``None``) will result in the value of
137
            ``KEEP_FILE_OPEN_DEFAULT`` being used.
138
        """
139 33
        if mmap not in (True, False, 'c', 'r'):
140 33
            raise ValueError("mmap should be one of {True, False, 'c', 'r'}")
141 33
        self.file_like = file_like
142 33
        if hasattr(spec, 'get_data_shape'):
143 33
            slope, inter = spec.get_slope_inter()
144 33
            par = (spec.get_data_shape(),
145
                   spec.get_data_dtype(),
146
                   spec.get_data_offset(),
147
                   1. if slope is None else slope,
148
                   0. if inter is None else inter)
149
            # Reference to original header; we will remove this soon
150 33
            self._header = spec.copy()
151 33
        elif 2 <= len(spec) <= 5:
152 33
            optional = (0, 1., 0.)
153 33
            par = spec + optional[len(spec) - 2:]
154
        else:
155 33
            raise TypeError('spec must be tuple of length 2-5 or header object')
156

157
        # Copies of values needed to read array
158 33
        self._shape, self._dtype, self._offset, self._slope, self._inter = par
159
        # Permit any specifier that can be interpreted as a numpy dtype
160 33
        self._dtype = np.dtype(self._dtype)
161 33
        self._mmap = mmap
162
        # Flags to keep track of whether a single ImageOpener is created, and
163
        # whether a single underlying file handle is created.
164 33
        self._keep_file_open, self._persist_opener = \
165
            self._should_keep_file_open(file_like, keep_file_open)
166 33
        self._lock = RLock()
167

168 33
    def __del__(self):
169
        """If this ``ArrayProxy`` was created with ``keep_file_open=True``,
170
        the open file object is closed if necessary.
171
        """
172 33
        if hasattr(self, '_opener') and not self._opener.closed:
173 33
            self._opener.close_if_mine()
174 33
            self._opener = None
175

176 33
    def __getstate__(self):
177
        """Returns the state of this ``ArrayProxy`` during pickling. """
178 33
        state = self.__dict__.copy()
179 33
        state.pop('_lock', None)
180 33
        return state
181

182 33
    def __setstate__(self, state):
183
        """Sets the state of this ``ArrayProxy`` during unpickling. """
184 33
        self.__dict__.update(state)
185 33
        self._lock = RLock()
186

187 33
    def _should_keep_file_open(self, file_like, keep_file_open):
188
        """Called by ``__init__``.
189

190
        This method determines how to manage ``ImageOpener`` instances,
191
        and the underlying file handles - the behaviour depends on:
192

193
         - whether ``file_like`` is an an open file handle, or a path to a
194
           ``'.gz'`` file, or a path to a non-gzip file.
195
         - whether ``indexed_gzip`` is present (see
196
           :attr:`.openers.HAVE_INDEXED_GZIP`).
197

198
        An ``ArrayProxy`` object uses two internal flags to manage
199
        ``ImageOpener`` instances and underlying file handles.
200

201
          - The ``_persist_opener`` flag controls whether a single
202
            ``ImageOpener`` should be created and used for the lifetime of
203
            this ``ArrayProxy``, or whether separate ``ImageOpener`` instances
204
            should be created on each file access.
205

206
          - The ``_keep_file_open`` flag controls qwhether the underlying file
207
            handle should be kept open for the lifetime of this
208
            ``ArrayProxy``, or whether the file handle should be (re-)opened
209
            and closed on each file access.
210

211
        The internal ``_keep_file_open`` flag is only relevant if
212
        ``file_like`` is a ``'.gz'`` file, and the ``indexed_gzip`` library is
213
        present.
214

215
        This method returns the values to be used for the internal
216
        ``_persist_opener`` and ``_keep_file_open`` flags; these values are
217
        derived according to the following rules:
218

219
        1. If ``file_like`` is a file(-like) object, both flags are set to
220
        ``False``.
221

222
        2. If ``keep_file_open`` (as passed to :meth:``__init__``) is
223
           ``True``, both internal flags are set to ``True``.
224

225
        3. If ``keep_file_open`` is ``False``, but ``file_like`` is not a path
226
           to a ``.gz`` file or ``indexed_gzip`` is not present, both flags
227
           are set to ``False``.
228

229
        4. If ``keep_file_open`` is ``False``, ``file_like`` is a path to a
230
           ``.gz`` file, and ``indexed_gzip`` is present, ``_persist_opener``
231
           is set to ``True``, and ``_keep_file_open`` is set to ``False``.
232
           In this case, file handle management is delegated to the
233
           ``indexed_gzip`` library.
234

235
        .. deprecated:: 2.4.1
236
            ``keep_file_open='auto'`` is redundant with `False` and has
237
            been deprecated. It raises an error as of nibabel 3.0.
238

239
        Parameters
240
        ----------
241

242
        file_like : object
243
            File-like object or filename, as passed to ``__init__``.
244
        keep_file_open : { True, False }
245
            Flag as passed to ``__init__``.
246

247
        Returns
248
        -------
249

250
        A tuple containing:
251
          - ``keep_file_open`` flag to control persistence of file handles
252
          - ``persist_opener`` flag to control persistence of ``ImageOpener``
253
            objects.
254
        """
255 33
        if keep_file_open is None:
256 33
            keep_file_open = KEEP_FILE_OPEN_DEFAULT
257 33
            if keep_file_open not in (True, False):
258 33
                raise ValueError("nibabel.arrayproxy.KEEP_FILE_OPEN_DEFAULT "
259
                                 f"must be boolean. Found: {keep_file_open}")
260 33
        elif keep_file_open not in (True, False):
261 33
            raise ValueError('keep_file_open must be one of {None, True, False}')
262

263
        # file_like is a handle - keep_file_open is irrelevant
264 33
        if hasattr(file_like, 'read') and hasattr(file_like, 'seek'):
265 33
            return False, False
266
        # if the file is a gzip file, and we have_indexed_gzip,
267 33
        have_igzip = openers.HAVE_INDEXED_GZIP and file_like.endswith('.gz')
268

269 33
        persist_opener = keep_file_open or have_igzip
270 33
        return keep_file_open, persist_opener
271

272 33
    @property
273 33
    @deprecate_with_version('ArrayProxy.header deprecated', '2.2', '3.0')
274 5
    def header(self):
275 0
        return self._header
276

277 33
    @property
278 5
    def shape(self):
279 33
        return self._shape
280

281 33
    @property
282 5
    def ndim(self):
283 33
        return len(self.shape)
284

285 33
    @property
286 5
    def dtype(self):
287 33
        return self._dtype
288

289 33
    @property
290 5
    def offset(self):
291 33
        return self._offset
292

293 33
    @property
294 5
    def slope(self):
295 33
        return self._slope
296

297 33
    @property
298 5
    def inter(self):
299 33
        return self._inter
300

301 33
    @property
302 5
    def is_proxy(self):
303 33
        return True
304

305 33
    @contextmanager
306 5
    def _get_fileobj(self):
307
        """Create and return a new ``ImageOpener``, or return an existing one.
308

309
        The specific behaviour depends on the value of the ``keep_file_open``
310
        flag that was passed to ``__init__``.
311

312
        Yields
313
        ------
314
        ImageOpener
315
            A newly created ``ImageOpener`` instance, or an existing one,
316
            which provides access to the file.
317
        """
318 33
        if self._persist_opener:
319 33
            if not hasattr(self, '_opener'):
320 33
                self._opener = openers.ImageOpener(
321
                    self.file_like, keep_open=self._keep_file_open)
322 33
            yield self._opener
323
        else:
324 33
            with openers.ImageOpener(
325
                    self.file_like, keep_open=False) as opener:
326 33
                yield opener
327

328 33
    def _get_unscaled(self, slicer):
329 33
        if canonical_slicers(slicer, self._shape, False) == \
330
                canonical_slicers((), self._shape, False):
331 33
            with self._get_fileobj() as fileobj, self._lock:
332 33
                return array_from_file(self._shape,
333
                                       self._dtype,
334
                                       fileobj,
335
                                       offset=self._offset,
336
                                       order=self.order,
337
                                       mmap=self._mmap)
338 33
        with self._get_fileobj() as fileobj:
339 33
            return fileslice(fileobj,
340
                             slicer,
341
                             self._shape,
342
                             self._dtype,
343
                             self._offset,
344
                             order=self.order,
345
                             lock=self._lock)
346

347 33
    def _get_scaled(self, dtype, slicer):
348
        # Ensure scale factors have dtypes
349 33
        scl_slope = np.asanyarray(self._slope)
350 33
        scl_inter = np.asanyarray(self._inter)
351 33
        use_dtype = scl_slope.dtype if dtype is None else dtype
352

353 33
        if np.can_cast(scl_slope, use_dtype):
354 33
            scl_slope = scl_slope.astype(use_dtype)
355 33
        if np.can_cast(scl_inter, use_dtype):
356 33
            scl_inter = scl_inter.astype(use_dtype)
357
        # Read array and upcast as necessary for big slopes, intercepts
358 33
        scaled = apply_read_scaling(self._get_unscaled(slicer=slicer), scl_slope, scl_inter)
359 33
        if dtype is not None:
360 33
            scaled = scaled.astype(np.promote_types(scaled.dtype, dtype), copy=False)
361 33
        return scaled
362

363 33
    def get_unscaled(self):
364
        """ Read data from file
365

366
        This is an optional part of the proxy API
367
        """
368 33
        return self._get_unscaled(slicer=())
369

370 33
    def __array__(self, dtype=None):
371
        """ Read data from file and apply scaling, casting to ``dtype``
372

373
        If ``dtype`` is unspecified, the dtype of the returned array is the
374
        narrowest dtype that can represent the data without overflow.
375
        Generally, it is the wider of the dtypes of the slopes or intercepts.
376

377
        The types of the scale factors will generally be determined by the
378
        parameter size in the image header, and so should be consistent for a
379
        given image format, but may vary across formats.
380

381
        Parameters
382
        ----------
383
        dtype : numpy dtype specifier, optional
384
            A numpy dtype specifier specifying the type of the returned array.
385

386
        Returns
387
        -------
388
        array
389
            Scaled image data with type `dtype`.
390
        """
391 33
        arr = self._get_scaled(dtype=dtype, slicer=())
392 33
        if dtype is not None:
393 33
            arr = arr.astype(dtype, copy=False)
394 33
        return arr
395

396 33
    def __getitem__(self, slicer):
397 33
        return self._get_scaled(dtype=None, slicer=slicer)
398

399 33
    def reshape(self, shape):
400
        """ Return an ArrayProxy with a new shape, without modifying data """
401 33
        size = np.prod(self._shape)
402

403
        # Calculate new shape if not fully specified
404 33
        from operator import mul
405 33
        from functools import reduce
406 33
        n_unknowns = len([e for e in shape if e == -1])
407 33
        if n_unknowns > 1:
408 33
            raise ValueError("can only specify one unknown dimension")
409 33
        elif n_unknowns == 1:
410 33
            known_size = reduce(mul, shape, -1)
411 33
            unknown_size = size // known_size
412 33
            shape = tuple(unknown_size if e == -1 else e for e in shape)
413

414 33
        if np.prod(shape) != size:
415 33
            raise ValueError(f"cannot reshape array of size {size:d} into shape {shape!s}")
416 33
        return self.__class__(file_like=self.file_like,
417
                              spec=(shape, self._dtype, self._offset,
418
                                    self._slope, self._inter),
419
                              mmap=self._mmap)
420

421

422 33
def is_proxy(obj):
423
    """ Return True if `obj` is an array proxy
424
    """
425 33
    try:
426 33
        return obj.is_proxy
427 33
    except AttributeError:
428 33
        return False
429

430

431 33
def reshape_dataobj(obj, shape):
432
    """ Use `obj` reshape method if possible, else numpy reshape function
433
    """
434 33
    return (obj.reshape(shape) if hasattr(obj, 'reshape')
435
            else np.reshape(obj, shape))

Read our documentation on viewing source code .

Loading