1
/*
2
 * This file contains low-level loops for data type transfers.
3
 * In particular the function PyArray_GetDTypeTransferFunction is
4
 * implemented here.
5
 *
6
 * Copyright (c) 2010 by Mark Wiebe (mwwiebe@gmail.com)
7
 * The University of British Columbia
8
 *
9
 * See LICENSE.txt for the license.
10

11
 */
12

13
#define PY_SSIZE_T_CLEAN
14
#include "Python.h"
15
#include "structmember.h"
16

17
#define NPY_NO_DEPRECATED_API NPY_API_VERSION
18
#define _MULTIARRAYMODULE
19
#include <numpy/arrayobject.h>
20

21
#include "npy_pycompat.h"
22

23
#include "convert_datatype.h"
24
#include "ctors.h"
25
#include "_datetime.h"
26
#include "datetime_strings.h"
27
#include "descriptor.h"
28
#include "array_assign.h"
29

30
#include "shape.h"
31
#include "lowlevel_strided_loops.h"
32
#include "alloc.h"
33

34
#define NPY_LOWLEVEL_BUFFER_BLOCKSIZE  128
35

36
/********** PRINTF DEBUG TRACING **************/
37
#define NPY_DT_DBG_TRACING 0
38
/* Tracing incref/decref can be very noisy */
39
#define NPY_DT_REF_DBG_TRACING 0
40

41
#if NPY_DT_REF_DBG_TRACING
42
#define NPY_DT_DBG_REFTRACE(msg, ref) \
43
    printf("%-12s %20p %s%d%s\n", msg, ref, \
44
                        ref ? "(refcnt " : "", \
45
                        ref ? (int)ref->ob_refcnt : 0, \
46
                        ref ? ((ref->ob_refcnt <= 0) ? \
47
                                        ") <- BIG PROBLEM!!!!" : ")") : ""); \
48
    fflush(stdout);
49
#else
50
#define NPY_DT_DBG_REFTRACE(msg, ref)
51
#endif
52
/**********************************************/
53

54
#if NPY_DT_DBG_TRACING
55
/*
56
 * Thin wrapper around print that ignores exceptions
57
 */
58
static void
59
_safe_print(PyObject *obj)
60
{
61
    if (PyObject_Print(obj, stdout, 0) < 0) {
62
        PyErr_Clear();
63
        printf("<error during print>");
64
    }
65
}
66
#endif
67

68
/*
69
 * Returns a transfer function which DECREFs any references in src_type.
70
 *
71
 * Returns NPY_SUCCEED or NPY_FAIL.
72
 */
73
static int
74
get_decsrcref_transfer_function(int aligned,
75
                            npy_intp src_stride,
76
                            PyArray_Descr *src_dtype,
77
                            PyArray_StridedUnaryOp **out_stransfer,
78
                            NpyAuxData **out_transferdata,
79
                            int *out_needs_api);
80

81
/*
82
 * Returns a transfer function which zeros out the dest values.
83
 *
84
 * Returns NPY_SUCCEED or NPY_FAIL.
85
 */
86
static int
87
get_setdstzero_transfer_function(int aligned,
88
                            npy_intp dst_stride,
89
                            PyArray_Descr *dst_dtype,
90
                            PyArray_StridedUnaryOp **out_stransfer,
91
                            NpyAuxData **out_transferdata,
92
                            int *out_needs_api);
93

94
/*
95
 * Returns a transfer function which sets a boolean type to ones.
96
 *
97
 * Returns NPY_SUCCEED or NPY_FAIL.
98
 */
99
NPY_NO_EXPORT int
100
get_bool_setdstone_transfer_function(npy_intp dst_stride,
101
                            PyArray_StridedUnaryOp **out_stransfer,
102
                            NpyAuxData **out_transferdata,
103
                            int *NPY_UNUSED(out_needs_api));
104

105
/*************************** COPY REFERENCES *******************************/
106

107
/* Moves references from src to dst */
108
static int
109 1
_strided_to_strided_move_references(char *dst, npy_intp dst_stride,
110
                        char *src, npy_intp src_stride,
111
                        npy_intp N, npy_intp src_itemsize,
112
                        NpyAuxData *data)
113
{
114 1
    PyObject *src_ref = NULL, *dst_ref = NULL;
115 1
    while (N > 0) {
116 1
        memcpy(&src_ref, src, sizeof(src_ref));
117 1
        memcpy(&dst_ref, dst, sizeof(dst_ref));
118

119
        /* Release the reference in dst */
120
        NPY_DT_DBG_REFTRACE("dec dst ref", dst_ref);
121 1
        Py_XDECREF(dst_ref);
122
        /* Move the reference */
123
        NPY_DT_DBG_REFTRACE("move src ref", src_ref);
124 1
        memcpy(dst, &src_ref, sizeof(src_ref));
125
        /* Set the source reference to NULL */
126 1
        src_ref = NULL;
127 1
        memcpy(src, &src_ref, sizeof(src_ref));
128

129 1
        src += src_stride;
130 1
        dst += dst_stride;
131 1
        --N;
132
    }
133 1
    return 0;
134
}
135

136
/* Copies references from src to dst */
137
static int
138 1
_strided_to_strided_copy_references(char *dst, npy_intp dst_stride,
139
                        char *src, npy_intp src_stride,
140
                        npy_intp N, npy_intp src_itemsize,
141
                        NpyAuxData *data)
142
{
143 1
    PyObject *src_ref = NULL, *dst_ref = NULL;
144 1
    while (N > 0) {
145 1
        memcpy(&src_ref, src, sizeof(src_ref));
146 1
        memcpy(&dst_ref, dst, sizeof(dst_ref));
147

148
        /* Copy the reference */
149
        NPY_DT_DBG_REFTRACE("copy src ref", src_ref);
150 1
        memcpy(dst, &src_ref, sizeof(src_ref));
151
        /* Claim the reference */
152 1
        Py_XINCREF(src_ref);
153
        /* Release the reference in dst */
154
        NPY_DT_DBG_REFTRACE("dec dst ref", dst_ref);
155 1
        Py_XDECREF(dst_ref);
156

157 1
        src += src_stride;
158 1
        dst += dst_stride;
159 1
        --N;
160
    }
161 1
    return 0;
162
}
163

164

165
/************************** ZERO-PADDED COPY ******************************/
166

167
/* Does a zero-padded copy */
168
typedef struct {
169
    NpyAuxData base;
170
    npy_intp dst_itemsize;
171
} _strided_zero_pad_data;
172

173
/* zero-padded data copy function */
174 0
static NpyAuxData *_strided_zero_pad_data_clone(NpyAuxData *data)
175
{
176 0
    _strided_zero_pad_data *newdata =
177
            (_strided_zero_pad_data *)PyArray_malloc(
178
                                    sizeof(_strided_zero_pad_data));
179 0
    if (newdata == NULL) {
180
        return NULL;
181
    }
182

183 0
    memcpy(newdata, data, sizeof(_strided_zero_pad_data));
184

185 0
    return (NpyAuxData *)newdata;
186
}
187

188
/*
189
 * Does a strided to strided zero-padded copy for the case where
190
 * dst_itemsize > src_itemsize
191
 */
192
static int
193 1
_strided_to_strided_zero_pad_copy(char *dst, npy_intp dst_stride,
194
                        char *src, npy_intp src_stride,
195
                        npy_intp N, npy_intp src_itemsize,
196
                        NpyAuxData *data)
197
{
198 1
    _strided_zero_pad_data *d = (_strided_zero_pad_data *)data;
199 1
    npy_intp dst_itemsize = d->dst_itemsize;
200 1
    npy_intp zero_size = dst_itemsize-src_itemsize;
201

202 1
    while (N > 0) {
203 1
        memcpy(dst, src, src_itemsize);
204 1
        memset(dst + src_itemsize, 0, zero_size);
205 1
        src += src_stride;
206 1
        dst += dst_stride;
207 1
        --N;
208
    }
209 1
    return 0;
210
}
211

212
/*
213
 * Does a strided to strided zero-padded copy for the case where
214
 * dst_itemsize < src_itemsize
215
 */
216
static int
217 1
_strided_to_strided_truncate_copy(char *dst, npy_intp dst_stride,
218
                        char *src, npy_intp src_stride,
219
                        npy_intp N, npy_intp src_itemsize,
220
                        NpyAuxData *data)
221
{
222 1
    _strided_zero_pad_data *d = (_strided_zero_pad_data *)data;
223 1
    npy_intp dst_itemsize = d->dst_itemsize;
224

225 1
    while (N > 0) {
226 1
        memcpy(dst, src, dst_itemsize);
227 1
        src += src_stride;
228 1
        dst += dst_stride;
229 1
        --N;
230
    }
231 1
    return 0;
232
}
233

234
/*
235
 * Does a strided to strided zero-padded or truncated copy for the case where
236
 * unicode swapping is needed.
237
 */
238
static int
239 1
_strided_to_strided_unicode_copyswap(char *dst, npy_intp dst_stride,
240
                        char *src, npy_intp src_stride,
241
                        npy_intp N, npy_intp src_itemsize,
242
                        NpyAuxData *data)
243
{
244 1
    _strided_zero_pad_data *d = (_strided_zero_pad_data *)data;
245 1
    npy_intp dst_itemsize = d->dst_itemsize;
246 1
    npy_intp zero_size = dst_itemsize - src_itemsize;
247 1
    npy_intp copy_size = zero_size > 0 ? src_itemsize : dst_itemsize;
248
    char *_dst;
249 1
    npy_intp characters = dst_itemsize / 4;
250
    int i;
251

252 1
    while (N > 0) {
253 1
        memcpy(dst, src, copy_size);
254 1
        if (zero_size > 0) {
255 1
            memset(dst + src_itemsize, 0, zero_size);
256
        }
257
        _dst = dst;
258 1
        for (i=0; i < characters; i++) {
259 1
            npy_bswap4_unaligned(_dst);
260 1
            _dst += 4;
261
        }
262 1
        src += src_stride;
263 1
        dst += dst_stride;
264 1
        --N;
265
    }
266 1
    return 0;
267
}
268

269

270
NPY_NO_EXPORT int
271 1
PyArray_GetStridedZeroPadCopyFn(int aligned, int unicode_swap,
272
                            npy_intp src_stride, npy_intp dst_stride,
273
                            npy_intp src_itemsize, npy_intp dst_itemsize,
274
                            PyArray_StridedUnaryOp **out_stransfer,
275
                            NpyAuxData **out_transferdata)
276
{
277 1
    if ((src_itemsize == dst_itemsize) && !unicode_swap) {
278 0
        *out_stransfer = PyArray_GetStridedCopyFn(aligned, src_stride,
279
                                dst_stride, src_itemsize);
280 0
        *out_transferdata = NULL;
281 0
        return (*out_stransfer == NULL) ? NPY_FAIL : NPY_SUCCEED;
282
    }
283
    else {
284 1
        _strided_zero_pad_data *d = PyArray_malloc(
285
                                        sizeof(_strided_zero_pad_data));
286 1
        if (d == NULL) {
287 0
            PyErr_NoMemory();
288 0
            return NPY_FAIL;
289
        }
290 1
        d->dst_itemsize = dst_itemsize;
291 1
        d->base.free = (NpyAuxData_FreeFunc *)&PyArray_free;
292 1
        d->base.clone = &_strided_zero_pad_data_clone;
293

294 1
        if (unicode_swap) {
295 1
            *out_stransfer = &_strided_to_strided_unicode_copyswap;
296
        }
297 1
        else if (src_itemsize < dst_itemsize) {
298 1
            *out_stransfer = &_strided_to_strided_zero_pad_copy;
299
        }
300
        else {
301 1
            *out_stransfer = &_strided_to_strided_truncate_copy;
302
        }
303

304 1
        *out_transferdata = (NpyAuxData *)d;
305 1
        return NPY_SUCCEED;
306
    }
307
}
308

309
/***************** WRAP ALIGNED CONTIGUOUS TRANSFER FUNCTION **************/
310

311
/* Wraps a transfer function + data in alignment code */
312
typedef struct {
313
    NpyAuxData base;
314
    PyArray_StridedUnaryOp *wrapped,
315
                *tobuffer, *frombuffer;
316
    NpyAuxData *wrappeddata, *todata, *fromdata;
317
    npy_intp src_itemsize, dst_itemsize;
318
    char *bufferin, *bufferout;
319
    npy_bool init_dest, out_needs_api;
320
} _align_wrap_data;
321

322
/* transfer data free function */
323 1
static void _align_wrap_data_free(NpyAuxData *data)
324
{
325 1
    _align_wrap_data *d = (_align_wrap_data *)data;
326 1
    NPY_AUXDATA_FREE(d->wrappeddata);
327 1
    NPY_AUXDATA_FREE(d->todata);
328 1
    NPY_AUXDATA_FREE(d->fromdata);
329 1
    PyArray_free(data);
330
}
331

332
/* transfer data copy function */
333 1
static NpyAuxData *_align_wrap_data_clone(NpyAuxData *data)
334
{
335 1
    _align_wrap_data *d = (_align_wrap_data *)data;
336
    _align_wrap_data *newdata;
337
    npy_intp basedatasize, datasize;
338

339
    /* Round up the structure size to 16-byte boundary */
340 1
    basedatasize = (sizeof(_align_wrap_data)+15)&(-0x10);
341
    /* Add space for two low level buffers */
342 1
    datasize = basedatasize +
343 1
                NPY_LOWLEVEL_BUFFER_BLOCKSIZE*d->src_itemsize +
344 1
                NPY_LOWLEVEL_BUFFER_BLOCKSIZE*d->dst_itemsize;
345

346
    /* Allocate the data, and populate it */
347 1
    newdata = (_align_wrap_data *)PyArray_malloc(datasize);
348 1
    if (newdata == NULL) {
349
        return NULL;
350
    }
351 1
    memcpy(newdata, data, basedatasize);
352 1
    newdata->bufferin = (char *)newdata + basedatasize;
353 1
    newdata->bufferout = newdata->bufferin +
354 1
                NPY_LOWLEVEL_BUFFER_BLOCKSIZE*newdata->src_itemsize;
355 1
    if (newdata->wrappeddata != NULL) {
356 0
        newdata->wrappeddata = NPY_AUXDATA_CLONE(d->wrappeddata);
357 0
        if (newdata->wrappeddata == NULL) {
358 0
            PyArray_free(newdata);
359 0
            return NULL;
360
        }
361
    }
362 1
    if (newdata->todata != NULL) {
363 0
        newdata->todata = NPY_AUXDATA_CLONE(d->todata);
364 0
        if (newdata->todata == NULL) {
365 0
            NPY_AUXDATA_FREE(newdata->wrappeddata);
366 0
            PyArray_free(newdata);
367 0
            return NULL;
368
        }
369
    }
370 1
    if (newdata->fromdata != NULL) {
371 0
        newdata->fromdata = NPY_AUXDATA_CLONE(d->fromdata);
372 0
        if (newdata->fromdata == NULL) {
373 0
            NPY_AUXDATA_FREE(newdata->wrappeddata);
374 0
            NPY_AUXDATA_FREE(newdata->todata);
375 0
            PyArray_free(newdata);
376 0
            return NULL;
377
        }
378
    }
379

380 1
    newdata->init_dest = d->init_dest;
381 1
    newdata->out_needs_api = d->out_needs_api;
382

383 1
    return (NpyAuxData *)newdata;
384
}
385

386
static int
387 1
_strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride,
388
                        char *src, npy_intp src_stride,
389
                        npy_intp N, npy_intp src_itemsize,
390
                        NpyAuxData *data)
391
{
392 1
    _align_wrap_data *d = (_align_wrap_data *)data;
393 1
    PyArray_StridedUnaryOp *wrapped = d->wrapped,
394 1
            *tobuffer = d->tobuffer,
395 1
            *frombuffer = d->frombuffer;
396 1
    npy_intp inner_src_itemsize = d->src_itemsize,
397 1
             dst_itemsize = d->dst_itemsize;
398 1
    NpyAuxData *wrappeddata = d->wrappeddata,
399 1
            *todata = d->todata,
400 1
            *fromdata = d->fromdata;
401 1
    char *bufferin = d->bufferin, *bufferout = d->bufferout;
402 1
    npy_bool init_dest = d->init_dest;
403

404
    for(;;) {
405 1
        if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) {
406 1
            if (tobuffer(
407
                    bufferin, inner_src_itemsize, src, src_stride,
408
                    NPY_LOWLEVEL_BUFFER_BLOCKSIZE, src_itemsize, todata) < 0) {
409
                return -1;
410
            }
411 1
            if (init_dest) {
412 1
                memset(bufferout, 0,
413 1
                       dst_itemsize*NPY_LOWLEVEL_BUFFER_BLOCKSIZE);
414
            }
415 1
            if (wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize,
416
                        NPY_LOWLEVEL_BUFFER_BLOCKSIZE,
417
                        inner_src_itemsize, wrappeddata) < 0) {
418
                return -1;
419
            }
420 1
            if (frombuffer(dst, dst_stride, bufferout, dst_itemsize,
421
                           NPY_LOWLEVEL_BUFFER_BLOCKSIZE,
422
                           dst_itemsize, fromdata) < 0) {
423
                return -1;
424
            }
425 1
            N -= NPY_LOWLEVEL_BUFFER_BLOCKSIZE;
426 1
            src += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_stride;
427 1
            dst += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_stride;
428
        }
429
        else {
430 1
            if (tobuffer(bufferin, inner_src_itemsize, src, src_stride,
431
                         N, src_itemsize, todata) < 0) {
432
                return -1;
433
            }
434 1
            if (init_dest) {
435 1
                memset(bufferout, 0, dst_itemsize*N);
436
            }
437 1
            if (wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize,
438
                        N, inner_src_itemsize, wrappeddata) < 0) {
439
                return -1;
440
            }
441 1
            if (frombuffer(dst, dst_stride, bufferout, dst_itemsize,
442
                           N, dst_itemsize, fromdata) < 0) {
443
                return -1;
444
            }
445 1
            return 0;
446
        }
447
    }
448
}
449

450
/*
451
 * Wraps an aligned contig to contig transfer function between either
452
 * copies or byte swaps to temporary buffers.
453
 *
454
 * src_itemsize/dst_itemsize - The sizes of the src and dst datatypes.
455
 * tobuffer - copy/swap function from src to an aligned contiguous buffer.
456
 * todata - data for tobuffer
457
 * frombuffer - copy/swap function from an aligned contiguous buffer to dst.
458
 * fromdata - data for frombuffer
459
 * wrapped - contig to contig transfer function being wrapped
460
 * wrappeddata - data for wrapped
461
 * init_dest - 1 means to memset the dest buffer to 0 before calling wrapped.
462
 * out_needs_api - if NPY_TRUE, check for (and break on) Python API errors.
463
 *
464
 * Returns NPY_SUCCEED or NPY_FAIL.
465
 */
466
NPY_NO_EXPORT int
467 1
wrap_aligned_contig_transfer_function(
468
            npy_intp src_itemsize, npy_intp dst_itemsize,
469
            PyArray_StridedUnaryOp *tobuffer, NpyAuxData *todata,
470
            PyArray_StridedUnaryOp *frombuffer, NpyAuxData *fromdata,
471
            PyArray_StridedUnaryOp *wrapped, NpyAuxData *wrappeddata,
472
            int init_dest,
473
            int out_needs_api,
474
            PyArray_StridedUnaryOp **out_stransfer,
475
            NpyAuxData **out_transferdata)
476
{
477
    _align_wrap_data *data;
478
    npy_intp basedatasize, datasize;
479

480
    /* Round up the structure size to 16-byte boundary */
481 1
    basedatasize = (sizeof(_align_wrap_data)+15)&(-0x10);
482
    /* Add space for two low level buffers */
483 1
    datasize = basedatasize +
484 1
                NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_itemsize +
485
                NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_itemsize;
486

487
    /* Allocate the data, and populate it */
488 1
    data = (_align_wrap_data *)PyArray_malloc(datasize);
489 1
    if (data == NULL) {
490 0
        PyErr_NoMemory();
491 0
        return NPY_FAIL;
492
    }
493 1
    data->base.free = &_align_wrap_data_free;
494 1
    data->base.clone = &_align_wrap_data_clone;
495 1
    data->tobuffer = tobuffer;
496 1
    data->todata = todata;
497 1
    data->frombuffer = frombuffer;
498 1
    data->fromdata = fromdata;
499 1
    data->wrapped = wrapped;
500 1
    data->wrappeddata = wrappeddata;
501 1
    data->src_itemsize = src_itemsize;
502 1
    data->dst_itemsize = dst_itemsize;
503 1
    data->bufferin = (char *)data + basedatasize;
504 1
    data->bufferout = data->bufferin +
505 1
                NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_itemsize;
506 1
    data->init_dest = (npy_bool) init_dest;
507 1
    data->out_needs_api = (npy_bool) out_needs_api;
508

509
    /* Set the function and data */
510 1
    *out_stransfer = &_strided_to_strided_contig_align_wrap;
511 1
    *out_transferdata = (NpyAuxData *)data;
512

513 1
    return NPY_SUCCEED;
514
}
515

516
/*************************** WRAP DTYPE COPY/SWAP *************************/
517
/* Wraps the dtype copy swap function */
518
typedef struct {
519
    NpyAuxData base;
520
    PyArray_CopySwapNFunc *copyswapn;
521
    int swap;
522
    PyArrayObject *arr;
523
} _wrap_copy_swap_data;
524

525
/* wrap copy swap data free function */
526 1
static void _wrap_copy_swap_data_free(NpyAuxData *data)
527
{
528 1
    _wrap_copy_swap_data *d = (_wrap_copy_swap_data *)data;
529 1
    Py_DECREF(d->arr);
530 1
    PyArray_free(data);
531
}
532

533
/* wrap copy swap data copy function */
534 0
static NpyAuxData *_wrap_copy_swap_data_clone(NpyAuxData *data)
535
{
536 0
    _wrap_copy_swap_data *newdata =
537
        (_wrap_copy_swap_data *)PyArray_malloc(sizeof(_wrap_copy_swap_data));
538 0
    if (newdata == NULL) {
539
        return NULL;
540
    }
541

542 0
    memcpy(newdata, data, sizeof(_wrap_copy_swap_data));
543 0
    Py_INCREF(newdata->arr);
544

545 0
    return (NpyAuxData *)newdata;
546
}
547

548
static int
549 1
_strided_to_strided_wrap_copy_swap(char *dst, npy_intp dst_stride,
550
                        char *src, npy_intp src_stride,
551
                        npy_intp N, npy_intp NPY_UNUSED(src_itemsize),
552
                        NpyAuxData *data)
553
{
554 1
    _wrap_copy_swap_data *d = (_wrap_copy_swap_data *)data;
555

556
    /* We assume that d->copyswapn should not be able to error. */
557 1
    d->copyswapn(dst, dst_stride, src, src_stride, N, d->swap, d->arr);
558 1
    return 0;
559
}
560

561
/* This only gets used for custom data types and for Unicode when swapping */
562
static int
563 1
wrap_copy_swap_function(int aligned,
564
                npy_intp src_stride, npy_intp dst_stride,
565
                PyArray_Descr *dtype,
566
                int should_swap,
567
                PyArray_StridedUnaryOp **out_stransfer,
568
                NpyAuxData **out_transferdata)
569
{
570
    _wrap_copy_swap_data *data;
571 1
    npy_intp shape = 1;
572

573
    /* Allocate the data for the copy swap */
574 1
    data = (_wrap_copy_swap_data *)PyArray_malloc(sizeof(_wrap_copy_swap_data));
575 1
    if (data == NULL) {
576 0
        PyErr_NoMemory();
577 0
        *out_stransfer = NULL;
578 0
        *out_transferdata = NULL;
579
        return NPY_FAIL;
580
    }
581

582 1
    data->base.free = &_wrap_copy_swap_data_free;
583 1
    data->base.clone = &_wrap_copy_swap_data_clone;
584 1
    data->copyswapn = dtype->f->copyswapn;
585 1
    data->swap = should_swap;
586

587
    /*
588
     * TODO: This is a hack so the copyswap functions have an array.
589
     *       The copyswap functions shouldn't need that.
590
     */
591 1
    Py_INCREF(dtype);
592 1
    data->arr = (PyArrayObject *)PyArray_NewFromDescr_int(
593
            &PyArray_Type, dtype,
594
            1, &shape, NULL, NULL,
595
            0, NULL, NULL,
596
            0, 1);
597 1
    if (data->arr == NULL) {
598 0
        PyArray_free(data);
599
        return NPY_FAIL;
600
    }
601

602 1
    *out_stransfer = &_strided_to_strided_wrap_copy_swap;
603 1
    *out_transferdata = (NpyAuxData *)data;
604

605
    return NPY_SUCCEED;
606
}
607

608
/*************************** DTYPE CAST FUNCTIONS *************************/
609

610
/* Does a simple aligned cast */
611
typedef struct {
612
    NpyAuxData base;
613
    PyArray_VectorUnaryFunc *castfunc;
614
    PyArrayObject *aip, *aop;
615
    npy_bool needs_api;
616
} _strided_cast_data;
617

618
/* strided cast data free function */
619 1
static void _strided_cast_data_free(NpyAuxData *data)
620
{
621 1
    _strided_cast_data *d = (_strided_cast_data *)data;
622 1
    Py_DECREF(d->aip);
623 1
    Py_DECREF(d->aop);
624 1
    PyArray_free(data);
625
}
626

627
/* strided cast data copy function */
628 0
static NpyAuxData *_strided_cast_data_clone(NpyAuxData *data)
629
{
630 0
    _strided_cast_data *newdata =
631
            (_strided_cast_data *)PyArray_malloc(sizeof(_strided_cast_data));
632 0
    if (newdata == NULL) {
633
        return NULL;
634
    }
635

636 0
    memcpy(newdata, data, sizeof(_strided_cast_data));
637 0
    Py_INCREF(newdata->aip);
638 0
    Py_INCREF(newdata->aop);
639

640 0
    return (NpyAuxData *)newdata;
641
}
642

643
static int
644 1
_aligned_strided_to_strided_cast(char *dst, npy_intp dst_stride,
645
                        char *src, npy_intp src_stride,
646
                        npy_intp N, npy_intp src_itemsize,
647
                        NpyAuxData *data)
648
{
649 1
    _strided_cast_data *d = (_strided_cast_data *)data;
650 1
    PyArray_VectorUnaryFunc *castfunc = d->castfunc;
651 1
    PyArrayObject *aip = d->aip, *aop = d->aop;
652 1
    npy_bool needs_api = d->needs_api;
653

654 1
    while (N > 0) {
655 1
        castfunc(src, dst, 1, aip, aop);
656
        /*
657
         * Since error handling in ufuncs is not ideal (at the time of
658
         * writing this, an error could be in process before calling this
659
         * function. For most of NumPy history these checks were completely
660
         * missing, so this is hopefully OK for the time being (until ufuncs
661
         * are fixed).
662
         */
663 1
        if (needs_api && PyErr_Occurred()) {
664
            return -1;
665
        }
666 1
        dst += dst_stride;
667 1
        src += src_stride;
668 1
        --N;
669
    }
670
    return 0;
671
}
672

673
/* This one requires src be of type NPY_OBJECT */
674
static int
675 1
_aligned_strided_to_strided_cast_decref_src(char *dst, npy_intp dst_stride,
676
                        char *src, npy_intp src_stride,
677
                        npy_intp N, npy_intp src_itemsize,
678
                        NpyAuxData *data)
679
{
680 1
    _strided_cast_data *d = (_strided_cast_data *)data;
681 1
    PyArray_VectorUnaryFunc *castfunc = d->castfunc;
682 1
    PyArrayObject *aip = d->aip, *aop = d->aop;
683 1
    npy_bool needs_api = d->needs_api;
684
    PyObject *src_ref;
685

686 1
    while (N > 0) {
687 1
        castfunc(src, dst, 1, aip, aop);
688
        /*
689
         * See comment in `_aligned_strided_to_strided_cast`, an error could
690
         * in principle be set before `castfunc` is called.
691
         */
692 1
        if (needs_api && PyErr_Occurred()) {
693
            return -1;
694
        }
695
        /* After casting, decrement the source ref and set it to NULL */
696 1
        memcpy(&src_ref, src, sizeof(src_ref));
697 1
        Py_XDECREF(src_ref);
698 1
        memset(src, 0, sizeof(PyObject *));
699
        NPY_DT_DBG_REFTRACE("dec src ref (cast object -> not object)", src_ref);
700

701 1
        dst += dst_stride;
702 1
        src += src_stride;
703 1
        --N;
704
    }
705
    return 0;
706
}
707

708
static int
709 1
_aligned_contig_to_contig_cast(char *dst, npy_intp NPY_UNUSED(dst_stride),
710
                        char *src, npy_intp NPY_UNUSED(src_stride),
711
                        npy_intp N, npy_intp NPY_UNUSED(itemsize),
712
                        NpyAuxData *data)
713
{
714 1
    _strided_cast_data *d = (_strided_cast_data *)data;
715 1
    npy_bool needs_api = d->needs_api;
716

717 1
    d->castfunc(src, dst, N, d->aip, d->aop);
718
    /*
719
     * See comment in `_aligned_strided_to_strided_cast`, an error could
720
     * in principle be set before `castfunc` is called.
721
     */
722 1
    if (needs_api && PyErr_Occurred()) {
723
        return -1;
724
    }
725
    return 0;
726
}
727

728
static int
729 1
get_nbo_cast_numeric_transfer_function(int aligned,
730
                            npy_intp src_stride, npy_intp dst_stride,
731
                            int src_type_num, int dst_type_num,
732
                            PyArray_StridedUnaryOp **out_stransfer,
733
                            NpyAuxData **out_transferdata)
734
{
735
    /* Emit a warning if complex imaginary is being cast away */
736 1
    if (PyTypeNum_ISCOMPLEX(src_type_num) &&
737 1
                    !PyTypeNum_ISCOMPLEX(dst_type_num) &&
738
                    !PyTypeNum_ISBOOL(dst_type_num)) {
739
        static PyObject *cls = NULL;
740
        int ret;
741 1
        npy_cache_import("numpy.core", "ComplexWarning", &cls);
742 1
        if (cls == NULL) {
743
            return NPY_FAIL;
744
        }
745 1
        ret = PyErr_WarnEx(cls,
746
                "Casting complex values to real discards "
747
                "the imaginary part", 1);
748 1
        if (ret < 0) {
749
            return NPY_FAIL;
750
        }
751
    }
752

753 1
    *out_stransfer = PyArray_GetStridedNumericCastFn(aligned,
754
                                src_stride, dst_stride,
755
                                src_type_num, dst_type_num);
756 1
    *out_transferdata = NULL;
757 1
    if (*out_stransfer == NULL) {
758 0
        PyErr_SetString(PyExc_ValueError,
759
                "unexpected error in GetStridedNumericCastFn");
760 0
        return NPY_FAIL;
761
    }
762

763
    return NPY_SUCCEED;
764
}
765

766
/*
767
 * Does a datetime->datetime, timedelta->timedelta,
768
 * datetime->ascii, or ascii->datetime cast
769
 */
770
typedef struct {
771
    NpyAuxData base;
772
    /* The conversion fraction */
773
    npy_int64 num, denom;
774
    /* For the datetime -> string conversion, the dst string length */
775
    npy_intp src_itemsize, dst_itemsize;
776
    /*
777
     * A buffer of size 'src_itemsize + 1', for when the input
778
     * string is exactly of length src_itemsize with no NULL
779
     * terminator.
780
     */
781
    char *tmp_buffer;
782
    /*
783
     * The metadata for when dealing with Months or Years
784
     * which behave non-linearly with respect to the other
785
     * units.
786
     */
787
    PyArray_DatetimeMetaData src_meta, dst_meta;
788
} _strided_datetime_cast_data;
789

790
/* strided datetime cast data free function */
791 1
static void _strided_datetime_cast_data_free(NpyAuxData *data)
792
{
793 1
    _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data;
794 1
    PyArray_free(d->tmp_buffer);
795 1
    PyArray_free(data);
796
}
797

798
/* strided datetime cast data copy function */
799 0
static NpyAuxData *_strided_datetime_cast_data_clone(NpyAuxData *data)
800
{
801 0
    _strided_datetime_cast_data *newdata =
802
            (_strided_datetime_cast_data *)PyArray_malloc(
803
                                        sizeof(_strided_datetime_cast_data));
804 0
    if (newdata == NULL) {
805
        return NULL;
806
    }
807

808 0
    memcpy(newdata, data, sizeof(_strided_datetime_cast_data));
809 0
    if (newdata->tmp_buffer != NULL) {
810 0
        newdata->tmp_buffer = PyArray_malloc(newdata->src_itemsize + 1);
811 0
        if (newdata->tmp_buffer == NULL) {
812 0
            PyArray_free(newdata);
813 0
            return NULL;
814
        }
815
    }
816

817
    return (NpyAuxData *)newdata;
818
}
819

820
static int
821 1
_strided_to_strided_datetime_general_cast(char *dst, npy_intp dst_stride,
822
                        char *src, npy_intp src_stride,
823
                        npy_intp N, npy_intp src_itemsize,
824
                        NpyAuxData *data)
825
{
826 1
    _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data;
827
    npy_int64 dt;
828
    npy_datetimestruct dts;
829

830 1
    while (N > 0) {
831 1
        memcpy(&dt, src, sizeof(dt));
832

833 1
        if (convert_datetime_to_datetimestruct(&d->src_meta,
834
                                               dt, &dts) < 0) {
835
            return -1;
836
        }
837
        else {
838 1
            if (convert_datetimestruct_to_datetime(&d->dst_meta,
839
                                                   &dts, &dt) < 0) {
840
                return -1;
841
            }
842
        }
843

844 1
        memcpy(dst, &dt, sizeof(dt));
845

846 1
        dst += dst_stride;
847 1
        src += src_stride;
848 1
        --N;
849
    }
850
    return 0;
851
}
852

853
static int
854 1
_strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride,
855
                        char *src, npy_intp src_stride,
856
                        npy_intp N, npy_intp src_itemsize,
857
                        NpyAuxData *data)
858
{
859 1
    _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data;
860 1
    npy_int64 num = d->num, denom = d->denom;
861
    npy_int64 dt;
862

863 1
    while (N > 0) {
864 1
        memcpy(&dt, src, sizeof(dt));
865

866 1
        if (dt != NPY_DATETIME_NAT) {
867
            /* Apply the scaling */
868 0
            if (dt < 0) {
869 0
                dt = (dt * num - (denom - 1)) / denom;
870
            }
871
            else {
872 0
                dt = dt * num / denom;
873
            }
874
        }
875

876 1
        memcpy(dst, &dt, sizeof(dt));
877

878 1
        dst += dst_stride;
879 1
        src += src_stride;
880 1
        --N;
881
    }
882 1
    return 0;
883
}
884

885
static int
886 1
_aligned_strided_to_strided_datetime_cast(char *dst,
887
                        npy_intp dst_stride,
888
                        char *src, npy_intp src_stride,
889
                        npy_intp N, npy_intp src_itemsize,
890
                        NpyAuxData *data)
891
{
892 1
    _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data;
893 1
    npy_int64 num = d->num, denom = d->denom;
894
    npy_int64 dt;
895

896 1
    while (N > 0) {
897 1
        dt = *(npy_int64 *)src;
898

899 1
        if (dt != NPY_DATETIME_NAT) {
900
            /* Apply the scaling */
901 1
            if (dt < 0) {
902 1
                dt = (dt * num - (denom - 1)) / denom;
903
            }
904
            else {
905 1
                dt = dt * num / denom;
906
            }
907
        }
908

909 1
        *(npy_int64 *)dst = dt;
910

911 1
        dst += dst_stride;
912 1
        src += src_stride;
913 1
        --N;
914
    }
915 1
    return 0;
916
}
917

918
static int
919 1
_strided_to_strided_datetime_to_string(char *dst, npy_intp dst_stride,
920
                        char *src, npy_intp src_stride,
921
                        npy_intp N, npy_intp NPY_UNUSED(src_itemsize),
922
                        NpyAuxData *data)
923
{
924 1
    _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data;
925 1
    npy_intp dst_itemsize = d->dst_itemsize;
926
    npy_int64 dt;
927
    npy_datetimestruct dts;
928

929 1
    while (N > 0) {
930 1
        memcpy(&dt, src, sizeof(dt));
931

932 1
        if (convert_datetime_to_datetimestruct(&d->src_meta,
933
                                               dt, &dts) < 0) {
934
            return -1;
935
        }
936

937
        /* Initialize the destination to all zeros */
938 1
        memset(dst, 0, dst_itemsize);
939

940 1
        if (make_iso_8601_datetime(&dts, dst, dst_itemsize,
941
                                0, 0, d->src_meta.base, -1,
942
                                NPY_UNSAFE_CASTING) < 0) {
943
            return -1;
944
        }
945

946 1
        dst += dst_stride;
947 1
        src += src_stride;
948 1
        --N;
949
    }
950
    return 0;
951
}
952

953
static int
954 1
_strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride,
955
                        char *src, npy_intp src_stride,
956
                        npy_intp N, npy_intp src_itemsize,
957
                        NpyAuxData *data)
958
{
959 1
    _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data;
960
    npy_datetimestruct dts;
961 1
    char *tmp_buffer = d->tmp_buffer;
962
    char *tmp;
963

964 1
    while (N > 0) {
965 1
        npy_int64 dt = ~NPY_DATETIME_NAT;
966

967
        /* Replicating strnlen with memchr, because Mac OS X lacks it */
968 1
        tmp = memchr(src, '\0', src_itemsize);
969

970
        /* If the string is all full, use the buffer */
971 1
        if (tmp == NULL) {
972 1
            memcpy(tmp_buffer, src, src_itemsize);
973 1
            tmp_buffer[src_itemsize] = '\0';
974

975 1
            if (parse_iso_8601_datetime(tmp_buffer, src_itemsize,
976
                                    d->dst_meta.base, NPY_SAME_KIND_CASTING,
977
                                    &dts, NULL, NULL) < 0) {
978 0
                return -1;
979
            }
980
        }
981
        /* Otherwise parse the data in place */
982
        else {
983 1
            if (parse_iso_8601_datetime(src, tmp - src,
984
                                    d->dst_meta.base, NPY_SAME_KIND_CASTING,
985
                                    &dts, NULL, NULL) < 0) {
986
                return -1;
987
            }
988
        }
989

990
        /* Convert to the datetime */
991 1
        if (dt != NPY_DATETIME_NAT &&
992 1
                convert_datetimestruct_to_datetime(&d->dst_meta,
993
                                               &dts, &dt) < 0) {
994
            return -1;
995
        }
996

997 1
        memcpy(dst, &dt, sizeof(dt));
998

999 1
        dst += dst_stride;
1000 1
        src += src_stride;
1001 1
        --N;
1002
    }
1003
    return 0;
1004
}
1005

1006
/*
1007
 * Assumes src_dtype and dst_dtype are both datetimes or both timedeltas
1008
 */
1009
static int
1010 1
get_nbo_cast_datetime_transfer_function(int aligned,
1011
                            npy_intp src_stride, npy_intp dst_stride,
1012
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1013
                            PyArray_StridedUnaryOp **out_stransfer,
1014
                            NpyAuxData **out_transferdata)
1015
{
1016
    PyArray_DatetimeMetaData *src_meta, *dst_meta;
1017 1
    npy_int64 num = 0, denom = 0;
1018
    _strided_datetime_cast_data *data;
1019

1020 1
    src_meta = get_datetime_metadata_from_dtype(src_dtype);
1021 1
    if (src_meta == NULL) {
1022
        return NPY_FAIL;
1023
    }
1024 1
    dst_meta = get_datetime_metadata_from_dtype(dst_dtype);
1025 1
    if (dst_meta == NULL) {
1026
        return NPY_FAIL;
1027
    }
1028

1029 1
    get_datetime_conversion_factor(src_meta, dst_meta, &num, &denom);
1030

1031 1
    if (num == 0) {
1032
        return NPY_FAIL;
1033
    }
1034

1035
    /* Allocate the data for the casting */
1036 1
    data = (_strided_datetime_cast_data *)PyArray_malloc(
1037
                                    sizeof(_strided_datetime_cast_data));
1038 1
    if (data == NULL) {
1039 0
        PyErr_NoMemory();
1040 0
        *out_stransfer = NULL;
1041 0
        *out_transferdata = NULL;
1042
        return NPY_FAIL;
1043
    }
1044 1
    data->base.free = &_strided_datetime_cast_data_free;
1045 1
    data->base.clone = &_strided_datetime_cast_data_clone;
1046 1
    data->num = num;
1047 1
    data->denom = denom;
1048 1
    data->tmp_buffer = NULL;
1049

1050
    /*
1051
     * Special case the datetime (but not timedelta) with the nonlinear
1052
     * units (years and months). For timedelta, an average
1053
     * years and months value is used.
1054
     */
1055 1
    if (src_dtype->type_num == NPY_DATETIME &&
1056 1
            (src_meta->base == NPY_FR_Y ||
1057 1
             src_meta->base == NPY_FR_M ||
1058 1
             dst_meta->base == NPY_FR_Y ||
1059
             dst_meta->base == NPY_FR_M)) {
1060 1
        memcpy(&data->src_meta, src_meta, sizeof(data->src_meta));
1061 1
        memcpy(&data->dst_meta, dst_meta, sizeof(data->dst_meta));
1062 1
        *out_stransfer = &_strided_to_strided_datetime_general_cast;
1063
    }
1064 1
    else if (aligned) {
1065 1
        *out_stransfer = &_aligned_strided_to_strided_datetime_cast;
1066
    }
1067
    else {
1068 1
        *out_stransfer = &_strided_to_strided_datetime_cast;
1069
    }
1070 1
    *out_transferdata = (NpyAuxData *)data;
1071

1072
#if NPY_DT_DBG_TRACING
1073
    printf("Dtype transfer from ");
1074
    _safe_print((PyObject *)src_dtype);
1075
    printf(" to ");
1076
    _safe_print((PyObject *)dst_dtype);
1077
    printf("\n");
1078
    printf("has conversion fraction %lld/%lld\n", num, denom);
1079
#endif
1080

1081

1082
    return NPY_SUCCEED;
1083
}
1084

1085
static int
1086 1
get_nbo_datetime_to_string_transfer_function(int aligned,
1087
                            npy_intp src_stride, npy_intp dst_stride,
1088
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1089
                            PyArray_StridedUnaryOp **out_stransfer,
1090
                            NpyAuxData **out_transferdata)
1091
{
1092
    PyArray_DatetimeMetaData *src_meta;
1093
    _strided_datetime_cast_data *data;
1094

1095 1
    src_meta = get_datetime_metadata_from_dtype(src_dtype);
1096 1
    if (src_meta == NULL) {
1097
        return NPY_FAIL;
1098
    }
1099

1100
    /* Allocate the data for the casting */
1101 1
    data = (_strided_datetime_cast_data *)PyArray_malloc(
1102
                                    sizeof(_strided_datetime_cast_data));
1103 1
    if (data == NULL) {
1104 0
        PyErr_NoMemory();
1105 0
        *out_stransfer = NULL;
1106 0
        *out_transferdata = NULL;
1107
        return NPY_FAIL;
1108
    }
1109 1
    data->base.free = &_strided_datetime_cast_data_free;
1110 1
    data->base.clone = &_strided_datetime_cast_data_clone;
1111 1
    data->dst_itemsize = dst_dtype->elsize;
1112 1
    data->tmp_buffer = NULL;
1113

1114 1
    memcpy(&data->src_meta, src_meta, sizeof(data->src_meta));
1115

1116 1
    *out_stransfer = &_strided_to_strided_datetime_to_string;
1117 1
    *out_transferdata = (NpyAuxData *)data;
1118

1119
#if NPY_DT_DBG_TRACING
1120
    printf("Dtype transfer from ");
1121
    _safe_print((PyObject *)src_dtype);
1122
    printf(" to ");
1123
    _safe_print((PyObject *)dst_dtype);
1124
    printf("\n");
1125
#endif
1126

1127
    return NPY_SUCCEED;
1128
}
1129

1130
static int
1131 1
get_datetime_to_unicode_transfer_function(int aligned,
1132
                            npy_intp src_stride, npy_intp dst_stride,
1133
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1134
                            PyArray_StridedUnaryOp **out_stransfer,
1135
                            NpyAuxData **out_transferdata,
1136
                            int *out_needs_api)
1137
{
1138 1
    NpyAuxData *castdata = NULL, *todata = NULL, *fromdata = NULL;
1139
    PyArray_StridedUnaryOp *caststransfer, *tobuffer, *frombuffer;
1140
    PyArray_Descr *str_dtype;
1141

1142
    /* Get an ASCII string data type, adapted to match the UNICODE one */
1143 1
    str_dtype = PyArray_DescrFromType(NPY_STRING);
1144 1
    str_dtype = PyArray_AdaptFlexibleDType(dst_dtype, str_dtype);
1145 1
    if (str_dtype == NULL) {
1146
        return NPY_FAIL;
1147
    }
1148

1149
    /* Get the copy/swap operation to dst */
1150 1
    if (PyArray_GetDTypeCopySwapFn(aligned,
1151 1
                            src_stride, src_dtype->elsize,
1152
                            src_dtype,
1153
                            &tobuffer, &todata) != NPY_SUCCEED) {
1154 0
        Py_DECREF(str_dtype);
1155
        return NPY_FAIL;
1156
    }
1157

1158
    /* Get the NBO datetime to string aligned contig function */
1159 1
    if (get_nbo_datetime_to_string_transfer_function(1,
1160 1
                            src_dtype->elsize, str_dtype->elsize,
1161
                            src_dtype, str_dtype,
1162
                            &caststransfer, &castdata) != NPY_SUCCEED) {
1163 0
        Py_DECREF(str_dtype);
1164 0
        NPY_AUXDATA_FREE(todata);
1165
        return NPY_FAIL;
1166
    }
1167

1168
    /* Get the cast operation to dst */
1169 1
    if (PyArray_GetDTypeTransferFunction(aligned,
1170 1
                            str_dtype->elsize, dst_stride,
1171
                            str_dtype, dst_dtype,
1172
                            0,
1173
                            &frombuffer, &fromdata,
1174
                            out_needs_api) != NPY_SUCCEED) {
1175 0
        Py_DECREF(str_dtype);
1176 0
        NPY_AUXDATA_FREE(todata);
1177 0
        NPY_AUXDATA_FREE(castdata);
1178
        return NPY_FAIL;
1179
    }
1180

1181
    /* Wrap it all up in a new transfer function + data */
1182 1
    if (wrap_aligned_contig_transfer_function(
1183 1
                        src_dtype->elsize, str_dtype->elsize,
1184
                        tobuffer, todata,
1185
                        frombuffer, fromdata,
1186
                        caststransfer, castdata,
1187 1
                        PyDataType_FLAGCHK(str_dtype, NPY_NEEDS_INIT),
1188
                        *out_needs_api,
1189
                        out_stransfer, out_transferdata) != NPY_SUCCEED) {
1190 0
        NPY_AUXDATA_FREE(castdata);
1191 0
        NPY_AUXDATA_FREE(todata);
1192 0
        NPY_AUXDATA_FREE(fromdata);
1193
        return NPY_FAIL;
1194
    }
1195

1196 1
    Py_DECREF(str_dtype);
1197

1198
    return NPY_SUCCEED;
1199
}
1200

1201
static int
1202 1
get_nbo_string_to_datetime_transfer_function(int aligned,
1203
                            npy_intp src_stride, npy_intp dst_stride,
1204
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1205
                            PyArray_StridedUnaryOp **out_stransfer,
1206
                            NpyAuxData **out_transferdata)
1207
{
1208
    PyArray_DatetimeMetaData *dst_meta;
1209
    _strided_datetime_cast_data *data;
1210

1211 1
    dst_meta = get_datetime_metadata_from_dtype(dst_dtype);
1212 1
    if (dst_meta == NULL) {
1213
        return NPY_FAIL;
1214
    }
1215

1216
    /* Allocate the data for the casting */
1217 1
    data = (_strided_datetime_cast_data *)PyArray_malloc(
1218
                                    sizeof(_strided_datetime_cast_data));
1219 1
    if (data == NULL) {
1220 0
        PyErr_NoMemory();
1221 0
        *out_stransfer = NULL;
1222 0
        *out_transferdata = NULL;
1223
        return NPY_FAIL;
1224
    }
1225 1
    data->base.free = &_strided_datetime_cast_data_free;
1226 1
    data->base.clone = &_strided_datetime_cast_data_clone;
1227 1
    data->src_itemsize = src_dtype->elsize;
1228 1
    data->tmp_buffer = PyArray_malloc(data->src_itemsize + 1);
1229 1
    if (data->tmp_buffer == NULL) {
1230 0
        PyErr_NoMemory();
1231 0
        PyArray_free(data);
1232 0
        *out_stransfer = NULL;
1233 0
        *out_transferdata = NULL;
1234
        return NPY_FAIL;
1235
    }
1236

1237 1
    memcpy(&data->dst_meta, dst_meta, sizeof(data->dst_meta));
1238

1239 1
    *out_stransfer = &_strided_to_strided_string_to_datetime;
1240 1
    *out_transferdata = (NpyAuxData *)data;
1241

1242
#if NPY_DT_DBG_TRACING
1243
    printf("Dtype transfer from ");
1244
    _safe_print((PyObject *)src_dtype);
1245
    printf(" to ");
1246
    _safe_print((PyObject *)dst_dtype);
1247
    printf("\n");
1248
#endif
1249

1250
    return NPY_SUCCEED;
1251
}
1252

1253
static int
1254 1
get_unicode_to_datetime_transfer_function(int aligned,
1255
                            npy_intp src_stride, npy_intp dst_stride,
1256
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1257
                            PyArray_StridedUnaryOp **out_stransfer,
1258
                            NpyAuxData **out_transferdata,
1259
                            int *out_needs_api)
1260
{
1261 1
    NpyAuxData *castdata = NULL, *todata = NULL, *fromdata = NULL;
1262
    PyArray_StridedUnaryOp *caststransfer, *tobuffer, *frombuffer;
1263
    PyArray_Descr *str_dtype;
1264

1265
    /* Get an ASCII string data type, adapted to match the UNICODE one */
1266 1
    str_dtype = PyArray_DescrFromType(NPY_STRING);
1267 1
    str_dtype = PyArray_AdaptFlexibleDType(src_dtype, str_dtype);
1268 1
    if (str_dtype == NULL) {
1269
        return NPY_FAIL;
1270
    }
1271

1272
    /* Get the cast operation from src */
1273 1
    if (PyArray_GetDTypeTransferFunction(aligned,
1274 1
                            src_stride, str_dtype->elsize,
1275
                            src_dtype, str_dtype,
1276
                            0,
1277
                            &tobuffer, &todata,
1278
                            out_needs_api) != NPY_SUCCEED) {
1279 0
        Py_DECREF(str_dtype);
1280
        return NPY_FAIL;
1281
    }
1282

1283
    /* Get the string to NBO datetime aligned contig function */
1284 1
    if (get_nbo_string_to_datetime_transfer_function(1,
1285 1
                            str_dtype->elsize, dst_dtype->elsize,
1286
                            str_dtype, dst_dtype,
1287
                            &caststransfer, &castdata) != NPY_SUCCEED) {
1288 0
        Py_DECREF(str_dtype);
1289 0
        NPY_AUXDATA_FREE(todata);
1290
        return NPY_FAIL;
1291
    }
1292

1293
    /* Get the copy/swap operation to dst */
1294 1
    if (PyArray_GetDTypeCopySwapFn(aligned,
1295 1
                            dst_dtype->elsize, dst_stride,
1296
                            dst_dtype,
1297
                            &frombuffer, &fromdata) != NPY_SUCCEED) {
1298 0
        Py_DECREF(str_dtype);
1299 0
        NPY_AUXDATA_FREE(todata);
1300 0
        NPY_AUXDATA_FREE(castdata);
1301
        return NPY_FAIL;
1302
    }
1303

1304
    /* Wrap it all up in a new transfer function + data */
1305 1
    if (wrap_aligned_contig_transfer_function(
1306 1
                        str_dtype->elsize, dst_dtype->elsize,
1307
                        tobuffer, todata,
1308
                        frombuffer, fromdata,
1309
                        caststransfer, castdata,
1310 1
                        PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT),
1311
                        *out_needs_api,
1312
                        out_stransfer, out_transferdata) != NPY_SUCCEED) {
1313 0
        Py_DECREF(str_dtype);
1314 0
        NPY_AUXDATA_FREE(castdata);
1315 0
        NPY_AUXDATA_FREE(todata);
1316 0
        NPY_AUXDATA_FREE(fromdata);
1317
        return NPY_FAIL;
1318
    }
1319

1320 1
    Py_DECREF(str_dtype);
1321

1322
    return NPY_SUCCEED;
1323
}
1324

1325
static int
1326 1
get_nbo_cast_transfer_function(int aligned,
1327
                            npy_intp src_stride, npy_intp dst_stride,
1328
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1329
                            int move_references,
1330
                            PyArray_StridedUnaryOp **out_stransfer,
1331
                            NpyAuxData **out_transferdata,
1332
                            int *out_needs_api,
1333
                            int *out_needs_wrap)
1334
{
1335
    _strided_cast_data *data;
1336
    PyArray_VectorUnaryFunc *castfunc;
1337
    PyArray_Descr *tmp_dtype;
1338 1
    npy_intp shape = 1, src_itemsize = src_dtype->elsize,
1339 1
            dst_itemsize = dst_dtype->elsize;
1340

1341 1
    if (PyTypeNum_ISNUMBER(src_dtype->type_num) &&
1342 1
                    PyTypeNum_ISNUMBER(dst_dtype->type_num)) {
1343 1
        *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) ||
1344 1
                          !PyArray_ISNBO(dst_dtype->byteorder);
1345 1
        return get_nbo_cast_numeric_transfer_function(aligned,
1346
                                    src_stride, dst_stride,
1347
                                    src_dtype->type_num, dst_dtype->type_num,
1348
                                    out_stransfer, out_transferdata);
1349
    }
1350

1351 1
    if (src_dtype->type_num == NPY_DATETIME ||
1352 1
            src_dtype->type_num == NPY_TIMEDELTA ||
1353 1
            dst_dtype->type_num == NPY_DATETIME ||
1354
            dst_dtype->type_num == NPY_TIMEDELTA) {
1355
        /* A parameterized type, datetime->datetime sometimes needs casting */
1356 1
        if ((src_dtype->type_num == NPY_DATETIME &&
1357 1
                    dst_dtype->type_num == NPY_DATETIME) ||
1358 1
                (src_dtype->type_num == NPY_TIMEDELTA &&
1359 1
                    dst_dtype->type_num == NPY_TIMEDELTA)) {
1360 1
            *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) ||
1361 1
                              !PyArray_ISNBO(dst_dtype->byteorder);
1362 1
            return get_nbo_cast_datetime_transfer_function(aligned,
1363
                                        src_stride, dst_stride,
1364
                                        src_dtype, dst_dtype,
1365
                                        out_stransfer, out_transferdata);
1366
        }
1367

1368
        /*
1369
         * Datetime <-> string conversions can be handled specially.
1370
         * The functions may raise an error if the strings have no
1371
         * space, or can't be parsed properly.
1372
         */
1373 1
        if (src_dtype->type_num == NPY_DATETIME) {
1374 1
            switch (dst_dtype->type_num) {
1375 1
                case NPY_STRING:
1376 1
                    *out_needs_api = 1;
1377 1
                    *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder);
1378 1
                    return get_nbo_datetime_to_string_transfer_function(
1379
                                        aligned,
1380
                                        src_stride, dst_stride,
1381
                                        src_dtype, dst_dtype,
1382
                                        out_stransfer, out_transferdata);
1383

1384 1
                case NPY_UNICODE:
1385 1
                    return get_datetime_to_unicode_transfer_function(
1386
                                        aligned,
1387
                                        src_stride, dst_stride,
1388
                                        src_dtype, dst_dtype,
1389
                                        out_stransfer, out_transferdata,
1390
                                        out_needs_api);
1391
            }
1392
        }
1393 1
        else if (dst_dtype->type_num == NPY_DATETIME) {
1394 1
            switch (src_dtype->type_num) {
1395 1
                case NPY_STRING:
1396 1
                    *out_needs_api = 1;
1397 1
                    *out_needs_wrap = !PyArray_ISNBO(dst_dtype->byteorder);
1398 1
                    return get_nbo_string_to_datetime_transfer_function(
1399
                                        aligned,
1400
                                        src_stride, dst_stride,
1401
                                        src_dtype, dst_dtype,
1402
                                        out_stransfer, out_transferdata);
1403

1404 1
                case NPY_UNICODE:
1405 1
                    return get_unicode_to_datetime_transfer_function(
1406
                                        aligned,
1407
                                        src_stride, dst_stride,
1408
                                        src_dtype, dst_dtype,
1409
                                        out_stransfer, out_transferdata,
1410
                                        out_needs_api);
1411
            }
1412
        }
1413
    }
1414

1415 1
    *out_needs_wrap = !aligned ||
1416 1
                      !PyArray_ISNBO(src_dtype->byteorder) ||
1417 1
                      !PyArray_ISNBO(dst_dtype->byteorder);
1418

1419
    /* Check the data types whose casting functions use API calls */
1420 1
    switch (src_dtype->type_num) {
1421 1
        case NPY_OBJECT:
1422
        case NPY_STRING:
1423
        case NPY_UNICODE:
1424
        case NPY_VOID:
1425 1
            if (out_needs_api) {
1426 1
                *out_needs_api = 1;
1427
            }
1428
            break;
1429
    }
1430 1
    switch (dst_dtype->type_num) {
1431 1
        case NPY_OBJECT:
1432
        case NPY_STRING:
1433
        case NPY_UNICODE:
1434
        case NPY_VOID:
1435 1
            if (out_needs_api) {
1436 1
                *out_needs_api = 1;
1437
            }
1438
            break;
1439
    }
1440

1441 1
    if (PyDataType_FLAGCHK(src_dtype, NPY_NEEDS_PYAPI) ||
1442 1
            PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_PYAPI)) {
1443 1
        if (out_needs_api) {
1444 1
            *out_needs_api = 1;
1445
        }
1446
    }
1447

1448
    /* Get the cast function */
1449 1
    castfunc = PyArray_GetCastFunc(src_dtype, dst_dtype->type_num);
1450 1
    if (!castfunc) {
1451 1
        *out_stransfer = NULL;
1452 1
        *out_transferdata = NULL;
1453 1
        return NPY_FAIL;
1454
    }
1455

1456
    /* Allocate the data for the casting */
1457 1
    data = (_strided_cast_data *)PyArray_malloc(sizeof(_strided_cast_data));
1458 1
    if (data == NULL) {
1459 0
        PyErr_NoMemory();
1460 0
        *out_stransfer = NULL;
1461 0
        *out_transferdata = NULL;
1462 0
        return NPY_FAIL;
1463
    }
1464 1
    data->base.free = &_strided_cast_data_free;
1465 1
    data->base.clone = &_strided_cast_data_clone;
1466 1
    data->castfunc = castfunc;
1467 1
    data->needs_api = *out_needs_api;
1468
    /*
1469
     * TODO: This is a hack so the cast functions have an array.
1470
     *       The cast functions shouldn't need that.  Also, since we
1471
     *       always handle byte order conversions, this array should
1472
     *       have native byte order.
1473
     */
1474 1
    if (PyArray_ISNBO(src_dtype->byteorder)) {
1475 1
        tmp_dtype = src_dtype;
1476 1
        Py_INCREF(tmp_dtype);
1477
    }
1478
    else {
1479 1
        tmp_dtype = PyArray_DescrNewByteorder(src_dtype, NPY_NATIVE);
1480 1
        if (tmp_dtype == NULL) {
1481 0
            PyArray_free(data);
1482 0
            return NPY_FAIL;
1483
        }
1484
    }
1485 1
    data->aip = (PyArrayObject *)PyArray_NewFromDescr_int(
1486
            &PyArray_Type, tmp_dtype,
1487
            1, &shape, NULL, NULL,
1488
            0, NULL, NULL,
1489
            0, 1);
1490 1
    if (data->aip == NULL) {
1491 0
        PyArray_free(data);
1492 0
        return NPY_FAIL;
1493
    }
1494
    /*
1495
     * TODO: This is a hack so the cast functions have an array.
1496
     *       The cast functions shouldn't need that.  Also, since we
1497
     *       always handle byte order conversions, this array should
1498
     *       have native byte order.
1499
     */
1500 1
    if (PyArray_ISNBO(dst_dtype->byteorder)) {
1501 1
        tmp_dtype = dst_dtype;
1502 1
        Py_INCREF(tmp_dtype);
1503
    }
1504
    else {
1505 0
        tmp_dtype = PyArray_DescrNewByteorder(dst_dtype, NPY_NATIVE);
1506 0
        if (tmp_dtype == NULL) {
1507 0
            Py_DECREF(data->aip);
1508 0
            PyArray_free(data);
1509 0
            return NPY_FAIL;
1510
        }
1511
    }
1512 1
    data->aop = (PyArrayObject *)PyArray_NewFromDescr_int(
1513
            &PyArray_Type, tmp_dtype,
1514
            1, &shape, NULL, NULL,
1515
            0, NULL, NULL,
1516
            0, 1);
1517 1
    if (data->aop == NULL) {
1518 0
        Py_DECREF(data->aip);
1519 0
        PyArray_free(data);
1520 0
        return NPY_FAIL;
1521
    }
1522

1523
    /* If it's aligned and all native byte order, we're all done */
1524 1
    if (move_references && src_dtype->type_num == NPY_OBJECT) {
1525 1
        *out_stransfer = _aligned_strided_to_strided_cast_decref_src;
1526
    }
1527
    else {
1528
        /*
1529
         * Use the contig version if the strides are contiguous or
1530
         * we're telling the caller to wrap the return, because
1531
         * the wrapping uses a contiguous buffer.
1532
         */
1533 1
        if ((src_stride == src_itemsize && dst_stride == dst_itemsize) ||
1534 1
                        *out_needs_wrap) {
1535 1
            *out_stransfer = _aligned_contig_to_contig_cast;
1536
        }
1537
        else {
1538 1
            *out_stransfer = _aligned_strided_to_strided_cast;
1539
        }
1540
    }
1541 1
    *out_transferdata = (NpyAuxData *)data;
1542

1543 1
    return NPY_SUCCEED;
1544
}
1545

1546
static int
1547 1
get_cast_transfer_function(int aligned,
1548
                            npy_intp src_stride, npy_intp dst_stride,
1549
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1550
                            int move_references,
1551
                            PyArray_StridedUnaryOp **out_stransfer,
1552
                            NpyAuxData **out_transferdata,
1553
                            int *out_needs_api)
1554
{
1555
    PyArray_StridedUnaryOp *caststransfer;
1556 1
    NpyAuxData *castdata, *todata = NULL, *fromdata = NULL;
1557 1
    int needs_wrap = 0;
1558 1
    npy_intp src_itemsize = src_dtype->elsize,
1559 1
            dst_itemsize = dst_dtype->elsize;
1560

1561 1
    if (get_nbo_cast_transfer_function(aligned,
1562
                            src_stride, dst_stride,
1563
                            src_dtype, dst_dtype,
1564
                            move_references,
1565
                            &caststransfer,
1566
                            &castdata,
1567
                            out_needs_api,
1568
                            &needs_wrap) != NPY_SUCCEED) {
1569
        return NPY_FAIL;
1570
    }
1571

1572
    /*
1573
     * If all native byte order and doesn't need alignment wrapping,
1574
     * return the function
1575
     */
1576 1
    if (!needs_wrap) {
1577 1
        *out_stransfer = caststransfer;
1578 1
        *out_transferdata = castdata;
1579

1580 1
        return NPY_SUCCEED;
1581
    }
1582
    /* Otherwise, we have to copy and/or swap to aligned temporaries */
1583
    else {
1584
        PyArray_StridedUnaryOp *tobuffer, *frombuffer;
1585

1586
        /* Get the copy/swap operation from src */
1587 1
        PyArray_GetDTypeCopySwapFn(aligned,
1588
                                src_stride, src_itemsize,
1589
                                src_dtype,
1590
                                &tobuffer, &todata);
1591

1592 1
        if (!PyDataType_REFCHK(dst_dtype)) {
1593
            /* Copying from buffer is a simple copy/swap operation */
1594 1
            PyArray_GetDTypeCopySwapFn(aligned,
1595
                                    dst_itemsize, dst_stride,
1596
                                    dst_dtype,
1597
                                    &frombuffer, &fromdata);
1598
        }
1599
        else {
1600
            /*
1601
             * Since the buffer is initialized to NULL, need to move the
1602
             * references in order to DECREF the existing data.
1603
             */
1604
             /* Object types cannot be byte swapped */
1605
            assert(PyDataType_ISNOTSWAPPED(dst_dtype));
1606
            /* The loop already needs the python api if this is reached */
1607
            assert(*out_needs_api);
1608

1609 1
            if (PyArray_GetDTypeTransferFunction(
1610
                    aligned, dst_itemsize, dst_stride,
1611
                    dst_dtype, dst_dtype, 1,
1612
                    &frombuffer, &fromdata, out_needs_api) != NPY_SUCCEED) {
1613
                return NPY_FAIL;
1614
            }
1615
        }
1616

1617 1
        if (frombuffer == NULL || tobuffer == NULL) {
1618 0
            NPY_AUXDATA_FREE(castdata);
1619 0
            NPY_AUXDATA_FREE(todata);
1620 0
            NPY_AUXDATA_FREE(fromdata);
1621
            return NPY_FAIL;
1622
        }
1623

1624 1
        *out_stransfer = caststransfer;
1625

1626
        /* Wrap it all up in a new transfer function + data */
1627 1
        if (wrap_aligned_contig_transfer_function(
1628
                            src_itemsize, dst_itemsize,
1629
                            tobuffer, todata,
1630
                            frombuffer, fromdata,
1631
                            caststransfer, castdata,
1632 1
                            PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT),
1633
                            *out_needs_api,
1634
                            out_stransfer, out_transferdata) != NPY_SUCCEED) {
1635 0
            NPY_AUXDATA_FREE(castdata);
1636 0
            NPY_AUXDATA_FREE(todata);
1637 0
            NPY_AUXDATA_FREE(fromdata);
1638
            return NPY_FAIL;
1639
        }
1640

1641
        return NPY_SUCCEED;
1642
    }
1643
}
1644

1645
/**************************** COPY 1 TO N CONTIGUOUS ************************/
1646

1647
/* Copies 1 element to N contiguous elements */
1648
typedef struct {
1649
    NpyAuxData base;
1650
    PyArray_StridedUnaryOp *stransfer;
1651
    NpyAuxData *data;
1652
    npy_intp N, dst_itemsize;
1653
    /* If this is non-NULL the source type has references needing a decref */
1654
    PyArray_StridedUnaryOp *stransfer_finish_src;
1655
    NpyAuxData *data_finish_src;
1656
} _one_to_n_data;
1657

1658
/* transfer data free function */
1659 1
static void _one_to_n_data_free(NpyAuxData *data)
1660
{
1661 1
    _one_to_n_data *d = (_one_to_n_data *)data;
1662 1
    NPY_AUXDATA_FREE(d->data);
1663 1
    NPY_AUXDATA_FREE(d->data_finish_src);
1664 1
    PyArray_free(data);
1665
}
1666

1667
/* transfer data copy function */
1668 0
static NpyAuxData *_one_to_n_data_clone(NpyAuxData *data)
1669
{
1670 0
    _one_to_n_data *d = (_one_to_n_data *)data;
1671
    _one_to_n_data *newdata;
1672

1673
    /* Allocate the data, and populate it */
1674 0
    newdata = (_one_to_n_data *)PyArray_malloc(sizeof(_one_to_n_data));
1675 0
    if (newdata == NULL) {
1676
        return NULL;
1677
    }
1678 0
    memcpy(newdata, data, sizeof(_one_to_n_data));
1679 0
    if (d->data != NULL) {
1680 0
        newdata->data = NPY_AUXDATA_CLONE(d->data);
1681 0
        if (newdata->data == NULL) {
1682 0
            PyArray_free(newdata);
1683 0
            return NULL;
1684
        }
1685
    }
1686 0
    if (d->data_finish_src != NULL) {
1687 0
        newdata->data_finish_src = NPY_AUXDATA_CLONE(d->data_finish_src);
1688 0
        if (newdata->data_finish_src == NULL) {
1689 0
            NPY_AUXDATA_FREE(newdata->data);
1690 0
            PyArray_free(newdata);
1691 0
            return NULL;
1692
        }
1693
    }
1694

1695
    return (NpyAuxData *)newdata;
1696
}
1697

1698
static int
1699 1
_strided_to_strided_one_to_n(char *dst, npy_intp dst_stride,
1700
                        char *src, npy_intp src_stride,
1701
                        npy_intp N, npy_intp src_itemsize,
1702
                        NpyAuxData *data)
1703
{
1704 1
    _one_to_n_data *d = (_one_to_n_data *)data;
1705 1
    PyArray_StridedUnaryOp *subtransfer = d->stransfer;
1706 1
    NpyAuxData *subdata = d->data;
1707 1
    npy_intp subN = d->N, dst_itemsize = d->dst_itemsize;
1708

1709 1
    while (N > 0) {
1710 1
        if (subtransfer(
1711
                dst, dst_itemsize, src, 0, subN, src_itemsize, subdata) < 0) {
1712
            return -1;
1713
        }
1714

1715 1
        src += src_stride;
1716 1
        dst += dst_stride;
1717 1
        --N;
1718
    }
1719
    return 0;
1720
}
1721

1722
static int
1723 1
_strided_to_strided_one_to_n_with_finish(char *dst, npy_intp dst_stride,
1724
                        char *src, npy_intp src_stride,
1725
                        npy_intp N, npy_intp src_itemsize,
1726
                        NpyAuxData *data)
1727
{
1728 1
    _one_to_n_data *d = (_one_to_n_data *)data;
1729 1
    PyArray_StridedUnaryOp *subtransfer = d->stransfer,
1730 1
                *stransfer_finish_src = d->stransfer_finish_src;
1731 1
    NpyAuxData *subdata = d->data, *data_finish_src = d->data_finish_src;
1732 1
    npy_intp subN = d->N, dst_itemsize = d->dst_itemsize;
1733

1734 1
    while (N > 0) {
1735 1
        if (subtransfer(
1736
                dst, dst_itemsize, src, 0, subN, src_itemsize, subdata) < 0) {
1737
            return -1;
1738
        }
1739

1740 1
        if (stransfer_finish_src(
1741
                NULL, 0, src, 0, 1, src_itemsize, data_finish_src) < 0) {
1742
            return -1;
1743
        }
1744

1745 1
        src += src_stride;
1746 1
        dst += dst_stride;
1747 1
        --N;
1748
    }
1749
    return 0;
1750
}
1751

1752
/*
1753
 * Wraps a transfer function to produce one that copies one element
1754
 * of src to N contiguous elements of dst.  If stransfer_finish_src is
1755
 * not NULL, it should be a transfer function which just affects
1756
 * src, for example to do a final DECREF operation for references.
1757
 */
1758
static int
1759 1
wrap_transfer_function_one_to_n(
1760
                            PyArray_StridedUnaryOp *stransfer_inner,
1761
                            NpyAuxData *data_inner,
1762
                            PyArray_StridedUnaryOp *stransfer_finish_src,
1763
                            NpyAuxData *data_finish_src,
1764
                            npy_intp dst_itemsize,
1765
                            npy_intp N,
1766
                            PyArray_StridedUnaryOp **out_stransfer,
1767
                            NpyAuxData **out_transferdata)
1768
{
1769
    _one_to_n_data *data;
1770

1771

1772 1
    data = PyArray_malloc(sizeof(_one_to_n_data));
1773 1
    if (data == NULL) {
1774 0
        PyErr_NoMemory();
1775 0
        return NPY_FAIL;
1776
    }
1777

1778 1
    data->base.free = &_one_to_n_data_free;
1779 1
    data->base.clone = &_one_to_n_data_clone;
1780 1
    data->stransfer = stransfer_inner;
1781 1
    data->data = data_inner;
1782 1
    data->stransfer_finish_src = stransfer_finish_src;
1783 1
    data->data_finish_src = data_finish_src;
1784 1
    data->N = N;
1785 1
    data->dst_itemsize = dst_itemsize;
1786

1787 1
    if (stransfer_finish_src == NULL) {
1788 1
        *out_stransfer = &_strided_to_strided_one_to_n;
1789
    }
1790
    else {
1791 1
        *out_stransfer = &_strided_to_strided_one_to_n_with_finish;
1792
    }
1793 1
    *out_transferdata = (NpyAuxData *)data;
1794

1795 1
    return NPY_SUCCEED;
1796
}
1797

1798
static int
1799 1
get_one_to_n_transfer_function(int aligned,
1800
                            npy_intp src_stride, npy_intp dst_stride,
1801
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1802
                            int move_references,
1803
                            npy_intp N,
1804
                            PyArray_StridedUnaryOp **out_stransfer,
1805
                            NpyAuxData **out_transferdata,
1806
                            int *out_needs_api)
1807
{
1808 1
    PyArray_StridedUnaryOp *stransfer, *stransfer_finish_src = NULL;
1809 1
    NpyAuxData *data, *data_finish_src = NULL;
1810

1811
    /*
1812
     * move_references is set to 0, handled in the wrapping transfer fn,
1813
     * src_stride is set to zero, because its 1 to N copying,
1814
     * and dst_stride is set to contiguous, because subarrays are always
1815
     * contiguous.
1816
     */
1817 1
    if (PyArray_GetDTypeTransferFunction(aligned,
1818 1
                    0, dst_dtype->elsize,
1819
                    src_dtype, dst_dtype,
1820
                    0,
1821
                    &stransfer, &data,
1822
                    out_needs_api) != NPY_SUCCEED) {
1823
        return NPY_FAIL;
1824
    }
1825

1826
    /* If the src object will need a DECREF, set src_dtype */
1827 1
    if (move_references && PyDataType_REFCHK(src_dtype)) {
1828 1
        if (get_decsrcref_transfer_function(aligned,
1829
                            src_stride,
1830
                            src_dtype,
1831
                            &stransfer_finish_src,
1832
                            &data_finish_src,
1833
                            out_needs_api) != NPY_SUCCEED) {
1834 0
            NPY_AUXDATA_FREE(data);
1835
            return NPY_FAIL;
1836
        }
1837
    }
1838

1839 1
    if (wrap_transfer_function_one_to_n(stransfer, data,
1840
                            stransfer_finish_src, data_finish_src,
1841 1
                            dst_dtype->elsize,
1842
                            N,
1843
                            out_stransfer, out_transferdata) != NPY_SUCCEED) {
1844 0
        NPY_AUXDATA_FREE(data);
1845 0
        NPY_AUXDATA_FREE(data_finish_src);
1846
        return NPY_FAIL;
1847
    }
1848

1849
    return NPY_SUCCEED;
1850
}
1851

1852
/**************************** COPY N TO N CONTIGUOUS ************************/
1853

1854
/* Copies N contiguous elements to N contiguous elements */
1855
typedef struct {
1856
    NpyAuxData base;
1857
    PyArray_StridedUnaryOp *stransfer;
1858
    NpyAuxData *data;
1859
    npy_intp N, src_itemsize, dst_itemsize;
1860
} _n_to_n_data;
1861

1862
/* transfer data free function */
1863 1
static void _n_to_n_data_free(NpyAuxData *data)
1864
{
1865 1
    _n_to_n_data *d = (_n_to_n_data *)data;
1866 1
    NPY_AUXDATA_FREE(d->data);
1867 1
    PyArray_free(data);
1868
}
1869

1870
/* transfer data copy function */
1871 0
static NpyAuxData *_n_to_n_data_clone(NpyAuxData *data)
1872
{
1873 0
    _n_to_n_data *d = (_n_to_n_data *)data;
1874
    _n_to_n_data *newdata;
1875

1876
    /* Allocate the data, and populate it */
1877 0
    newdata = (_n_to_n_data *)PyArray_malloc(sizeof(_n_to_n_data));
1878 0
    if (newdata == NULL) {
1879
        return NULL;
1880
    }
1881 0
    memcpy(newdata, data, sizeof(_n_to_n_data));
1882 0
    if (newdata->data != NULL) {
1883 0
        newdata->data = NPY_AUXDATA_CLONE(d->data);
1884 0
        if (newdata->data == NULL) {
1885 0
            PyArray_free(newdata);
1886 0
            return NULL;
1887
        }
1888
    }
1889

1890
    return (NpyAuxData *)newdata;
1891
}
1892

1893
static int
1894 1
_strided_to_strided_n_to_n(char *dst, npy_intp dst_stride,
1895
                        char *src, npy_intp src_stride,
1896
                        npy_intp N, npy_intp src_itemsize,
1897
                        NpyAuxData *data)
1898
{
1899 1
    _n_to_n_data *d = (_n_to_n_data *)data;
1900 1
    PyArray_StridedUnaryOp *subtransfer = d->stransfer;
1901 1
    NpyAuxData *subdata = d->data;
1902 1
    npy_intp subN = d->N, src_subitemsize = d->src_itemsize,
1903 1
                dst_subitemsize = d->dst_itemsize;
1904

1905 1
    while (N > 0) {
1906 1
        if (subtransfer(
1907
                dst, dst_subitemsize, src, src_subitemsize,
1908
                subN, src_subitemsize, subdata) < 0) {
1909
            return -1;
1910
        }
1911 1
        src += src_stride;
1912 1
        dst += dst_stride;
1913 1
        --N;
1914
    }
1915
    return 0;
1916
}
1917

1918
static int
1919 1
_contig_to_contig_n_to_n(char *dst, npy_intp NPY_UNUSED(dst_stride),
1920
                        char *src, npy_intp NPY_UNUSED(src_stride),
1921
                        npy_intp N, npy_intp NPY_UNUSED(src_itemsize),
1922
                        NpyAuxData *data)
1923
{
1924 1
    _n_to_n_data *d = (_n_to_n_data *)data;
1925 1
    PyArray_StridedUnaryOp *subtransfer = d->stransfer;
1926 1
    NpyAuxData *subdata = d->data;
1927 1
    npy_intp subN = d->N, src_subitemsize = d->src_itemsize,
1928 1
                dst_subitemsize = d->dst_itemsize;
1929

1930 1
    if (subtransfer(
1931
            dst, dst_subitemsize, src, src_subitemsize,
1932
            subN*N, src_subitemsize, subdata) < 0) {
1933
        return -1;
1934
    }
1935 1
    return 0;
1936
}
1937

1938
/*
1939
 * Wraps a transfer function to produce one that copies N contiguous elements
1940
 * of src to N contiguous elements of dst.
1941
 */
1942
static int
1943 1
wrap_transfer_function_n_to_n(
1944
                            PyArray_StridedUnaryOp *stransfer_inner,
1945
                            NpyAuxData *data_inner,
1946
                            npy_intp src_stride, npy_intp dst_stride,
1947
                            npy_intp src_itemsize, npy_intp dst_itemsize,
1948
                            npy_intp N,
1949
                            PyArray_StridedUnaryOp **out_stransfer,
1950
                            NpyAuxData **out_transferdata)
1951
{
1952
    _n_to_n_data *data;
1953

1954 1
    data = PyArray_malloc(sizeof(_n_to_n_data));
1955 1
    if (data == NULL) {
1956 0
        PyErr_NoMemory();
1957 0
        return NPY_FAIL;
1958
    }
1959

1960 1
    data->base.free = &_n_to_n_data_free;
1961 1
    data->base.clone = &_n_to_n_data_clone;
1962 1
    data->stransfer = stransfer_inner;
1963 1
    data->data = data_inner;
1964 1
    data->N = N;
1965 1
    data->src_itemsize = src_itemsize;
1966 1
    data->dst_itemsize = dst_itemsize;
1967

1968
    /*
1969
     * If the N subarray elements exactly fit in the strides,
1970
     * then can do a faster contiguous transfer.
1971
     */
1972 1
    if (src_stride == N * src_itemsize &&
1973 1
                    dst_stride == N * dst_itemsize) {
1974 1
        *out_stransfer = &_contig_to_contig_n_to_n;
1975
    }
1976
    else {
1977 1
        *out_stransfer = &_strided_to_strided_n_to_n;
1978
    }
1979 1
    *out_transferdata = (NpyAuxData *)data;
1980

1981 1
    return NPY_SUCCEED;
1982
}
1983

1984
static int
1985 1
get_n_to_n_transfer_function(int aligned,
1986
                            npy_intp src_stride, npy_intp dst_stride,
1987
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
1988
                            int move_references,
1989
                            npy_intp N,
1990
                            PyArray_StridedUnaryOp **out_stransfer,
1991
                            NpyAuxData **out_transferdata,
1992
                            int *out_needs_api)
1993
{
1994
    PyArray_StridedUnaryOp *stransfer;
1995
    NpyAuxData *data;
1996

1997
    /*
1998
     * src_stride and dst_stride are set to contiguous, because
1999
     * subarrays are always contiguous.
2000
     */
2001 1
    if (PyArray_GetDTypeTransferFunction(aligned,
2002 1
                    src_dtype->elsize, dst_dtype->elsize,
2003
                    src_dtype, dst_dtype,
2004
                    move_references,
2005
                    &stransfer, &data,
2006
                    out_needs_api) != NPY_SUCCEED) {
2007
        return NPY_FAIL;
2008
    }
2009

2010 1
    if (wrap_transfer_function_n_to_n(stransfer, data,
2011
                            src_stride, dst_stride,
2012 1
                            src_dtype->elsize, dst_dtype->elsize,
2013
                            N,
2014
                            out_stransfer,
2015
                            out_transferdata) != NPY_SUCCEED) {
2016 0
        NPY_AUXDATA_FREE(data);
2017
        return NPY_FAIL;
2018
    }
2019

2020
    return NPY_SUCCEED;
2021
}
2022

2023
/********************** COPY WITH SUBARRAY BROADCAST ************************/
2024

2025
typedef struct {
2026
    npy_intp offset, count;
2027
} _subarray_broadcast_offsetrun;
2028

2029
/* Copies element with subarray broadcasting */
2030
typedef struct {
2031
    NpyAuxData base;
2032
    PyArray_StridedUnaryOp *stransfer;
2033
    NpyAuxData *data;
2034
    npy_intp src_N, dst_N, src_itemsize, dst_itemsize;
2035
    PyArray_StridedUnaryOp *stransfer_decsrcref;
2036
    NpyAuxData *data_decsrcref;
2037
    PyArray_StridedUnaryOp *stransfer_decdstref;
2038
    NpyAuxData *data_decdstref;
2039
    /* This gets a run-length encoded representation of the transfer */
2040
    npy_intp run_count;
2041
    _subarray_broadcast_offsetrun offsetruns;
2042
} _subarray_broadcast_data;
2043

2044

2045
/* transfer data free function */
2046 1
static void _subarray_broadcast_data_free(NpyAuxData *data)
2047
{
2048 1
    _subarray_broadcast_data *d = (_subarray_broadcast_data *)data;
2049 1
    NPY_AUXDATA_FREE(d->data);
2050 1
    NPY_AUXDATA_FREE(d->data_decsrcref);
2051 1
    NPY_AUXDATA_FREE(d->data_decdstref);
2052 1
    PyArray_free(data);
2053
}
2054

2055
/* transfer data copy function */
2056 0
static NpyAuxData *_subarray_broadcast_data_clone( NpyAuxData *data)
2057
{
2058 0
    _subarray_broadcast_data *d = (_subarray_broadcast_data *)data;
2059
    _subarray_broadcast_data *newdata;
2060 0
    npy_intp run_count = d->run_count, structsize;
2061

2062 0
    structsize = sizeof(_subarray_broadcast_data) +
2063 0
                        run_count*sizeof(_subarray_broadcast_offsetrun);
2064

2065
    /* Allocate the data and populate it */
2066 0
    newdata = (_subarray_broadcast_data *)PyArray_malloc(structsize);
2067 0
    if (newdata == NULL) {
2068
        return NULL;
2069
    }
2070 0
    memcpy(newdata, data, structsize);
2071 0
    if (d->data != NULL) {
2072 0
        newdata->data = NPY_AUXDATA_CLONE(d->data);
2073 0
        if (newdata->data == NULL) {
2074 0
            PyArray_free(newdata);
2075 0
            return NULL;
2076
        }
2077
    }
2078 0
    if (d->data_decsrcref != NULL) {
2079 0
        newdata->data_decsrcref = NPY_AUXDATA_CLONE(d->data_decsrcref);
2080 0
        if (newdata->data_decsrcref == NULL) {
2081 0
            NPY_AUXDATA_FREE(newdata->data);
2082 0
            PyArray_free(newdata);
2083 0
            return NULL;
2084
        }
2085
    }
2086 0
    if (d->data_decdstref != NULL) {
2087 0
        newdata->data_decdstref = NPY_AUXDATA_CLONE(d->data_decdstref);
2088 0
        if (newdata->data_decdstref == NULL) {
2089 0
            NPY_AUXDATA_FREE(newdata->data);
2090 0
            NPY_AUXDATA_FREE(newdata->data_decsrcref);
2091 0
            PyArray_free(newdata);
2092 0
            return NULL;
2093
        }
2094
    }
2095

2096
    return (NpyAuxData *)newdata;
2097
}
2098

2099
static int
2100 1
_strided_to_strided_subarray_broadcast(char *dst, npy_intp dst_stride,
2101
                        char *src, npy_intp src_stride,
2102
                        npy_intp N, npy_intp NPY_UNUSED(src_itemsize),
2103
                        NpyAuxData *data)
2104
{
2105 1
    _subarray_broadcast_data *d = (_subarray_broadcast_data *)data;
2106 1
    PyArray_StridedUnaryOp *subtransfer = d->stransfer;
2107 1
    NpyAuxData *subdata = d->data;
2108 1
    npy_intp run, run_count = d->run_count,
2109 1
            src_subitemsize = d->src_itemsize,
2110 1
            dst_subitemsize = d->dst_itemsize;
2111
    npy_intp loop_index, offset, count;
2112
    char *dst_ptr;
2113 1
    _subarray_broadcast_offsetrun *offsetruns = &d->offsetruns;
2114

2115 1
    while (N > 0) {
2116
        loop_index = 0;
2117 1
        for (run = 0; run < run_count; ++run) {
2118 1
            offset = offsetruns[run].offset;
2119 1
            count = offsetruns[run].count;
2120 1
            dst_ptr = dst + loop_index*dst_subitemsize;
2121 1
            if (offset != -1) {
2122 1
                if (subtransfer(
2123
                        dst_ptr, dst_subitemsize, src + offset, src_subitemsize,
2124
                        count, src_subitemsize, subdata) < 0) {
2125
                    return -1;
2126
                }
2127
            }
2128
            else {
2129 1
                memset(dst_ptr, 0, count*dst_subitemsize);
2130
            }
2131 1
            loop_index += count;
2132
        }
2133

2134 1
        src += src_stride;
2135 1
        dst += dst_stride;
2136 1
        --N;
2137
    }
2138
    return 0;
2139
}
2140

2141

2142
static int
2143 1
_strided_to_strided_subarray_broadcast_withrefs(char *dst, npy_intp dst_stride,
2144
                        char *src, npy_intp src_stride,
2145
                        npy_intp N, npy_intp NPY_UNUSED(src_itemsize),
2146
                        NpyAuxData *data)
2147
{
2148 1
    _subarray_broadcast_data *d = (_subarray_broadcast_data *)data;
2149 1
    PyArray_StridedUnaryOp *subtransfer = d->stransfer;
2150 1
    NpyAuxData *subdata = d->data;
2151 1
    PyArray_StridedUnaryOp *stransfer_decsrcref = d->stransfer_decsrcref;
2152 1
    NpyAuxData *data_decsrcref = d->data_decsrcref;
2153 1
    PyArray_StridedUnaryOp *stransfer_decdstref = d->stransfer_decdstref;
2154 1
    NpyAuxData *data_decdstref = d->data_decdstref;
2155 1
    npy_intp run, run_count = d->run_count,
2156 1
            src_subitemsize = d->src_itemsize,
2157 1
            dst_subitemsize = d->dst_itemsize,
2158 1
            src_subN = d->src_N;
2159
    npy_intp loop_index, offset, count;
2160
    char *dst_ptr;
2161 1
    _subarray_broadcast_offsetrun *offsetruns = &d->offsetruns;
2162

2163 1
    while (N > 0) {
2164
        loop_index = 0;
2165 1
        for (run = 0; run < run_count; ++run) {
2166 1
            offset = offsetruns[run].offset;
2167 1
            count = offsetruns[run].count;
2168 1
            dst_ptr = dst + loop_index*dst_subitemsize;
2169 1
            if (offset != -1) {
2170 1
                if (subtransfer(
2171
                        dst_ptr, dst_subitemsize, src + offset, src_subitemsize,
2172
                        count, src_subitemsize, subdata) < 0) {
2173
                    return -1;
2174
                }
2175
            }
2176
            else {
2177 0
                if (stransfer_decdstref != NULL) {
2178 0
                    if (stransfer_decdstref(
2179
                            NULL, 0, dst_ptr, dst_subitemsize,
2180
                            count, dst_subitemsize, data_decdstref) < 0) {
2181
                        return -1;
2182
                    }
2183
                }
2184 0
                memset(dst_ptr, 0, count*dst_subitemsize);
2185
            }
2186 1
            loop_index += count;
2187
        }
2188

2189 1
        if (stransfer_decsrcref != NULL) {
2190 1
            if (stransfer_decsrcref(
2191
                    NULL, 0, src, src_subitemsize,
2192
                    src_subN, src_subitemsize, data_decsrcref) < 0) {
2193
                return -1;
2194
            }
2195
        }
2196

2197 1
        src += src_stride;
2198 1
        dst += dst_stride;
2199 1
        --N;
2200
    }
2201
    return 0;
2202
}
2203

2204

2205
static int
2206 1
get_subarray_broadcast_transfer_function(int aligned,
2207
                            npy_intp src_stride, npy_intp dst_stride,
2208
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
2209
                            npy_intp src_size, npy_intp dst_size,
2210
                            PyArray_Dims src_shape, PyArray_Dims dst_shape,
2211
                            int move_references,
2212
                            PyArray_StridedUnaryOp **out_stransfer,
2213
                            NpyAuxData **out_transferdata,
2214
                            int *out_needs_api)
2215
{
2216
    _subarray_broadcast_data *data;
2217
    npy_intp structsize, loop_index, run, run_size,
2218
             src_index, dst_index, i, ndim;
2219
    _subarray_broadcast_offsetrun *offsetruns;
2220

2221 1
    structsize = sizeof(_subarray_broadcast_data) +
2222 1
                        dst_size*sizeof(_subarray_broadcast_offsetrun);
2223

2224
    /* Allocate the data and populate it */
2225 1
    data = (_subarray_broadcast_data *)PyArray_malloc(structsize);
2226 1
    if (data == NULL) {
2227 0
        PyErr_NoMemory();
2228
        return NPY_FAIL;
2229
    }
2230

2231
    /*
2232
     * move_references is set to 0, handled in the wrapping transfer fn,
2233
     * src_stride and dst_stride are set to contiguous, as N will always
2234
     * be 1 when it's called.
2235
     */
2236 1
    if (PyArray_GetDTypeTransferFunction(aligned,
2237 1
                    src_dtype->elsize, dst_dtype->elsize,
2238
                    src_dtype, dst_dtype,
2239
                    0,
2240
                    &data->stransfer, &data->data,
2241
                    out_needs_api) != NPY_SUCCEED) {
2242 0
        PyArray_free(data);
2243
        return NPY_FAIL;
2244
    }
2245 1
    data->base.free = &_subarray_broadcast_data_free;
2246 1
    data->base.clone = &_subarray_broadcast_data_clone;
2247 1
    data->src_N = src_size;
2248 1
    data->dst_N = dst_size;
2249 1
    data->src_itemsize = src_dtype->elsize;
2250 1
    data->dst_itemsize = dst_dtype->elsize;
2251

2252
    /* If the src object will need a DECREF */
2253 1
    if (move_references && PyDataType_REFCHK(src_dtype)) {
2254 1
        if (PyArray_GetDTypeTransferFunction(aligned,
2255
                        src_dtype->elsize, 0,
2256
                        src_dtype, NULL,
2257
                        1,
2258
                        &data->stransfer_decsrcref,
2259
                        &data->data_decsrcref,
2260
                        out_needs_api) != NPY_SUCCEED) {
2261 0
            NPY_AUXDATA_FREE(data->data);
2262 0
            PyArray_free(data);
2263
            return NPY_FAIL;
2264
        }
2265
    }
2266
    else {
2267 1
        data->stransfer_decsrcref = NULL;
2268 1
        data->data_decsrcref = NULL;
2269
    }
2270

2271
    /* If the dst object needs a DECREF to set it to NULL */
2272 1
    if (PyDataType_REFCHK(dst_dtype)) {
2273 1
        if (PyArray_GetDTypeTransferFunction(aligned,
2274 1
                        dst_dtype->elsize, 0,
2275
                        dst_dtype, NULL,
2276
                        1,
2277
                        &data->stransfer_decdstref,
2278
                        &data->data_decdstref,
2279
                        out_needs_api) != NPY_SUCCEED) {
2280 0
            NPY_AUXDATA_FREE(data->data);
2281 0
            NPY_AUXDATA_FREE(data->data_decsrcref);
2282 0
            PyArray_free(data);
2283
            return NPY_FAIL;
2284
        }
2285
    }
2286
    else {
2287 1
        data->stransfer_decdstref = NULL;
2288 1
        data->data_decdstref = NULL;
2289
    }
2290

2291
    /* Calculate the broadcasting and set the offsets */
2292 1
    offsetruns = &data->offsetruns;
2293 1
    ndim = (src_shape.len > dst_shape.len) ? src_shape.len : dst_shape.len;
2294 1
    for (loop_index = 0; loop_index < dst_size; ++loop_index) {
2295 1
        npy_intp src_factor = 1;
2296

2297 1
        dst_index = loop_index;
2298 1
        src_index = 0;
2299 1
        for (i = ndim-1; i >= 0; --i) {
2300 1
            npy_intp coord = 0, shape;
2301

2302
            /* Get the dst coord of this index for dimension i */
2303 1
            if (i >= ndim - dst_shape.len) {
2304 1
                shape = dst_shape.ptr[i-(ndim-dst_shape.len)];
2305 1
                coord = dst_index % shape;
2306 1
                dst_index /= shape;
2307
            }
2308

2309
            /* Translate it into a src coord and update src_index */
2310 1
            if (i >= ndim - src_shape.len) {
2311 1
                shape = src_shape.ptr[i-(ndim-src_shape.len)];
2312 1
                if (shape == 1) {
2313
                    coord = 0;
2314
                }
2315
                else {
2316 1
                    if (coord < shape) {
2317 1
                        src_index += src_factor*coord;
2318 1
                        src_factor *= shape;
2319
                    }
2320
                    else {
2321
                        /* Out of bounds, flag with -1 */
2322
                        src_index = -1;
2323
                        break;
2324
                    }
2325
                }
2326
            }
2327
        }
2328
        /* Set the offset */
2329 1
        if (src_index == -1) {
2330 1
            offsetruns[loop_index].offset = -1;
2331
        }
2332
        else {
2333 1
            offsetruns[loop_index].offset = src_index;
2334
        }
2335
    }
2336

2337
    /* Run-length encode the result */
2338
    run = 0;
2339
    run_size = 1;
2340 1
    for (loop_index = 1; loop_index < dst_size; ++loop_index) {
2341 1
        if (offsetruns[run].offset == -1) {
2342
            /* Stop the run when there's a valid index again */
2343 1
            if (offsetruns[loop_index].offset != -1) {
2344 0
                offsetruns[run].count = run_size;
2345 0
                run++;
2346 0
                run_size = 1;
2347 0
                offsetruns[run].offset = offsetruns[loop_index].offset;
2348
            }
2349
            else {
2350 1
                run_size++;
2351
            }
2352
        }
2353
        else {
2354
            /* Stop the run when there's a valid index again */
2355 1
            if (offsetruns[loop_index].offset !=
2356 1
                            offsetruns[loop_index-1].offset + 1) {
2357 1
                offsetruns[run].count = run_size;
2358 1
                run++;
2359 1
                run_size = 1;
2360 1
                offsetruns[run].offset = offsetruns[loop_index].offset;
2361
            }
2362
            else {
2363 1
                run_size++;
2364
            }
2365
        }
2366
    }
2367 1
    offsetruns[run].count = run_size;
2368 1
    run++;
2369 1
    data->run_count = run;
2370

2371
    /* Multiply all the offsets by the src item size */
2372 1
    while (run--) {
2373 1
        if (offsetruns[run].offset != -1) {
2374 1
            offsetruns[run].offset *= src_dtype->elsize;
2375
        }
2376
    }
2377

2378 1
    if (data->stransfer_decsrcref == NULL &&
2379 1
                                data->stransfer_decdstref == NULL) {
2380 1
        *out_stransfer = &_strided_to_strided_subarray_broadcast;
2381
    }
2382
    else {
2383 1
        *out_stransfer = &_strided_to_strided_subarray_broadcast_withrefs;
2384
    }
2385 1
    *out_transferdata = (NpyAuxData *)data;
2386

2387
    return NPY_SUCCEED;
2388
}
2389

2390
/*
2391
 * Handles subarray transfer.  To call this, at least one of the dtype's
2392
 * subarrays must be non-NULL
2393
 */
2394
static int
2395 1
get_subarray_transfer_function(int aligned,
2396
                            npy_intp src_stride, npy_intp dst_stride,
2397
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
2398
                            int move_references,
2399
                            PyArray_StridedUnaryOp **out_stransfer,
2400
                            NpyAuxData **out_transferdata,
2401
                            int *out_needs_api)
2402
{
2403 1
    PyArray_Dims src_shape = {NULL, -1}, dst_shape = {NULL, -1};
2404 1
    npy_intp src_size = 1, dst_size = 1;
2405

2406
    /* Get the subarray shapes and sizes */
2407 1
    if (PyDataType_HASSUBARRAY(src_dtype)) {
2408 1
        if (!(PyArray_IntpConverter(src_dtype->subarray->shape,
2409
                                            &src_shape))) {
2410 0
            PyErr_SetString(PyExc_ValueError,
2411
                    "invalid subarray shape");
2412 0
            return NPY_FAIL;
2413
        }
2414 1
        src_size = PyArray_MultiplyList(src_shape.ptr, src_shape.len);
2415 1
        src_dtype = src_dtype->subarray->base;
2416
    }
2417 1
    if (PyDataType_HASSUBARRAY(dst_dtype)) {
2418 1
        if (!(PyArray_IntpConverter(dst_dtype->subarray->shape,
2419
                                            &dst_shape))) {
2420 0
            npy_free_cache_dim_obj(src_shape);
2421 0
            PyErr_SetString(PyExc_ValueError,
2422
                    "invalid subarray shape");
2423 0
            return NPY_FAIL;
2424
        }
2425 1
        dst_size = PyArray_MultiplyList(dst_shape.ptr, dst_shape.len);
2426 1
        dst_dtype = dst_dtype->subarray->base;
2427
    }
2428

2429
    /*
2430
     * Just a straight one-element copy.
2431
     */
2432 1
    if (dst_size == 1 && src_size == 1) {
2433 1
        npy_free_cache_dim_obj(src_shape);
2434 1
        npy_free_cache_dim_obj(dst_shape);
2435

2436 1
        return PyArray_GetDTypeTransferFunction(aligned,
2437
                src_stride, dst_stride,
2438
                src_dtype, dst_dtype,
2439
                move_references,
2440
                out_stransfer, out_transferdata,
2441
                out_needs_api);
2442
    }
2443
    /* Copy the src value to all the dst values */
2444 1
    else if (src_size == 1) {
2445 1
        npy_free_cache_dim_obj(src_shape);
2446 1
        npy_free_cache_dim_obj(dst_shape);
2447

2448 1
        return get_one_to_n_transfer_function(aligned,
2449
                        src_stride, dst_stride,
2450
                        src_dtype, dst_dtype,
2451
                        move_references,
2452
                        dst_size,
2453
                        out_stransfer, out_transferdata,
2454
                        out_needs_api);
2455
    }
2456
    /* If the shapes match exactly, do an n to n copy */
2457 1
    else if (src_shape.len == dst_shape.len &&
2458 1
               PyArray_CompareLists(src_shape.ptr, dst_shape.ptr,
2459
                                                    src_shape.len)) {
2460 1
        npy_free_cache_dim_obj(src_shape);
2461 1
        npy_free_cache_dim_obj(dst_shape);
2462

2463 1
        return get_n_to_n_transfer_function(aligned,
2464
                        src_stride, dst_stride,
2465
                        src_dtype, dst_dtype,
2466
                        move_references,
2467
                        src_size,
2468
                        out_stransfer, out_transferdata,
2469
                        out_needs_api);
2470
    }
2471
    /*
2472
     * Copy the subarray with broadcasting, truncating, and zero-padding
2473
     * as necessary.
2474
     */
2475
    else {
2476 1
        int ret = get_subarray_broadcast_transfer_function(aligned,
2477
                        src_stride, dst_stride,
2478
                        src_dtype, dst_dtype,
2479
                        src_size, dst_size,
2480
                        src_shape, dst_shape,
2481
                        move_references,
2482
                        out_stransfer, out_transferdata,
2483
                        out_needs_api);
2484

2485 1
        npy_free_cache_dim_obj(src_shape);
2486 1
        npy_free_cache_dim_obj(dst_shape);
2487 1
        return ret;
2488
    }
2489
}
2490

2491
/**************************** COPY FIELDS *******************************/
2492
typedef struct {
2493
    npy_intp src_offset, dst_offset, src_itemsize;
2494
    PyArray_StridedUnaryOp *stransfer;
2495
    NpyAuxData *data;
2496
} _single_field_transfer;
2497

2498
typedef struct {
2499
    NpyAuxData base;
2500
    npy_intp field_count;
2501

2502
    _single_field_transfer fields;
2503
} _field_transfer_data;
2504

2505
/* transfer data free function */
2506 1
static void _field_transfer_data_free(NpyAuxData *data)
2507
{
2508 1
    _field_transfer_data *d = (_field_transfer_data *)data;
2509
    npy_intp i, field_count;
2510
    _single_field_transfer *fields;
2511

2512 1
    field_count = d->field_count;
2513 1
    fields = &d->fields;
2514

2515 1
    for (i = 0; i < field_count; ++i) {
2516 1
        NPY_AUXDATA_FREE(fields[i].data);
2517
    }
2518 1
    PyArray_free(d);
2519
}
2520

2521
/* transfer data copy function */
2522 0
static NpyAuxData *_field_transfer_data_clone(NpyAuxData *data)
2523
{
2524 0
    _field_transfer_data *d = (_field_transfer_data *)data;
2525
    _field_transfer_data *newdata;
2526 0
    npy_intp i, field_count = d->field_count, structsize;
2527
    _single_field_transfer *fields, *newfields;
2528

2529 0
    structsize = sizeof(_field_transfer_data) +
2530 0
                    field_count * sizeof(_single_field_transfer);
2531

2532
    /* Allocate the data and populate it */
2533 0
    newdata = (_field_transfer_data *)PyArray_malloc(structsize);
2534 0
    if (newdata == NULL) {
2535
        return NULL;
2536
    }
2537 0
    memcpy(newdata, d, structsize);
2538
    /* Copy all the fields transfer data */
2539 0
    fields = &d->fields;
2540 0
    newfields = &newdata->fields;
2541 0
    for (i = 0; i < field_count; ++i) {
2542 0
        if (fields[i].data != NULL) {
2543 0
            newfields[i].data = NPY_AUXDATA_CLONE(fields[i].data);
2544 0
            if (newfields[i].data == NULL) {
2545 0
                for (i = i-1; i >= 0; --i) {
2546 0
                    NPY_AUXDATA_FREE(newfields[i].data);
2547
                }
2548 0
                PyArray_free(newdata);
2549 0
                return NULL;
2550
            }
2551
        }
2552

2553
    }
2554

2555
    return (NpyAuxData *)newdata;
2556
}
2557

2558
static int
2559 1
_strided_to_strided_field_transfer(char *dst, npy_intp dst_stride,
2560
                        char *src, npy_intp src_stride,
2561
                        npy_intp N, npy_intp NPY_UNUSED(src_itemsize),
2562
                        NpyAuxData *data)
2563
{
2564 1
    _field_transfer_data *d = (_field_transfer_data *)data;
2565 1
    npy_intp i, field_count = d->field_count;
2566
    _single_field_transfer *field;
2567

2568
    /* Do the transfer a block at a time */
2569
    for (;;) {
2570 1
        field = &d->fields;
2571 1
        if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) {
2572 1
            for (i = 0; i < field_count; ++i, ++field) {
2573 1
                if (field->stransfer(
2574 1
                        dst + field->dst_offset, dst_stride,
2575 1
                        src + field->src_offset, src_stride,
2576
                        NPY_LOWLEVEL_BUFFER_BLOCKSIZE,
2577
                        field->src_itemsize, field->data) < 0) {
2578
                    return -1;
2579
                }
2580
            }
2581 1
            N -= NPY_LOWLEVEL_BUFFER_BLOCKSIZE;
2582 1
            src += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_stride;
2583 1
            dst += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_stride;
2584
        }
2585
        else {
2586 1
            for (i = 0; i < field_count; ++i, ++field) {
2587 1
                if (field->stransfer(
2588 1
                        dst + field->dst_offset, dst_stride,
2589 1
                        src + field->src_offset, src_stride,
2590
                        N,
2591
                        field->src_itemsize, field->data) < 0) {
2592
                    return -1;
2593
                }
2594
            }
2595
            return 0;
2596
        }
2597
    }
2598
}
2599

2600
/*
2601
 * Handles fields transfer.  To call this, at least one of the dtypes
2602
 * must have fields. Does not take care of object<->structure conversion
2603
 */
2604
static int
2605 1
get_fields_transfer_function(int aligned,
2606
                            npy_intp src_stride, npy_intp dst_stride,
2607
                            PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype,
2608
                            int move_references,
2609
                            PyArray_StridedUnaryOp **out_stransfer,
2610
                            NpyAuxData **out_transferdata,
2611
                            int *out_needs_api)
2612
{
2613
    PyObject *key, *tup, *title;
2614
    PyArray_Descr *src_fld_dtype, *dst_fld_dtype;
2615
    npy_int i, field_count, structsize;
2616
    int src_offset, dst_offset;
2617
    _field_transfer_data *data;
2618
    _single_field_transfer *fields;
2619 1
    int failed = 0;
2620

2621
    /*
2622
     * There are three cases to take care of: 1. src is non-structured,
2623
     * 2. dst is non-structured, or 3. both are structured.
2624
     */
2625

2626
    /* 1. src is non-structured. Copy the src value to all the fields of dst */
2627 1
    if (!PyDataType_HASFIELDS(src_dtype)) {
2628 1
        field_count = PyTuple_GET_SIZE(dst_dtype->names);
2629

2630
        /* Allocate the field-data structure and populate it */
2631 1
        structsize = sizeof(_field_transfer_data) +
2632 1
                        (field_count + 1) * sizeof(_single_field_transfer);
2633 1
        data = (_field_transfer_data *)PyArray_malloc(structsize);
2634 1
        if (data == NULL) {
2635 0
            PyErr_NoMemory();
2636
            return NPY_FAIL;
2637
        }
2638 1
        data->base.free = &_field_transfer_data_free;
2639 1
        data->base.clone = &_field_transfer_data_clone;
2640 1
        fields = &data->fields;
2641

2642 1
        for (i = 0; i < field_count; ++i) {
2643 1
            key = PyTuple_GET_ITEM(dst_dtype->names, i);
2644 1
            tup = PyDict_GetItem(dst_dtype->fields, key);
2645 1
            if (!PyArg_ParseTuple(tup, "Oi|O", &dst_fld_dtype,
2646
                                                    &dst_offset, &title)) {
2647 0
                PyArray_free(data);
2648
                return NPY_FAIL;
2649
            }
2650 1
            if (PyArray_GetDTypeTransferFunction(0,
2651
                                    src_stride, dst_stride,
2652
                                    src_dtype, dst_fld_dtype,
2653
                                    0,
2654
                                    &fields[i].stransfer,
2655 1
                                    &fields[i].data,
2656
                                    out_needs_api) != NPY_SUCCEED) {
2657 0
                for (i = i-1; i >= 0; --i) {
2658 0
                    NPY_AUXDATA_FREE(fields[i].data);
2659
                }
2660 0
                PyArray_free(data);
2661
                return NPY_FAIL;
2662
            }
2663 1
            fields[i].src_offset = 0;
2664 1
            fields[i].dst_offset = dst_offset;
2665 1
            fields[i].src_itemsize = src_dtype->elsize;
2666
        }
2667

2668
        /*
2669
         * If references should be decrefd in src, add
2670
         * another transfer function to do that.
2671
         */
2672 1
        if (move_references && PyDataType_REFCHK(src_dtype)) {
2673 0
            if (get_decsrcref_transfer_function(0,
2674
                                    src_stride,
2675
                                    src_dtype,
2676
                                    &fields[field_count].stransfer,
2677 0
                                    &fields[field_count].data,
2678
                                    out_needs_api) != NPY_SUCCEED) {
2679 0
                for (i = 0; i < field_count; ++i) {
2680 0
                    NPY_AUXDATA_FREE(fields[i].data);
2681
                }
2682 0
                PyArray_free(data);
2683
                return NPY_FAIL;
2684
            }
2685 0
            fields[field_count].src_offset = 0;
2686 0
            fields[field_count].dst_offset = 0;
2687 0
            fields[field_count].src_itemsize = src_dtype->elsize;
2688 0
            field_count++;
2689
        }
2690 1
        data->field_count = field_count;
2691

2692 1
        *out_stransfer = &_strided_to_strided_field_transfer;
2693 1
        *out_transferdata = (NpyAuxData *)data;
2694

2695
        return NPY_SUCCEED;
2696
    }
2697

2698
    /* 2. dst is non-structured. Allow transfer from single-field src to dst */
2699 1
    if (!PyDataType_HASFIELDS(dst_dtype)) {
2700 1
        if (PyTuple_GET_SIZE(src_dtype->names) != 1) {
2701 1
            PyErr_SetString(PyExc_ValueError,
2702
                    "Can't cast from structure to non-structure, except if the "
2703
                    "structure only has a single field.");
2704
            return NPY_FAIL;
2705
        }
2706

2707
        /* Allocate the field-data structure and populate it */
2708 1
        structsize = sizeof(_field_transfer_data) +
2709
                        1 * sizeof(_single_field_transfer);
2710 1
        data = (_field_transfer_data *)PyArray_malloc(structsize);
2711 1
        if (data == NULL) {
2712 0
            PyErr_NoMemory();
2713
            return NPY_FAIL;
2714
        }
2715 1
        data->base.free = &_field_transfer_data_free;
2716 1
        data->base.clone = &_field_transfer_data_clone;
2717 1
        fields = &data->fields;
2718

2719 1
        key = PyTuple_GET_ITEM(src_dtype->names, 0);
2720 1
        tup = PyDict_GetItem(src_dtype->fields, key);
2721 1
        if (!PyArg_ParseTuple(tup, "Oi|O",
2722
                              &src_fld_dtype, &src_offset, &title)) {
2723
            return NPY_FAIL;
2724
        }
2725

2726 1
        if (PyArray_GetDTypeTransferFunction(0,
2727
                                             src_stride, dst_stride,
2728
                                             src_fld_dtype, dst_dtype,
2729
                                             move_references,
2730
                                             &fields[0].stransfer,
2731
                                             &fields[0].data,
2732
                                             out_needs_api) != NPY_SUCCEED) {
2733 0
            PyArray_free(data);
2734
            return NPY_FAIL;
2735
        }
2736 1
        fields[0].src_offset = src_offset;
2737 1
        fields[0].dst_offset = 0;
2738 1
        fields[0].src_itemsize = src_fld_dtype->elsize;
2739

2740 1
        data->field_count = 1;
2741

2742 1
        *out_stransfer = &_strided_to_strided_field_transfer;
2743 1
        *out_transferdata = (NpyAuxData *)data;
2744

2745
        return NPY_SUCCEED;
2746
    }
2747

2748
    /* 3. Otherwise both src and dst are structured arrays */
2749 1
    field_count = PyTuple_GET_SIZE(dst_dtype->names);
2750

2751
    /* Match up the fields to copy (field-by-field transfer) */
2752 1
    if (PyTuple_GET_SIZE(src_dtype->names) != field_count) {
2753 1
        PyErr_SetString(PyExc_ValueError, "structures must have the same size");
2754
        return NPY_FAIL;
2755
    }
2756

2757
    /* Allocate the field-data structure and populate it */
2758 1
    structsize = sizeof(_field_transfer_data) +
2759
                    field_count * sizeof(_single_field_transfer);
2760 1
    data = (_field_transfer_data *)PyArray_malloc(structsize);
2761 1
    if (data == NULL) {
2762 0
        PyErr_NoMemory();
2763
        return NPY_FAIL;
2764
    }
2765 1
    data->base.free = &_field_transfer_data_free;
2766 1
    data->base.clone = &_field_transfer_data_clone;
2767 1
    fields = &data->fields;
2768

2769
    /* set up the transfer function for each field */
2770 1
    for (i = 0; i < field_count; ++i) {
2771 1
        key = PyTuple_GET_ITEM(dst_dtype->names, i);
2772 1
        tup = PyDict_GetItem(dst_dtype->fields, key);
2773 1
        if (!PyArg_ParseTuple(tup, "Oi|O", &dst_fld_dtype,
2774
                                                &dst_offset, &title)) {
2775
            failed = 1;
2776
            break;
2777
        }
2778 1
        key = PyTuple_GET_ITEM(src_dtype->names, i);
2779 1
        tup = PyDict_GetItem(src_dtype->fields, key);
2780 1
        if (!PyArg_ParseTuple(tup, "Oi|O", &src_fld_dtype,
2781
                                                &src_offset, &title)) {
2782
            failed = 1;
2783
            break;
2784
        }
2785

2786 1
        if (PyArray_GetDTypeTransferFunction(0,
2787
                                             src_stride, dst_stride,
2788
                                             src_fld_dtype, dst_fld_dtype,
2789
                                             move_references,
2790
                                             &fields[i].stransfer,
2791 1
                                             &fields[i].data,
2792
                                             out_needs_api) != NPY_SUCCEED) {
2793
            failed = 1;
2794
            break;
2795
        }
2796 1
        fields[i].src_offset = src_offset;
2797 1
        fields[i].dst_offset = dst_offset;
2798 1
        fields[i].src_itemsize = src_fld_dtype->elsize;
2799
    }
2800

2801 1
    if (failed) {
2802 0
        for (i = i-1; i >= 0; --i) {
2803 0
            NPY_AUXDATA_FREE(fields[i].data);
2804
        }
2805 0
        PyArray_free(data);
2806
        return NPY_FAIL;
2807
    }
2808

2809 1
    data->field_count = field_count;
2810

2811 1
    *out_stransfer = &_strided_to_strided_field_transfer;
2812 1
    *out_transferdata = (NpyAuxData *)data;
2813

2814
    return NPY_SUCCEED;
2815
}
2816

2817
static int
2818 1
get_decsrcref_fields_transfer_function(int aligned,
2819
                            npy_intp src_stride,
2820
                            PyArray_Descr *src_dtype,
2821
                            PyArray_StridedUnaryOp **out_stransfer,
2822
                            NpyAuxData **out_transferdata,
2823
                            int *out_needs_api)
2824
{
2825
    PyObject *names, *key, *tup, *title;
2826
    PyArray_Descr *src_fld_dtype;
2827
    npy_int i, names_size, field_count, structsize;
2828
    int src_offset;
2829
    _field_transfer_data *data;
2830
    _single_field_transfer *fields;
2831

2832 1
    names = src_dtype->names;
2833 1
    names_size = PyTuple_GET_SIZE(src_dtype->names);
2834

2835 1
    field_count = names_size;
2836 1
    structsize = sizeof(_field_transfer_data) +
2837
                    field_count * sizeof(_single_field_transfer);
2838
    /* Allocate the data and populate it */
2839 1
    data = (_field_transfer_data *)PyArray_malloc(structsize);
2840 1
    if (data == NULL) {
2841 0
        PyErr_NoMemory();
2842
        return NPY_FAIL;
2843
    }
2844 1
    data->base.free = &_field_transfer_data_free;
2845 1
    data->base.clone = &_field_transfer_data_clone;
2846 1
    fields = &data->fields;
2847

2848 1
    field_count = 0;
2849 1
    for (i = 0; i < names_size; ++i) {
2850 1
        key = PyTuple_GET_ITEM(names, i);
2851 1
        tup = PyDict_GetItem(src_dtype->fields, key);
2852 1
        if (!PyArg_ParseTuple(tup, "Oi|O", &src_fld_dtype,
2853
                                                &src_offset, &title)) {
2854 0
            PyArray_free(data);
2855
            return NPY_FAIL;
2856
        }
2857 1
        if (PyDataType_REFCHK(src_fld_dtype)) {
2858 1
            if (out_needs_api) {
2859 1
                *out_needs_api = 1;
2860
            }
2861 1
            if (get_decsrcref_transfer_function(0,
2862
                                    src_stride,
2863
                                    src_fld_dtype,
2864
                                    &fields[field_count].stransfer,
2865 1
                                    &fields[field_count].data,
2866
                                    out_needs_api) != NPY_SUCCEED) {
2867 0
                for (i = field_count-1; i >= 0; --i) {
2868 0
                    NPY_AUXDATA_FREE(fields[i].data);
2869
                }
2870 0
                PyArray_free(data);
2871
                return NPY_FAIL;
2872
            }
2873 1
            fields[field_count].src_offset = src_offset;
2874 1
            fields[field_count].dst_offset = 0;
2875 1
            fields[field_count].src_itemsize = src_dtype->elsize;
2876 1
            field_count++;
2877
        }
2878
    }
2879

2880 1
    data->field_count = field_count;
2881

2882 1
    *out_stransfer = &_strided_to_strided_field_transfer;
2883 1
    *out_transferdata = (NpyAuxData *)data;
2884

2885
    return NPY_SUCCEED;
2886
}
2887

2888
static int
2889 0
get_setdestzero_fields_transfer_function(int aligned,
2890
                            npy_intp dst_stride,
2891
                            PyArray_Descr *dst_dtype,
2892
                            PyArray_StridedUnaryOp **out_stransfer,
2893
                            NpyAuxData **out_transferdata,
2894
                            int *out_needs_api)
2895
{
2896
    PyObject *names, *key, *tup, *title;
2897
    PyArray_Descr *dst_fld_dtype;
2898
    npy_int i, names_size, field_count, structsize;
2899
    int dst_offset;
2900
    _field_transfer_data *data;
2901
    _single_field_transfer *fields;
2902

2903 0
    names = dst_dtype->names;
2904 0
    names_size = PyTuple_GET_SIZE(dst_dtype->names);
2905

2906 0
    field_count = names_size;
2907 0
    structsize = sizeof(_field_transfer_data) +
2908
                    field_count * sizeof(_single_field_transfer);
2909
    /* Allocate the data and populate it */
2910 0
    data = (_field_transfer_data *)PyArray_malloc(structsize);
2911 0
    if (data == NULL) {
2912 0
        PyErr_NoMemory();
2913
        return NPY_FAIL;
2914
    }
2915 0
    data->base.free = &_field_transfer_data_free;
2916 0
    data->base.clone = &_field_transfer_data_clone;
2917 0
    fields = &data->fields;
2918

2919 0
    for (i = 0; i < names_size; ++i) {
2920 0
        key = PyTuple_GET_ITEM(names, i);
2921 0
        tup = PyDict_GetItem(dst_dtype->fields, key);
2922 0
        if (!PyArg_ParseTuple(tup, "Oi|O", &dst_fld_dtype,
2923
                                                &dst_offset, &title)) {
2924 0
            PyArray_free(data);
2925
            return NPY_FAIL;
2926
        }
2927 0
        if (get_setdstzero_transfer_function(0,
2928
                                dst_stride,
2929
                                dst_fld_dtype,
2930
                                &fields[i].stransfer,
2931 0
                                &fields[i].data,
2932
                                out_needs_api) != NPY_SUCCEED) {
2933 0
            for (i = i-1; i >= 0; --i) {
2934 0
                NPY_AUXDATA_FREE(fields[i].data);
2935
            }
2936 0
            PyArray_free(data);
2937
            return NPY_FAIL;
2938
        }
2939 0
        fields[i].src_offset = 0;
2940 0
        fields[i].dst_offset = dst_offset;
2941 0
        fields[i].src_itemsize = 0;
2942
    }
2943

2944 0
    data->field_count = field_count;
2945

2946 0
    *out_stransfer = &_strided_to_strided_field_transfer;
2947 0
    *out_transferdata = (NpyAuxData *)data;
2948

2949
    return NPY_SUCCEED;
2950
}
2951

2952
/************************* MASKED TRANSFER WRAPPER *************************/
2953

2954
typedef struct {
2955
    NpyAuxData base;
2956
    /* The transfer function being wrapped */
2957
    PyArray_StridedUnaryOp *stransfer;
2958
    NpyAuxData *transferdata;
2959

2960
    /* The src decref function if necessary */
2961
    PyArray_StridedUnaryOp *decsrcref_stransfer;
2962
    NpyAuxData *decsrcref_transferdata;
2963
} _masked_wrapper_transfer_data;
2964

2965
/* transfer data free function */
2966 1
static void _masked_wrapper_transfer_data_free(NpyAuxData *data)
2967
{
2968 1
    _masked_wrapper_transfer_data *d = (_masked_wrapper_transfer_data *)data;
2969 1
    NPY_AUXDATA_FREE(d->transferdata);
2970 1
    NPY_AUXDATA_FREE(d->decsrcref_transferdata);
2971 1
    PyArray_free(data);
2972
}
2973

2974
/* transfer data copy function */
2975 0
static NpyAuxData *_masked_wrapper_transfer_data_clone(NpyAuxData *data)
2976
{
2977 0
    _masked_wrapper_transfer_data *d = (_masked_wrapper_transfer_data *)data;
2978
    _masked_wrapper_transfer_data *newdata;
2979

2980
    /* Allocate the data and populate it */
2981 0
    newdata = (_masked_wrapper_transfer_data *)PyArray_malloc(
2982
                                    sizeof(_masked_wrapper_transfer_data));
2983 0
    if (newdata == NULL) {
2984
        return NULL;
2985
    }
2986 0
    memcpy(newdata, d, sizeof(_masked_wrapper_transfer_data));
2987

2988
    /* Clone all the owned auxdata as well */
2989 0
    if (newdata->transferdata != NULL) {
2990 0
        newdata->transferdata = NPY_AUXDATA_CLONE(newdata->transferdata);
2991 0
        if (newdata->transferdata == NULL) {
2992 0
            PyArray_free(newdata);
2993 0
            return NULL;
2994
        }
2995
    }
2996 0
    if (newdata->decsrcref_transferdata != NULL) {
2997 0
        newdata->decsrcref_transferdata =
2998 0
                        NPY_AUXDATA_CLONE(newdata->decsrcref_transferdata);
2999 0
        if (newdata->decsrcref_transferdata == NULL) {
3000 0
            NPY_AUXDATA_FREE(newdata->transferdata);
3001 0
            PyArray_free(newdata);
3002 0
            return NULL;
3003
        }
3004
    }
3005

3006
    return (NpyAuxData *)newdata;
3007
}
3008

3009
static int
3010 0
_strided_masked_wrapper_decsrcref_transfer_function(
3011
                                    char *dst, npy_intp dst_stride,
3012
                                    char *src, npy_intp src_stride,
3013
                                    npy_bool *mask, npy_intp mask_stride,
3014
                                    npy_intp N, npy_intp src_itemsize,
3015
                                    NpyAuxData *transferdata)
3016
{
3017 0
    _masked_wrapper_transfer_data *d =
3018
                        (_masked_wrapper_transfer_data *)transferdata;
3019
    npy_intp subloopsize;
3020
    PyArray_StridedUnaryOp *unmasked_stransfer, *decsrcref_stransfer;
3021
    NpyAuxData *unmasked_transferdata, *decsrcref_transferdata;
3022

3023 0
    unmasked_stransfer = d->stransfer;
3024 0
    unmasked_transferdata = d->transferdata;
3025 0
    decsrcref_stransfer = d->decsrcref_stransfer;
3026 0
    decsrcref_transferdata = d->decsrcref_transferdata;
3027

3028 0
    while (N > 0) {
3029
        /* Skip masked values, still calling decsrcref for move_references */
3030 0
        mask = (npy_bool*)npy_memchr((char *)mask, 0, mask_stride, N,
3031
                                     &subloopsize, 1);
3032 0
        if (decsrcref_stransfer(
3033
                NULL, 0, src, src_stride,
3034
                subloopsize, src_itemsize, decsrcref_transferdata) < 0) {
3035