1
/* Array Descr Object */
2

3
#define PY_SSIZE_T_CLEAN
4
#include <Python.h>
5
#include "structmember.h"
6
#include "assert.h"
7

8
#define NPY_NO_DEPRECATED_API NPY_API_VERSION
9
#define _MULTIARRAYMODULE
10
#include <numpy/ndarraytypes.h>
11
#include <numpy/arrayscalars.h>
12
#include "npy_pycompat.h"
13

14
#include "common.h"
15
#include "dtypemeta.h"
16
#include "_datetime.h"
17
#include "array_coercion.h"
18

19

20
static void
21 0
dtypemeta_dealloc(PyArray_DTypeMeta *self) {
22
    /* Do not accidentally delete a statically defined DType: */
23
    assert(((PyTypeObject *)self)->tp_flags & Py_TPFLAGS_HEAPTYPE);
24

25 0
    Py_XDECREF(self->scalar_type);
26 0
    Py_XDECREF(self->singleton);
27 0
    PyType_Type.tp_dealloc((PyObject *) self);
28
}
29

30
static PyObject *
31 0
dtypemeta_new(PyTypeObject *NPY_UNUSED(type),
32
        PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds))
33
{
34 0
    PyErr_SetString(PyExc_TypeError,
35
            "Preliminary-API: Cannot subclass DType.");
36 0
    return NULL;
37
}
38

39
static int
40 0
dtypemeta_init(PyTypeObject *NPY_UNUSED(type),
41
        PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds))
42
{
43 0
    PyErr_SetString(PyExc_TypeError,
44
            "Preliminary-API: Cannot __init__ DType class.");
45 0
    return -1;
46
}
47

48
/**
49
 * tp_is_gc slot of Python types. This is implemented only for documentation
50
 * purposes to indicate and document the subtleties involved.
51
 *
52
 * Python Type objects are either statically created (typical C-Extension type)
53
 * or HeapTypes (typically created in Python).
54
 * HeapTypes have the Py_TPFLAGS_HEAPTYPE flag and are garbage collected.
55
 * Our DTypeMeta instances (`np.dtype` and its subclasses) *may* be HeapTypes
56
 * if the Py_TPFLAGS_HEAPTYPE flag is set (they are created from Python).
57
 * They are not for legacy DTypes or np.dtype itself.
58
 *
59
 * @param self
60
 * @return nonzero if the object is garbage collected
61
 */
62
static NPY_INLINE int
63 1
dtypemeta_is_gc(PyObject *dtype_class)
64
{
65 1
    return PyType_Type.tp_is_gc(dtype_class);
66
}
67

68

69
static int
70 0
dtypemeta_traverse(PyArray_DTypeMeta *type, visitproc visit, void *arg)
71
{
72
    /*
73
     * We have to traverse the base class (if it is a HeapType).
74
     * PyType_Type will handle this logic for us.
75
     * This function is currently not used, but will probably be necessary
76
     * in the future when we implement HeapTypes (python/dynamically
77
     * defined types). It should be revised at that time.
78
     */
79
    assert(0);
80
    assert(!type->legacy && (PyTypeObject *)type != &PyArrayDescr_Type);
81 0
    Py_VISIT(type->singleton);
82 0
    Py_VISIT(type->scalar_type);
83 0
    return PyType_Type.tp_traverse((PyObject *)type, visit, arg);
84
}
85

86

87
static PyObject *
88 1
legacy_dtype_default_new(PyArray_DTypeMeta *self,
89
        PyObject *args, PyObject *kwargs)
90
{
91
    /* TODO: This should allow endianess and possibly metadata */
92 1
    if (self->parametric) {
93
        /* reject parametric ones since we would need to get unit, etc. info */
94 1
        PyErr_Format(PyExc_TypeError,
95
                "Preliminary-API: Flexible/Parametric legacy DType '%S' can "
96
                "only be instantiated using `np.dtype(...)`", self);
97 1
        return NULL;
98
    }
99

100 1
    if (PyTuple_GET_SIZE(args) != 0 ||
101 0
                (kwargs != NULL && PyDict_Size(kwargs))) {
102 0
        PyErr_Format(PyExc_TypeError,
103
                "currently only the no-argument instantiation is supported; "
104
                "use `np.dtype` instead.");
105 0
        return NULL;
106
    }
107 1
    Py_INCREF(self->singleton);
108 1
    return (PyObject *)self->singleton;
109
}
110

111

112
static PyArray_Descr *
113 1
nonparametric_discover_descr_from_pyobject(
114
        PyArray_DTypeMeta *cls, PyObject *obj)
115
{
116
    /* If the object is of the correct scalar type return our singleton */
117
    assert(!cls->parametric);
118 1
    Py_INCREF(cls->singleton);
119 1
    return cls->singleton;
120
}
121

122

123
static PyArray_Descr *
124 1
string_discover_descr_from_pyobject(
125
        PyArray_DTypeMeta *cls, PyObject *obj)
126
{
127 1
    npy_intp itemsize = -1;
128 1
    if (PyBytes_Check(obj)) {
129 1
        itemsize = PyBytes_Size(obj);
130
    }
131 1
    else if (PyUnicode_Check(obj)) {
132 1
        itemsize = PyUnicode_GetLength(obj);
133
    }
134 1
    if (itemsize != -1) {
135 1
        if (cls->type_num == NPY_UNICODE) {
136 1
            itemsize *= 4;
137
        }
138 1
        if (itemsize > NPY_MAX_INT) {
139 0
            PyErr_SetString(PyExc_TypeError,
140
                    "string to large to store inside array.");
141
        }
142 1
        PyArray_Descr *res = PyArray_DescrNewFromType(cls->type_num);
143 1
        res->elsize = (int)itemsize;
144 1
        return res;
145
    }
146 1
    return PyArray_DTypeFromObjectStringDiscovery(obj, NULL, cls->type_num);
147
}
148

149

150
static PyArray_Descr *
151 1
void_discover_descr_from_pyobject(
152
        PyArray_DTypeMeta *NPY_UNUSED(cls), PyObject *obj)
153
{
154 1
    if (PyArray_IsScalar(obj, Void)) {
155 1
        PyVoidScalarObject *void_obj = (PyVoidScalarObject *)obj;
156 1
        Py_INCREF(void_obj->descr);
157 1
        return void_obj->descr;
158
    }
159 1
    if (PyBytes_Check(obj)) {
160 1
        PyArray_Descr *descr = PyArray_DescrNewFromType(NPY_VOID);
161 1
        Py_ssize_t itemsize = (int)PyBytes_Size(obj);
162
        if (itemsize > NPY_MAX_INT) {
163
            PyErr_SetString(PyExc_TypeError,
164
                    "byte-like to large to store inside array.");
165
        }
166 1
        descr->elsize = itemsize;
167 1
        return descr;
168
    }
169 1
    PyErr_Format(PyExc_TypeError,
170
            "A bytes-like object is required, not '%s'", Py_TYPE(obj)->tp_name);
171 1
    return NULL;
172
}
173

174

175
static PyArray_Descr *
176 1
discover_datetime_and_timedelta_from_pyobject(
177
        PyArray_DTypeMeta *cls, PyObject *obj) {
178 1
    if (PyArray_IsScalar(obj, Datetime) ||
179 1
            PyArray_IsScalar(obj, Timedelta)) {
180
        PyArray_DatetimeMetaData *meta;
181 1
        PyArray_Descr *descr = PyArray_DescrFromScalar(obj);
182 1
        meta = get_datetime_metadata_from_dtype(descr);
183 1
        if (meta == NULL) {
184
            return NULL;
185
        }
186 1
        PyArray_Descr *new_descr = create_datetime_dtype(cls->type_num, meta);
187 1
        Py_DECREF(descr);
188
        return new_descr;
189
    }
190
    else {
191 1
        return find_object_datetime_type(obj, cls->type_num);
192
    }
193
}
194

195

196
static PyArray_Descr *
197 1
flexible_default_descr(PyArray_DTypeMeta *cls)
198
{
199 1
    PyArray_Descr *res = PyArray_DescrNewFromType(cls->type_num);
200 1
    if (res == NULL) {
201
        return NULL;
202
    }
203 1
    res->elsize = 1;
204 1
    if (cls->type_num == NPY_UNICODE) {
205 0
        res->elsize *= 4;
206
    }
207
    return res;
208
}
209

210

211
static int
212 1
python_builtins_are_known_scalar_types(
213
        PyArray_DTypeMeta *NPY_UNUSED(cls), PyTypeObject *pytype)
214
{
215
    /*
216
     * Always accept the common Python types, this ensures that we do not
217
     * convert pyfloat->float64->integers. Subclasses are hopefully rejected
218
     * as being discovered.
219
     * This is necessary only for python scalar classes which we discover
220
     * as valid DTypes.
221
     */
222 1
    if (pytype == &PyFloat_Type) {
223
        return 1;
224
    }
225 1
    if (pytype == &PyLong_Type) {
226
        return 1;
227
    }
228 1
    if (pytype == &PyBool_Type) {
229
        return 1;
230
    }
231 1
    if (pytype == &PyComplex_Type) {
232
        return 1;
233
    }
234 1
    if (pytype == &PyUnicode_Type) {
235
        return 1;
236
    }
237 1
    if (pytype == &PyBytes_Type) {
238
        return 1;
239
    }
240 1
    return 0;
241
}
242

243

244
static int
245 1
datetime_known_scalar_types(
246
        PyArray_DTypeMeta *cls, PyTypeObject *pytype)
247
{
248 1
    if (python_builtins_are_known_scalar_types(cls, pytype)) {
249
        return 1;
250
    }
251
    /*
252
     * To be able to identify the descriptor from e.g. any string, datetime
253
     * must take charge. Otherwise we would attempt casting which does not
254
     * truly support this. Only object arrays are special cased in this way.
255
     */
256 1
    return (PyType_IsSubtype(pytype, &PyBytes_Type) ||
257 1
            PyType_IsSubtype(pytype, &PyUnicode_Type));
258
}
259

260

261
static int
262 1
string_known_scalar_types(
263
        PyArray_DTypeMeta *cls, PyTypeObject *pytype) {
264 1
    if (python_builtins_are_known_scalar_types(cls, pytype)) {
265
        return 1;
266
    }
267 1
    if (PyType_IsSubtype(pytype, &PyDatetimeArrType_Type)) {
268
        /*
269
         * TODO: This should likely be deprecated or otherwise resolved.
270
         *       Deprecation has to occur in `String->setitem` unfortunately.
271
         *
272
         * Datetime currently do not cast to shorter strings, but string
273
         * coercion for arbitrary values uses `str(obj)[:len]` so it works.
274
         * This means `np.array(np.datetime64("2020-01-01"), "U9")`
275
         * and `np.array(np.datetime64("2020-01-01")).astype("U9")` behave
276
         * differently.
277
         */
278
        return 1;
279
    }
280 1
    return 0;
281
}
282

283

284
/**
285
 * This function takes a PyArray_Descr and replaces its base class with
286
 * a newly created dtype subclass (DTypeMeta instances).
287
 * There are some subtleties that need to be remembered when doing this,
288
 * first for the class objects itself it could be either a HeapType or not.
289
 * Since we are defining the DType from C, we will not make it a HeapType,
290
 * thus making it identical to a typical *static* type (except that we
291
 * malloc it). We could do it the other way, but there seems no reason to
292
 * do so.
293
 *
294
 * The DType instances (the actual dtypes or descriptors), are based on
295
 * prototypes which are passed in. These should not be garbage collected
296
 * and thus Py_TPFLAGS_HAVE_GC is not set. (We could allow this, but than
297
 * would have to allocate a new object, since the GC needs information before
298
 * the actual struct).
299
 *
300
 * The above is the reason why we should works exactly like we would for a
301
 * static type here.
302
 * Otherwise, we blurry the lines between C-defined extension classes
303
 * and Python subclasses. e.g. `class MyInt(int): pass` is very different
304
 * from our `class Float64(np.dtype): pass`, because the latter should not
305
 * be a HeapType and its instances should be exact PyArray_Descr structs.
306
 *
307
 * @param descr The descriptor that should be wrapped.
308
 * @param name The name for the DType, if NULL the type character is used.
309
 *
310
 * @returns 0 on success, -1 on failure.
311
 */
312
NPY_NO_EXPORT int
313 1
dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr)
314
{
315 1
    if (Py_TYPE(descr) != &PyArrayDescr_Type) {
316 0
        PyErr_Format(PyExc_RuntimeError,
317
                "During creation/wrapping of legacy DType, the original class "
318
                "was not PyArrayDescr_Type (it is replaced in this step).");
319 0
        return -1;
320
    }
321

322
    /*
323
     * Note: we have no intention of freeing the memory again since this
324
     * behaves identically to static type definition (see comment above).
325
     * This is seems cleaner for the legacy API, in the new API both static
326
     * and heap types are possible (some difficulty arises from the fact that
327
     * these are instances of DTypeMeta and not type).
328
     * In particular our own DTypes can be true static declarations.
329
     * However, this function remains necessary for legacy user dtypes.
330
     */
331

332 1
    const char *scalar_name = descr->typeobj->tp_name;
333
    /*
334
     * We have to take only the name, and ignore the module to get
335
     * a reasonable __name__, since static types are limited in this regard
336
     * (this is not ideal, but not a big issue in practice).
337
     * This is what Python does to print __name__ for static types.
338
     */
339 1
    const char *dot = strrchr(scalar_name, '.');
340 1
    if (dot) {
341 1
        scalar_name = dot + 1;
342
    }
343 1
    ssize_t name_length = strlen(scalar_name) + 14;
344

345 1
    char *tp_name = malloc(name_length);
346 1
    if (tp_name == NULL) {
347 0
        PyErr_NoMemory();
348 0
        return -1;
349
    }
350

351 1
    snprintf(tp_name, name_length, "numpy.dtype[%s]", scalar_name);
352

353 1
    PyArray_DTypeMeta *dtype_class = malloc(sizeof(PyArray_DTypeMeta));
354 1
    if (dtype_class == NULL) {
355 0
        PyDataMem_FREE(tp_name);
356 0
        return -1;
357
    }
358
    /*
359
     * Initialize the struct fields identically to static code by copying
360
     * a prototype instances for everything except our own fields which
361
     * vary between the DTypes.
362
     * In particular any Object initialization must be strictly copied from
363
     * the untouched prototype to avoid complexities (e.g. with PyPy).
364
     * Any Type slots need to be fixed before PyType_Ready, although most
365
     * will be inherited automatically there.
366
     */
367
    static PyArray_DTypeMeta prototype = {
368
        {{
369
            PyVarObject_HEAD_INIT(&PyArrayDTypeMeta_Type, 0)
370
            .tp_name = NULL,  /* set below */
371
            .tp_basicsize = sizeof(PyArray_Descr),
372
            .tp_flags = Py_TPFLAGS_DEFAULT,
373
            .tp_base = &PyArrayDescr_Type,
374
            .tp_new = (newfunc)legacy_dtype_default_new,
375
        },},
376
        .legacy = 1,
377
        .abstract = 0, /* this is a concrete DType */
378
        /* Further fields are not common between DTypes */
379
    };
380 1
    memcpy(dtype_class, &prototype, sizeof(PyArray_DTypeMeta));
381
    /* Fix name of the Type*/
382 1
    ((PyTypeObject *)dtype_class)->tp_name = tp_name;
383

384
    /* Let python finish the initialization (probably unnecessary) */
385 1
    if (PyType_Ready((PyTypeObject *)dtype_class) < 0) {
386
        return -1;
387
    }
388

389
    /*
390
     * Fill DTypeMeta information that varies between DTypes, any variable
391
     * type information would need to be set before PyType_Ready().
392
     */
393 1
    dtype_class->singleton = descr;
394 1
    Py_INCREF(descr->typeobj);
395 1
    dtype_class->scalar_type = descr->typeobj;
396 1
    dtype_class->type_num = descr->type_num;
397 1
    dtype_class->type = descr->type;
398 1
    dtype_class->f = descr->f;
399 1
    dtype_class->kind = descr->kind;
400

401
    /* Strings and voids have (strange) logic around scalars. */
402 1
    dtype_class->is_known_scalar_type = python_builtins_are_known_scalar_types;
403

404 1
    if (PyTypeNum_ISDATETIME(descr->type_num)) {
405
        /* Datetimes are flexible, but were not considered previously */
406 1
        dtype_class->parametric = NPY_TRUE;
407 1
        dtype_class->discover_descr_from_pyobject = (
408
                discover_datetime_and_timedelta_from_pyobject);
409 1
        if (descr->type_num == NPY_DATETIME) {
410 1
            dtype_class->is_known_scalar_type = datetime_known_scalar_types;
411
        }
412
    }
413 1
    else if (PyTypeNum_ISFLEXIBLE(descr->type_num)) {
414 1
        dtype_class->parametric = NPY_TRUE;
415 1
        dtype_class->default_descr = flexible_default_descr;
416 1
        if (descr->type_num == NPY_VOID) {
417 1
            dtype_class->discover_descr_from_pyobject = (
418
                    void_discover_descr_from_pyobject);
419
        }
420
        else {
421 1
            dtype_class->is_known_scalar_type = string_known_scalar_types;
422 1
            dtype_class->discover_descr_from_pyobject = (
423
                    string_discover_descr_from_pyobject);
424
        }
425
    }
426
    else {
427
        /* nonparametric case */
428 1
        dtype_class->discover_descr_from_pyobject = (
429
                nonparametric_discover_descr_from_pyobject);
430
    }
431

432 1
    if (_PyArray_MapPyTypeToDType(dtype_class, descr->typeobj,
433 1
            PyTypeNum_ISUSERDEF(dtype_class->type_num)) < 0) {
434 0
        Py_DECREF(dtype_class);
435
        return -1;
436
    }
437

438
    /* Finally, replace the current class of the descr */
439 1
    Py_SET_TYPE(descr, (PyTypeObject *)dtype_class);
440

441 1
    return 0;
442
}
443

444

445
/*
446
 * Simple exposed information, defined for each DType (class). This is
447
 * preliminary (the flags should also return bools).
448
 */
449
static PyMemberDef dtypemeta_members[] = {
450
    {"_abstract",
451
        T_BYTE, offsetof(PyArray_DTypeMeta, abstract), READONLY, NULL},
452
    {"type",
453
        T_OBJECT, offsetof(PyArray_DTypeMeta, scalar_type), READONLY, NULL},
454
    {"_parametric",
455
        T_BYTE, offsetof(PyArray_DTypeMeta, parametric), READONLY, NULL},
456
    {NULL, 0, 0, 0, NULL},
457
};
458

459

460
NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type = {
461
    PyVarObject_HEAD_INIT(NULL, 0)
462
    .tp_name = "numpy._DTypeMeta",
463
    .tp_basicsize = sizeof(PyArray_DTypeMeta),
464
    .tp_dealloc = (destructor)dtypemeta_dealloc,
465
    /* Types are garbage collected (see dtypemeta_is_gc documentation) */
466
    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
467
    .tp_doc = "Preliminary NumPy API: The Type of NumPy DTypes (metaclass)",
468
    .tp_members = dtypemeta_members,
469
    .tp_base = NULL,  /* set to PyType_Type at import time */
470
    .tp_init = (initproc)dtypemeta_init,
471
    .tp_new = dtypemeta_new,
472
    .tp_is_gc = dtypemeta_is_gc,
473
    .tp_traverse = (traverseproc)dtypemeta_traverse,
474
};

Read our documentation on viewing source code .

Loading