Skip to content

Commit 8a8e967

Browse files
WillAydjreback
WillAyd
authored and
jreback
committed
Split out JSON Date Converters (#31057)
1 parent f1bbb21 commit 8a8e967

File tree

4 files changed

+158
-123
lines changed

4 files changed

+158
-123
lines changed

‎pandas/_libs/src/ujson/python/date_conversions.c

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Conversion routines that are useful for serialization,
2+
// but which don't interact with JSON objects directly
3+
4+
#include "date_conversions.h"
5+
#include <../../../tslibs/src/datetime/np_datetime.h>
6+
#include <../../../tslibs/src/datetime/np_datetime_strings.h>
7+
8+
/*
9+
* Function: scaleNanosecToUnit
10+
* -----------------------------
11+
*
12+
* Scales an integer value representing time in nanoseconds to provided unit.
13+
*
14+
* Mutates the provided value directly. Returns 0 on success, non-zero on error.
15+
*/
16+
int scaleNanosecToUnit(npy_int64 *value, NPY_DATETIMEUNIT unit) {
17+
switch (unit) {
18+
case NPY_FR_ns:
19+
break;
20+
case NPY_FR_us:
21+
*value /= 1000LL;
22+
break;
23+
case NPY_FR_ms:
24+
*value /= 1000000LL;
25+
break;
26+
case NPY_FR_s:
27+
*value /= 1000000000LL;
28+
break;
29+
default:
30+
return -1;
31+
}
32+
33+
return 0;
34+
}
35+
36+
/* Converts the int64_t representation of a datetime to ISO; mutates len */
37+
char *int64ToIso(int64_t value, NPY_DATETIMEUNIT base, size_t *len) {
38+
npy_datetimestruct dts;
39+
int ret_code;
40+
41+
pandas_datetime_to_datetimestruct(value, NPY_FR_ns, &dts);
42+
43+
*len = (size_t)get_datetime_iso_8601_strlen(0, base);
44+
char *result = PyObject_Malloc(*len);
45+
46+
if (result == NULL) {
47+
PyErr_NoMemory();
48+
return NULL;
49+
}
50+
51+
ret_code = make_iso_8601_datetime(&dts, result, *len, base);
52+
if (ret_code != 0) {
53+
PyErr_SetString(PyExc_ValueError,
54+
"Could not convert datetime value to string");
55+
PyObject_Free(result);
56+
}
57+
58+
// Note that get_datetime_iso_8601_strlen just gives a generic size
59+
// for ISO string conversion, not the actual size used
60+
*len = strlen(result);
61+
return result;
62+
}
63+
64+
npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base) {
65+
scaleNanosecToUnit(&dt, base);
66+
return dt;
67+
}
68+
69+
/* Convert PyDatetime To ISO C-string. mutates len */
70+
char *PyDateTimeToIso(PyDateTime_Date *obj, NPY_DATETIMEUNIT base,
71+
size_t *len) {
72+
npy_datetimestruct dts;
73+
int ret;
74+
75+
ret = convert_pydatetime_to_datetimestruct(obj, &dts);
76+
if (ret != 0) {
77+
if (!PyErr_Occurred()) {
78+
PyErr_SetString(PyExc_ValueError,
79+
"Could not convert PyDateTime to numpy datetime");
80+
}
81+
return NULL;
82+
}
83+
84+
*len = (size_t)get_datetime_iso_8601_strlen(0, base);
85+
char *result = PyObject_Malloc(*len);
86+
ret = make_iso_8601_datetime(&dts, result, *len, base);
87+
88+
if (ret != 0) {
89+
PyErr_SetString(PyExc_ValueError,
90+
"Could not convert datetime value to string");
91+
PyObject_Free(result);
92+
return NULL;
93+
}
94+
95+
// Note that get_datetime_iso_8601_strlen just gives a generic size
96+
// for ISO string conversion, not the actual size used
97+
*len = strlen(result);
98+
return result;
99+
}
100+
101+
npy_datetime PyDateTimeToEpoch(PyDateTime_Date *dt, NPY_DATETIMEUNIT base) {
102+
npy_datetimestruct dts;
103+
int ret;
104+
105+
ret = convert_pydatetime_to_datetimestruct(dt, &dts);
106+
if (ret != 0) {
107+
if (!PyErr_Occurred()) {
108+
PyErr_SetString(PyExc_ValueError,
109+
"Could not convert PyDateTime to numpy datetime");
110+
}
111+
// TODO: is setting errMsg required?
112+
//((JSONObjectEncoder *)tc->encoder)->errorMsg = "";
113+
// return NULL;
114+
}
115+
116+
npy_datetime npy_dt = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts);
117+
return NpyDateTimeToEpoch(npy_dt, base);
118+
}

‎pandas/_libs/src/ujson/python/date_conversions.h

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#ifndef PANDAS__LIBS_SRC_UJSON_DATE_CONVERSIONS
2+
#define PANDAS__LIBS_SRC_UJSON_DATE_CONVERSIONS
3+
4+
#define PY_SSIZE_T_CLEAN
5+
#include <Python.h>
6+
#include <numpy/ndarraytypes.h>
7+
#include "datetime.h"
8+
9+
// Scales value inplace from nanosecond resolution to unit resolution
10+
int scaleNanosecToUnit(npy_int64 *value, NPY_DATETIMEUNIT unit);
11+
12+
// Converts an int64 object representing a date to ISO format
13+
// up to precision `base` e.g. base="s" yields 2020-01-03T00:00:00Z
14+
// while base="ns" yields "2020-01-01T00:00:00.000000000Z"
15+
// len is mutated to save the length of the returned string
16+
char *int64ToIso(int64_t value, NPY_DATETIMEUNIT base, size_t *len);
17+
18+
// TODO: this function doesn't do a lot; should augment or replace with
19+
// scaleNanosecToUnit
20+
npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base);
21+
22+
// Converts a Python object representing a Date / Datetime to ISO format
23+
// up to precision `base` e.g. base="s" yields 2020-01-03T00:00:00Z
24+
// while base="ns" yields "2020-01-01T00:00:00.000000000Z"
25+
// len is mutated to save the length of the returned string
26+
char *PyDateTimeToIso(PyDateTime_Date *obj, NPY_DATETIMEUNIT base, size_t *len);
27+
28+
// Convert a Python Date/Datetime to Unix epoch with resolution base
29+
npy_datetime PyDateTimeToEpoch(PyDateTime_Date *dt, NPY_DATETIMEUNIT base);
30+
31+
#endif

‎pandas/_libs/src/ujson/python/objToJSON.c

+3-122
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ Numeric decoder derived from from TCL library
4545
#include <numpy/ndarraytypes.h>
4646
#include <numpy/npy_math.h>
4747
#include <ultrajson.h>
48-
#include <../../../tslibs/src/datetime/np_datetime.h>
49-
#include <../../../tslibs/src/datetime/np_datetime_strings.h>
48+
#include "date_conversions.h"
5049
#include "datetime.h"
5150

5251
static PyTypeObject *type_decimal;
@@ -209,34 +208,6 @@ static TypeContext *createTypeContext(void) {
209208
return pc;
210209
}
211210

212-
/*
213-
* Function: scaleNanosecToUnit
214-
* -----------------------------
215-
*
216-
* Scales an integer value representing time in nanoseconds to provided unit.
217-
*
218-
* Mutates the provided value directly. Returns 0 on success, non-zero on error.
219-
*/
220-
static int scaleNanosecToUnit(npy_int64 *value, NPY_DATETIMEUNIT unit) {
221-
switch (unit) {
222-
case NPY_FR_ns:
223-
break;
224-
case NPY_FR_us:
225-
*value /= 1000LL;
226-
break;
227-
case NPY_FR_ms:
228-
*value /= 1000000LL;
229-
break;
230-
case NPY_FR_s:
231-
*value /= 1000000000LL;
232-
break;
233-
default:
234-
return -1;
235-
}
236-
237-
return 0;
238-
}
239-
240211
static PyObject *get_values(PyObject *obj) {
241212
PyObject *values = NULL;
242213

@@ -379,79 +350,13 @@ static char *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *Py_UNUSED(tc),
379350
return (char *)PyUnicode_AsUTF8AndSize(_obj, (Py_ssize_t *)_outLen);
380351
}
381352

382-
/* Converts the int64_t representation of a datetime to ISO; mutates len */
383-
static char *int64ToIso(int64_t value, NPY_DATETIMEUNIT base, size_t *len) {
384-
npy_datetimestruct dts;
385-
int ret_code;
386-
387-
pandas_datetime_to_datetimestruct(value, NPY_FR_ns, &dts);
388-
389-
*len = (size_t)get_datetime_iso_8601_strlen(0, base);
390-
char *result = PyObject_Malloc(*len);
391-
392-
if (result == NULL) {
393-
PyErr_NoMemory();
394-
return NULL;
395-
}
396-
397-
ret_code = make_iso_8601_datetime(&dts, result, *len, base);
398-
if (ret_code != 0) {
399-
PyErr_SetString(PyExc_ValueError,
400-
"Could not convert datetime value to string");
401-
PyObject_Free(result);
402-
}
403-
404-
// Note that get_datetime_iso_8601_strlen just gives a generic size
405-
// for ISO string conversion, not the actual size used
406-
*len = strlen(result);
407-
return result;
408-
}
409-
410353
/* JSON callback. returns a char* and mutates the pointer to *len */
411354
static char *NpyDateTimeToIsoCallback(JSOBJ Py_UNUSED(unused),
412355
JSONTypeContext *tc, size_t *len) {
413356
NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit;
414357
return int64ToIso(GET_TC(tc)->longValue, base, len);
415358
}
416359

417-
static npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base) {
418-
scaleNanosecToUnit(&dt, base);
419-
return dt;
420-
}
421-
422-
/* Convert PyDatetime To ISO C-string. mutates len */
423-
static char *PyDateTimeToIso(PyDateTime_Date *obj, NPY_DATETIMEUNIT base,
424-
size_t *len) {
425-
npy_datetimestruct dts;
426-
int ret;
427-
428-
ret = convert_pydatetime_to_datetimestruct(obj, &dts);
429-
if (ret != 0) {
430-
if (!PyErr_Occurred()) {
431-
PyErr_SetString(PyExc_ValueError,
432-
"Could not convert PyDateTime to numpy datetime");
433-
}
434-
return NULL;
435-
}
436-
437-
*len = (size_t)get_datetime_iso_8601_strlen(0, base);
438-
char *result = PyObject_Malloc(*len);
439-
ret = make_iso_8601_datetime(&dts, result, *len, base);
440-
441-
if (ret != 0) {
442-
PRINTMARK();
443-
PyErr_SetString(PyExc_ValueError,
444-
"Could not convert datetime value to string");
445-
PyObject_Free(result);
446-
return NULL;
447-
}
448-
449-
// Note that get_datetime_iso_8601_strlen just gives a generic size
450-
// for ISO string conversion, not the actual size used
451-
*len = strlen(result);
452-
return result;
453-
}
454-
455360
/* JSON callback */
456361
static char *PyDateTimeToIsoCallback(JSOBJ obj, JSONTypeContext *tc,
457362
size_t *len) {
@@ -465,30 +370,6 @@ static char *PyDateTimeToIsoCallback(JSOBJ obj, JSONTypeContext *tc,
465370
return PyDateTimeToIso(obj, base, len);
466371
}
467372

468-
static npy_datetime PyDateTimeToEpoch(PyObject *obj, NPY_DATETIMEUNIT base) {
469-
npy_datetimestruct dts;
470-
int ret;
471-
472-
if (!PyDate_Check(obj)) {
473-
// TODO: raise TypeError
474-
}
475-
PyDateTime_Date *dt = (PyDateTime_Date *)obj;
476-
477-
ret = convert_pydatetime_to_datetimestruct(dt, &dts);
478-
if (ret != 0) {
479-
if (!PyErr_Occurred()) {
480-
PyErr_SetString(PyExc_ValueError,
481-
"Could not convert PyDateTime to numpy datetime");
482-
}
483-
// TODO: is setting errMsg required?
484-
//((JSONObjectEncoder *)tc->encoder)->errorMsg = "";
485-
// return NULL;
486-
}
487-
488-
npy_datetime npy_dt = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts);
489-
return NpyDateTimeToEpoch(npy_dt, base);
490-
}
491-
492373
static char *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, size_t *outLen) {
493374
PyObject *obj = (PyObject *)_obj;
494375
PyObject *str;
@@ -1814,7 +1695,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
18141695
PRINTMARK();
18151696
NPY_DATETIMEUNIT base =
18161697
((PyObjectEncoder *)tc->encoder)->datetimeUnit;
1817-
GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base);
1698+
GET_TC(tc)->longValue = PyDateTimeToEpoch((PyDateTime_Date *)obj, base);
18181699
tc->type = JT_LONG;
18191700
}
18201701
return;
@@ -1840,7 +1721,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
18401721
PRINTMARK();
18411722
NPY_DATETIMEUNIT base =
18421723
((PyObjectEncoder *)tc->encoder)->datetimeUnit;
1843-
GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base);
1724+
GET_TC(tc)->longValue = PyDateTimeToEpoch((PyDateTime_Date *)obj, base);
18441725
tc->type = JT_LONG;
18451726
}
18461727
return;

‎setup.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ def initialize_options(self):
240240
pjoin(ujson_python, "ujson.c"),
241241
pjoin(ujson_python, "objToJSON.c"),
242242
pjoin(ujson_python, "JSONtoObj.c"),
243+
pjoin(ujson_python, "date_conversions.c"),
243244
pjoin(ujson_lib, "ultrajsonenc.c"),
244245
pjoin(ujson_lib, "ultrajsondec.c"),
245246
pjoin(util, "move.c"),
@@ -714,11 +715,15 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
714715

715716
ujson_ext = Extension(
716717
"pandas._libs.json",
717-
depends=["pandas/_libs/src/ujson/lib/ultrajson.h"],
718+
depends=[
719+
"pandas/_libs/src/ujson/lib/ultrajson.h",
720+
"pandas/_libs/src/ujson/python/date_conversions.h",
721+
],
718722
sources=(
719723
[
720724
"pandas/_libs/src/ujson/python/ujson.c",
721725
"pandas/_libs/src/ujson/python/objToJSON.c",
726+
"pandas/_libs/src/ujson/python/date_conversions.c",
722727
"pandas/_libs/src/ujson/python/JSONtoObj.c",
723728
"pandas/_libs/src/ujson/lib/ultrajsonenc.c",
724729
"pandas/_libs/src/ujson/lib/ultrajsondec.c",

0 commit comments

Comments
 (0)