virtuelle Umgebung teil20b

This commit is contained in:
Olli Graf
2023-09-01 16:01:37 +02:00
parent c9aee44812
commit c04c88e2af
7025 changed files with 1325157 additions and 0 deletions

View File

@@ -0,0 +1,30 @@
/*
Copyright (c) 2020, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
The full license is in the LICENSE file, distributed with this software.
*/
#pragma once
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <numpy/ndarraytypes.h>
// Scales value inplace from nanosecond resolution to unit resolution
int scaleNanosecToUnit(npy_int64 *value, NPY_DATETIMEUNIT unit);
// Converts an int64 object representing a date to ISO format
// up to precision `base` e.g. base="s" yields 2020-01-03T00:00:00Z
// while base="ns" yields "2020-01-01T00:00:00.000000000Z"
// len is mutated to save the length of the returned string
char *int64ToIso(int64_t value,
NPY_DATETIMEUNIT valueUnit,
NPY_DATETIMEUNIT base,
size_t *len);
// TODO(username): this function doesn't do a lot; should augment or
// replace with scaleNanosecToUnit
npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base);
char *int64ToIsoDuration(int64_t value, size_t *len);

View File

@@ -0,0 +1,113 @@
/*
Copyright (c) 2016, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
The full license is in the LICENSE file, distributed with this software.
Written by Mark Wiebe (mwwiebe@gmail.com)
Copyright (c) 2011 by Enthought, Inc.
Copyright (c) 2005-2011, NumPy Developers
All rights reserved.
See NUMPY_LICENSE.txt for the license.
*/
#pragma once
#ifndef NPY_NO_DEPRECATED_API
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#endif // NPY_NO_DEPRECATED_API
#include <numpy/ndarraytypes.h>
#include "pandas/vendored/numpy/datetime/np_datetime.h"
#include "pandas/vendored/numpy/datetime/np_datetime_strings.h"
#include "pandas/datetime/date_conversions.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
npy_datetime (*npy_datetimestruct_to_datetime)(NPY_DATETIMEUNIT,
const npy_datetimestruct *);
int (*scaleNanosecToUnit)(npy_int64 *, NPY_DATETIMEUNIT);
char *(*int64ToIso)(int64_t, NPY_DATETIMEUNIT, NPY_DATETIMEUNIT, size_t *);
npy_datetime (*NpyDateTimeToEpoch)(npy_datetime, NPY_DATETIMEUNIT);
char *(*PyDateTimeToIso)(PyObject *, NPY_DATETIMEUNIT, size_t *);
npy_datetime (*PyDateTimeToEpoch)(PyObject *, NPY_DATETIMEUNIT);
char *(*int64ToIsoDuration)(int64_t, size_t *);
void (*pandas_datetime_to_datetimestruct)(npy_datetime, NPY_DATETIMEUNIT,
npy_datetimestruct *);
void (*pandas_timedelta_to_timedeltastruct)(npy_datetime, NPY_DATETIMEUNIT,
pandas_timedeltastruct *);
int (*convert_pydatetime_to_datetimestruct)(PyObject *, npy_datetimestruct *);
int (*cmp_npy_datetimestruct)(const npy_datetimestruct *,
const npy_datetimestruct *);
PyArray_DatetimeMetaData (*get_datetime_metadata_from_dtype)(PyArray_Descr *);
int (*parse_iso_8601_datetime)(const char *, int, int, npy_datetimestruct *,
NPY_DATETIMEUNIT *, int *, int *, const char *,
int, FormatRequirement);
int (*get_datetime_iso_8601_strlen)(int, NPY_DATETIMEUNIT);
int (*make_iso_8601_datetime)(npy_datetimestruct *, char *, int, int,
NPY_DATETIMEUNIT);
int (*make_iso_8601_timedelta)(pandas_timedeltastruct *, char *, size_t *);
} PandasDateTime_CAPI;
// The capsule name appears limited to module.attributename; see bpo-32414
// cpython has an open PR gh-6898 to fix, but hasn't had traction for years
#define PandasDateTime_CAPSULE_NAME "pandas._pandas_datetime_CAPI"
/* block used as part of public API */
#ifndef _PANDAS_DATETIME_IMPL
static PandasDateTime_CAPI *PandasDateTimeAPI = NULL;
#define PandasDateTime_IMPORT \
PandasDateTimeAPI = \
(PandasDateTime_CAPI *)PyCapsule_Import(PandasDateTime_CAPSULE_NAME, 0)
#define npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT, npy_datetimestruct) \
PandasDateTimeAPI->npy_datetimestruct_to_datetime((NPY_DATETIMEUNIT), \
(npy_datetimestruct))
#define scaleNanosecToUnit(value, unit) \
PandasDateTimeAPI->scaleNanosecToUnit((value), (unit))
#define int64ToIso(value, valueUnit, base, len) \
PandasDateTimeAPI->int64ToIso((value), (valueUnit), (base), (len))
#define NpyDateTimeToEpoch(dt, base) \
PandasDateTimeAPI->NpyDateTimeToEpoch((dt), (base))
#define PyDateTimeToIso(obj, base, len) \
PandasDateTimeAPI->PyDateTimeToIso((obj), (base), (len))
#define PyDateTimeToEpoch(dt, base) \
PandasDateTimeAPI->PyDateTimeToEpoch((dt), (base))
#define int64ToIsoDuration(value, len) \
PandasDateTimeAPI->int64ToIsoDuration((value), (len))
#define pandas_datetime_to_datetimestruct(dt, base, out) \
PandasDateTimeAPI->pandas_datetime_to_datetimestruct((dt), (base), (out))
#define pandas_timedelta_to_timedeltastruct(td, base, out) \
PandasDateTimeAPI->pandas_timedelta_to_timedeltastruct((td), (base), (out))
#define convert_pydatetime_to_datetimestruct(dtobj, out) \
PandasDateTimeAPI->convert_pydatetime_to_datetimestruct((dtobj), (out))
#define cmp_npy_datetimestruct(a, b) \
PandasDateTimeAPI->cmp_npy_datetimestruct((a), (b))
#define get_datetime_metadata_from_dtype(dtype) \
PandasDateTimeAPI->get_datetime_metadata_from_dtype((dtype))
#define parse_iso_8601_datetime(str, len, want_exc, out, out_bestunit, \
out_local, out_tzoffset, format, format_len, \
format_requirement) \
PandasDateTimeAPI->parse_iso_8601_datetime( \
(str), (len), (want_exc), (out), (out_bestunit), (out_local), \
(out_tzoffset), (format), (format_len), (format_requirement))
#define get_datetime_iso_8601_strlen(local, base) \
PandasDateTimeAPI->get_datetime_iso_8601_strlen((local), (base))
#define make_iso_8601_datetime(dts, outstr, outlen, utc, base) \
PandasDateTimeAPI->make_iso_8601_datetime((dts), (outstr), (outlen), (utc), \
(base))
#define make_iso_8601_timedelta(tds, outstr, outlen) \
PandasDateTimeAPI->make_iso_8601_timedelta((tds), (outstr), (outlen))
#endif /* !defined(_PANDAS_DATETIME_IMPL) */
#ifdef __cplusplus
}
#endif

View File

@@ -0,0 +1,24 @@
/*
Copyright (c) 2016, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
The full license is in the LICENSE file, distributed with this software.
*/
#pragma once
#ifndef PANDAS_INLINE
#if defined(__clang__)
#define PANDAS_INLINE static __inline__ __attribute__ ((__unused__))
#elif defined(__GNUC__)
#define PANDAS_INLINE static __inline__
#elif defined(_MSC_VER)
#define PANDAS_INLINE static __inline
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
#define PANDAS_INLINE static inline
#else
#define PANDAS_INLINE
#endif // __GNUC__
#endif // PANDAS_INLINE

View File

@@ -0,0 +1,31 @@
/*
Copyright (c) 2016, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
The full license is in the LICENSE file, distributed with this software.
*/
#pragma once
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "tokenizer.h"
#define FS(source) ((file_source *)source)
typedef struct _rd_source {
PyObject *obj;
PyObject *buffer;
size_t position;
} rd_source;
#define RDS(source) ((rd_source *)source)
void *new_rd_source(PyObject *obj);
int del_rd_source(void *src);
void *buffer_rd_bytes(void *source, size_t nbytes, size_t *bytes_read,
int *status, const char *encoding_errors);

View File

@@ -0,0 +1,111 @@
/*
Copyright (c) 2023, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
*/
#pragma once
#ifdef __cplusplus
extern "C" {
#endif
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "pandas/parser/tokenizer.h"
typedef struct {
int (*to_double)(char *, double *, char, char, int *);
int (*floatify)(PyObject *, double *, int *);
void *(*new_rd_source)(PyObject *);
int (*del_rd_source)(void *);
void *(*buffer_rd_bytes)(void *, size_t, size_t *, int *, const char *);
void (*uint_state_init)(uint_state *);
int (*uint64_conflict)(uint_state *);
void (*coliter_setup)(coliter_t *, parser_t *, int64_t, int64_t);
parser_t *(*parser_new)(void);
int (*parser_init)(parser_t *);
void (*parser_free)(parser_t *);
void (*parser_del)(parser_t *);
int (*parser_add_skiprow)(parser_t *, int64_t);
int (*parser_set_skipfirstnrows)(parser_t *, int64_t);
void (*parser_set_default_options)(parser_t *);
int (*parser_consume_rows)(parser_t *, size_t);
int (*parser_trim_buffers)(parser_t *);
int (*tokenize_all_rows)(parser_t *, const char *);
int (*tokenize_nrows)(parser_t *, size_t, const char *);
int64_t (*str_to_int64)(const char *, int64_t, int64_t, int *, char);
uint64_t (*str_to_uint64)(uint_state *, const char *, int64_t, uint64_t,
int *, char);
double (*xstrtod)(const char *, char **, char, char, char, int, int *, int *);
double (*precise_xstrtod)(const char *, char **, char, char, char, int, int *,
int *);
double (*round_trip)(const char *, char **, char, char, char, int, int *,
int *);
int (*to_boolean)(const char *, uint8_t *);
} PandasParser_CAPI;
#define PandasParser_CAPSULE_NAME "pandas._pandas_parser_CAPI"
#ifndef _PANDAS_PARSER_IMPL
static PandasParser_CAPI *PandasParserAPI = NULL;
#define PandasParser_IMPORT \
PandasParserAPI = \
(PandasParser_CAPI *)PyCapsule_Import(PandasParser_CAPSULE_NAME, 0)
#define to_double(item, p_value, sci, decimal, maybe_int) \
PandasParserAPI->to_double((item), (p_value), (sci), (decimal), (maybe_int))
#define floatify(str, result, maybe_int) \
PandasParserAPI->floatify((str), (result), (maybe_int))
#define new_rd_source(obj) PandasParserAPI->new_rd_source((obj))
#define del_rd_source(src) PandasParserAPI->del_rd_source((src))
#define buffer_rd_bytes(source, nbytes, bytes_read, status, encoding_errors) \
PandasParserAPI->buffer_rd_bytes((source), (nbytes), (bytes_read), (status), \
(encoding_errors))
#define uint_state_init(self) PandasParserAPI->uint_state_init((self))
#define uint64_conflict(self) PandasParserAPI->uint64_conflict((self))
#define coliter_setup(self, parser, i, start) \
PandasParserAPI->coliter_setup((self), (parser), (i), (start))
#define parser_new PandasParserAPI->parser_new
#define parser_init(self) PandasParserAPI->parser_init((self))
#define parser_free(self) PandasParserAPI->parser_free((self))
#define parser_del(self) PandasParserAPI->parser_del((self))
#define parser_add_skiprow(self, row) \
PandasParserAPI->parser_add_skiprow((self), (row))
#define parser_set_skipfirstnrows(self, nrows) \
PandasParserAPI->parser_set_skipfirstnrows((self), (nrows))
#define parser_set_default_options(self) \
PandasParserAPI->parser_set_default_options((self))
#define parser_consume_rows(self, nrows) \
PandasParserAPI->parser_consume_rows((self), (nrows))
#define parser_trim_buffers(self) \
PandasParserAPI->parser_trim_buffers((self))
#define tokenize_all_rows(self, encoding_errors) \
PandasParserAPI->tokenize_all_rows((self), (encoding_errors))
#define tokenize_nrows(self, nrows, encoding_errors) \
PandasParserAPI->tokenize_nrows((self), (nrows), (encoding_errors))
#define str_to_int64(p_item, int_min, int_max, error, t_sep) \
PandasParserAPI->str_to_int64((p_item), (int_min), (int_max), (error), \
(t_sep))
#define str_to_uint64(state, p_item, int_max, uint_max, error, t_sep) \
PandasParserAPI->str_to_uint64((state), (p_item), (int_max), (uint_max), \
(error), (t_sep))
#define xstrtod(p, q, decimal, sci, tsep, skip_trailing, error, maybe_int) \
PandasParserAPI->xstrtod((p), (q), (decimal), (sci), (tsep), \
(skip_trailing), (error), (maybe_int))
#define precise_xstrtod(p, q, decimal, sci, tsep, skip_trailing, error, \
maybe_int) \
PandasParserAPI->precise_xstrtod((p), (q), (decimal), (sci), (tsep), \
(skip_trailing), (error), (maybe_int))
#define round_trip(p, q, decimal, sci, tsep, skip_trailing, error, maybe_int) \
PandasParserAPI->round_trip((p), (q), (decimal), (sci), (tsep), \
(skip_trailing), (error), (maybe_int))
#define to_boolean(item, val) PandasParserAPI->to_boolean((item), (val))
#endif /* !defined(_PANDAS_PARSER_IMPL) */
#ifdef __cplusplus
}
#endif

View File

@@ -0,0 +1,233 @@
/*
Copyright (c) 2012, Lambda Foundry, Inc., except where noted
Incorporates components of WarrenWeckesser/textreader, licensed under 3-clause
BSD
See LICENSE for the license
*/
#pragma once
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#define ERROR_NO_DIGITS 1
#define ERROR_OVERFLOW 2
#define ERROR_INVALID_CHARS 3
#include <stdint.h>
#include "pandas/inline_helper.h"
#include "pandas/portable.h"
#include "pandas/vendored/klib/khash.h"
#define STREAM_INIT_SIZE 32
#define REACHED_EOF 1
#define CALLING_READ_FAILED 2
/*
C flat file parsing low level code for pandas / NumPy
*/
/*
* Common set of error types for the read_rows() and tokenize()
* functions.
*/
// #define VERBOSE
#if defined(VERBOSE)
#define TRACE(X) printf X;
#else
#define TRACE(X)
#endif // VERBOSE
#define PARSER_OUT_OF_MEMORY -1
/*
* TODO: Might want to couple count_rows() with read_rows() to avoid
* duplication of some file I/O.
*/
typedef enum {
START_RECORD,
START_FIELD,
ESCAPED_CHAR,
IN_FIELD,
IN_QUOTED_FIELD,
ESCAPE_IN_QUOTED_FIELD,
QUOTE_IN_QUOTED_FIELD,
EAT_CRNL,
EAT_CRNL_NOP,
EAT_WHITESPACE,
EAT_COMMENT,
EAT_LINE_COMMENT,
WHITESPACE_LINE,
START_FIELD_IN_SKIP_LINE,
IN_FIELD_IN_SKIP_LINE,
IN_QUOTED_FIELD_IN_SKIP_LINE,
QUOTE_IN_QUOTED_FIELD_IN_SKIP_LINE,
FINISHED
} ParserState;
typedef enum {
QUOTE_MINIMAL,
QUOTE_ALL,
QUOTE_NONNUMERIC,
QUOTE_NONE
} QuoteStyle;
typedef enum {
ERROR,
WARN,
SKIP
} BadLineHandleMethod;
typedef void *(*io_callback)(void *src, size_t nbytes, size_t *bytes_read,
int *status, const char *encoding_errors);
typedef int (*io_cleanup)(void *src);
typedef struct parser_t {
void *source;
io_callback cb_io;
io_cleanup cb_cleanup;
int64_t chunksize; // Number of bytes to prepare for each chunk
char *data; // pointer to data to be processed
int64_t datalen; // amount of data available
int64_t datapos;
// where to write out tokenized data
char *stream;
uint64_t stream_len;
uint64_t stream_cap;
// Store words in (potentially ragged) matrix for now, hmm
char **words;
int64_t *word_starts; // where we are in the stream
uint64_t words_len;
uint64_t words_cap;
uint64_t max_words_cap; // maximum word cap encountered
char *pword_start; // pointer to stream start of current field
int64_t word_start; // position start of current field
int64_t *line_start; // position in words for start of line
int64_t *line_fields; // Number of fields in each line
uint64_t lines; // Number of (good) lines observed
uint64_t file_lines; // Number of lines (including bad or skipped)
uint64_t lines_cap; // Vector capacity
// Tokenizing stuff
ParserState state;
int doublequote; /* is " represented by ""? */
char delimiter; /* field separator */
int delim_whitespace; /* delimit by consuming space/tabs instead */
char quotechar; /* quote character */
char escapechar; /* escape character */
char lineterminator;
int skipinitialspace; /* ignore spaces following delimiter? */
int quoting; /* style of quoting to write */
char commentchar;
int allow_embedded_newline;
int usecols; // Boolean: 1: usecols provided, 0: none provided
Py_ssize_t expected_fields;
BadLineHandleMethod on_bad_lines;
// floating point options
char decimal;
char sci;
// thousands separator (comma, period)
char thousands;
int header; // Boolean: 1: has header, 0: no header
int64_t header_start; // header row start
uint64_t header_end; // header row end
void *skipset;
PyObject *skipfunc;
int64_t skip_first_N_rows;
int64_t skip_footer;
double (*double_converter)(const char *, char **,
char, char, char, int, int *, int *);
// error handling
char *warn_msg;
char *error_msg;
int skip_empty_lines;
} parser_t;
typedef struct coliter_t {
char **words;
int64_t *line_start;
int64_t col;
} coliter_t;
void coliter_setup(coliter_t *self, parser_t *parser, int64_t i, int64_t start);
#define COLITER_NEXT(iter, word) \
do { \
const int64_t i = *iter.line_start++ + iter.col; \
word = i >= *iter.line_start ? "" : iter.words[i]; \
} while (0)
parser_t *parser_new(void);
int parser_init(parser_t *self);
int parser_consume_rows(parser_t *self, size_t nrows);
int parser_trim_buffers(parser_t *self);
int parser_add_skiprow(parser_t *self, int64_t row);
int parser_set_skipfirstnrows(parser_t *self, int64_t nrows);
void parser_free(parser_t *self);
void parser_del(parser_t *self);
void parser_set_default_options(parser_t *self);
int tokenize_nrows(parser_t *self, size_t nrows, const char *encoding_errors);
int tokenize_all_rows(parser_t *self, const char *encoding_errors);
// Have parsed / type-converted a chunk of data
// and want to free memory from the token stream
typedef struct uint_state {
int seen_sint;
int seen_uint;
int seen_null;
} uint_state;
void uint_state_init(uint_state *self);
int uint64_conflict(uint_state *self);
uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
uint64_t uint_max, int *error, char tsep);
int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
int *error, char tsep);
double xstrtod(const char *p, char **q, char decimal, char sci, char tsep,
int skip_trailing, int *error, int *maybe_int);
double precise_xstrtod(const char *p, char **q, char decimal,
char sci, char tsep, int skip_trailing,
int *error, int *maybe_int);
// GH-15140 - round_trip requires and acquires the GIL on its own
double round_trip(const char *p, char **q, char decimal, char sci, char tsep,
int skip_trailing, int *error, int *maybe_int);
int to_boolean(const char *item, uint8_t *val);

View File

@@ -0,0 +1,24 @@
/*
Copyright (c) 2016, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
The full license is in the LICENSE file, distributed with this software.
*/
#pragma once
#include <string.h>
#if defined(_MSC_VER)
#define strcasecmp(s1, s2) _stricmp(s1, s2)
#endif
// GH-23516 - works around locale perf issues
// from MUSL libc, MIT Licensed - see LICENSES
#define isdigit_ascii(c) (((unsigned)(c) - '0') < 10u)
#define getdigit_ascii(c, default) (isdigit_ascii(c) ? ((int)((c) - '0')) : default)
#define isspace_ascii(c) (((c) == ' ') || (((unsigned)(c) - '\t') < 5))
#define toupper_ascii(c) ((((unsigned)(c) - 'a') < 26) ? ((c) & 0x5f) : (c))
#define tolower_ascii(c) ((((unsigned)(c) - 'A') < 26) ? ((c) | 0x20) : (c))

View File

@@ -0,0 +1,297 @@
/*
Copyright (c) 2016, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
The full license is in the LICENSE file, distributed with this software.
Flexibly-sized, index-able skiplist data structure for maintaining a sorted
list of values
Port of Wes McKinney's Cython version of Raymond Hettinger's original pure
Python recipe (https://rhettinger.wordpress.com/2010/02/06/lost-knowledge/)
*/
#pragma once
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "pandas/inline_helper.h"
PANDAS_INLINE float __skiplist_nanf(void) {
const union {
int __i;
float __f;
} __bint = {0x7fc00000UL};
return __bint.__f;
}
#define PANDAS_NAN ((double)__skiplist_nanf())
PANDAS_INLINE double Log2(double val) { return log(val) / log(2.); }
typedef struct node_t node_t;
struct node_t {
node_t **next;
int *width;
double value;
int is_nil;
int levels;
int ref_count;
};
typedef struct {
node_t *head;
node_t **tmp_chain;
int *tmp_steps;
int size;
int maxlevels;
} skiplist_t;
PANDAS_INLINE double urand(void) {
return ((double)rand() + 1) / ((double)RAND_MAX + 2);
}
PANDAS_INLINE int int_min(int a, int b) { return a < b ? a : b; }
PANDAS_INLINE node_t *node_init(double value, int levels) {
node_t *result;
result = (node_t *)malloc(sizeof(node_t));
if (result) {
result->value = value;
result->levels = levels;
result->is_nil = 0;
result->ref_count = 0;
result->next = (node_t **)malloc(levels * sizeof(node_t *));
result->width = (int *)malloc(levels * sizeof(int));
if (!(result->next && result->width) && (levels != 0)) {
free(result->next);
free(result->width);
free(result);
return NULL;
}
}
return result;
}
// do this ourselves
PANDAS_INLINE void node_incref(node_t *node) { ++(node->ref_count); }
PANDAS_INLINE void node_decref(node_t *node) { --(node->ref_count); }
static void node_destroy(node_t *node) {
int i;
if (node) {
if (node->ref_count <= 1) {
for (i = 0; i < node->levels; ++i) {
node_destroy(node->next[i]);
}
free(node->next);
free(node->width);
// printf("Reference count was 1, freeing\n");
free(node);
} else {
node_decref(node);
}
// pretty sure that freeing the struct above will be enough
}
}
PANDAS_INLINE void skiplist_destroy(skiplist_t *skp) {
if (skp) {
node_destroy(skp->head);
free(skp->tmp_steps);
free(skp->tmp_chain);
free(skp);
}
}
PANDAS_INLINE skiplist_t *skiplist_init(int expected_size) {
skiplist_t *result;
node_t *NIL, *head;
int maxlevels, i;
maxlevels = 1 + Log2((double)expected_size);
result = (skiplist_t *)malloc(sizeof(skiplist_t));
if (!result) {
return NULL;
}
result->tmp_chain = (node_t **)malloc(maxlevels * sizeof(node_t *));
result->tmp_steps = (int *)malloc(maxlevels * sizeof(int));
result->maxlevels = maxlevels;
result->size = 0;
head = result->head = node_init(PANDAS_NAN, maxlevels);
NIL = node_init(0.0, 0);
if (!(result->tmp_chain && result->tmp_steps && result->head && NIL)) {
skiplist_destroy(result);
node_destroy(NIL);
return NULL;
}
node_incref(head);
NIL->is_nil = 1;
for (i = 0; i < maxlevels; ++i) {
head->next[i] = NIL;
head->width[i] = 1;
node_incref(NIL);
}
return result;
}
// 1 if left < right, 0 if left == right, -1 if left > right
PANDAS_INLINE int _node_cmp(node_t *node, double value) {
if (node->is_nil || node->value > value) {
return -1;
} else if (node->value < value) {
return 1;
} else {
return 0;
}
}
PANDAS_INLINE double skiplist_get(skiplist_t *skp, int i, int *ret) {
node_t *node;
int level;
if (i < 0 || i >= skp->size) {
*ret = 0;
return 0;
}
node = skp->head;
++i;
for (level = skp->maxlevels - 1; level >= 0; --level) {
while (node->width[level] <= i) {
i -= node->width[level];
node = node->next[level];
}
}
*ret = 1;
return node->value;
}
// Returns the lowest rank of all elements with value `value`, as opposed to the
// highest rank returned by `skiplist_insert`.
PANDAS_INLINE int skiplist_min_rank(skiplist_t *skp, double value) {
node_t *node;
int level, rank = 0;
node = skp->head;
for (level = skp->maxlevels - 1; level >= 0; --level) {
while (_node_cmp(node->next[level], value) > 0) {
rank += node->width[level];
node = node->next[level];
}
}
return rank + 1;
}
// Returns the rank of the inserted element. When there are duplicates,
// `rank` is the highest of the group, i.e. the 'max' method of
// https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.rank.html
PANDAS_INLINE int skiplist_insert(skiplist_t *skp, double value) {
node_t *node, *prevnode, *newnode, *next_at_level;
int *steps_at_level;
int size, steps, level, rank = 0;
node_t **chain;
chain = skp->tmp_chain;
steps_at_level = skp->tmp_steps;
memset(steps_at_level, 0, skp->maxlevels * sizeof(int));
node = skp->head;
for (level = skp->maxlevels - 1; level >= 0; --level) {
next_at_level = node->next[level];
while (_node_cmp(next_at_level, value) >= 0) {
steps_at_level[level] += node->width[level];
rank += node->width[level];
node = next_at_level;
next_at_level = node->next[level];
}
chain[level] = node;
}
size = int_min(skp->maxlevels, 1 - ((int)Log2(urand())));
newnode = node_init(value, size);
if (!newnode) {
return -1;
}
steps = 0;
for (level = 0; level < size; ++level) {
prevnode = chain[level];
newnode->next[level] = prevnode->next[level];
prevnode->next[level] = newnode;
node_incref(newnode); // increment the reference count
newnode->width[level] = prevnode->width[level] - steps;
prevnode->width[level] = steps + 1;
steps += steps_at_level[level];
}
for (level = size; level < skp->maxlevels; ++level) {
chain[level]->width[level] += 1;
}
++(skp->size);
return rank + 1;
}
PANDAS_INLINE int skiplist_remove(skiplist_t *skp, double value) {
int level, size;
node_t *node, *prevnode, *tmpnode, *next_at_level;
node_t **chain;
chain = skp->tmp_chain;
node = skp->head;
for (level = skp->maxlevels - 1; level >= 0; --level) {
next_at_level = node->next[level];
while (_node_cmp(next_at_level, value) > 0) {
node = next_at_level;
next_at_level = node->next[level];
}
chain[level] = node;
}
if (value != chain[0]->next[0]->value) {
return 0;
}
size = chain[0]->next[0]->levels;
for (level = 0; level < size; ++level) {
prevnode = chain[level];
tmpnode = prevnode->next[level];
prevnode->width[level] += tmpnode->width[level] - 1;
prevnode->next[level] = tmpnode->next[level];
tmpnode->next[level] = NULL;
node_destroy(tmpnode); // decrement refcount or free
}
for (level = size; level < skp->maxlevels; ++level) {
--(chain[level]->width[level]);
}
--(skp->size);
return 1;
}

View File

@@ -0,0 +1,719 @@
/* The MIT License
Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
/*
An example:
#include "khash.h"
KHASH_MAP_INIT_INT(32, char)
int main() {
int ret, is_missing;
khiter_t k;
khash_t(32) *h = kh_init(32);
k = kh_put(32, h, 5, &ret);
if (!ret) kh_del(32, h, k);
kh_value(h, k) = 10;
k = kh_get(32, h, 10);
is_missing = (k == kh_end(h));
k = kh_get(32, h, 5);
kh_del(32, h, k);
for (k = kh_begin(h); k != kh_end(h); ++k)
if (kh_exist(h, k)) kh_value(h, k) = 1;
kh_destroy(32, h);
return 0;
}
*/
/*
2011-09-16 (0.2.6):
* The capacity is a power of 2. This seems to dramatically improve the
speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
- https://github.com/stefanocasazza/ULib
- https://nothings.org/computer/judy/
* Allow to optionally use linear probing which usually has better
performance for random input. Double hashing is still the default as it
is more robust to certain non-random input.
* Added Wang's integer hash function (not used by default). This hash
function is more robust to certain non-random input.
2011-02-14 (0.2.5):
* Allow to declare global functions.
2009-09-26 (0.2.4):
* Improve portability
2008-09-19 (0.2.3):
* Corrected the example
* Improved interfaces
2008-09-11 (0.2.2):
* Improved speed a little in kh_put()
2008-09-10 (0.2.1):
* Added kh_clear()
* Fixed a compiling error
2008-09-02 (0.2.0):
* Changed to token concatenation which increases flexibility.
2008-08-31 (0.1.2):
* Fixed a bug in kh_get(), which has not been tested previously.
2008-08-31 (0.1.1):
* Added destructor
*/
#ifndef __AC_KHASH_H
#define __AC_KHASH_H
/*!
@header
Generic hash table library.
*/
#define AC_VERSION_KHASH_H "0.2.6"
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include "pandas/inline_helper.h"
// hooks for memory allocator, C-runtime allocator used per default
#ifndef KHASH_MALLOC
#define KHASH_MALLOC malloc
#endif
#ifndef KHASH_REALLOC
#define KHASH_REALLOC realloc
#endif
#ifndef KHASH_CALLOC
#define KHASH_CALLOC calloc
#endif
#ifndef KHASH_FREE
#define KHASH_FREE free
#endif
#if UINT_MAX == 0xffffffffu
typedef unsigned int khuint32_t;
typedef signed int khint32_t;
#elif ULONG_MAX == 0xffffffffu
typedef unsigned long khuint32_t;
typedef signed long khint32_t;
#endif
#if ULONG_MAX == ULLONG_MAX
typedef unsigned long khuint64_t;
typedef signed long khint64_t;
#else
typedef unsigned long long khuint64_t;
typedef signed long long khint64_t;
#endif
#if UINT_MAX == 0xffffu
typedef unsigned int khuint16_t;
typedef signed int khint16_t;
#elif USHRT_MAX == 0xffffu
typedef unsigned short khuint16_t;
typedef signed short khint16_t;
#endif
#if UCHAR_MAX == 0xffu
typedef unsigned char khuint8_t;
typedef signed char khint8_t;
#endif
typedef double khfloat64_t;
typedef float khfloat32_t;
typedef khuint32_t khuint_t;
typedef khuint_t khiter_t;
#define __ac_isempty(flag, i) ((flag[i>>5]>>(i&0x1fU))&1)
#define __ac_isdel(flag, i) (0)
#define __ac_iseither(flag, i) __ac_isempty(flag, i)
#define __ac_set_isdel_false(flag, i) (0)
#define __ac_set_isempty_false(flag, i) (flag[i>>5]&=~(1ul<<(i&0x1fU)))
#define __ac_set_isempty_true(flag, i) (flag[i>>5]|=(1ul<<(i&0x1fU)))
#define __ac_set_isboth_false(flag, i) __ac_set_isempty_false(flag, i)
#define __ac_set_isdel_true(flag, i) ((void)0)
// specializations of https://github.com/aappleby/smhasher/blob/master/src/MurmurHash2.cpp
khuint32_t PANDAS_INLINE murmur2_32to32(khuint32_t k){
const khuint32_t SEED = 0xc70f6907UL;
// 'm' and 'r' are mixing constants generated offline.
// They're not really 'magic', they just happen to work well.
const khuint32_t M_32 = 0x5bd1e995;
const int R_32 = 24;
// Initialize the hash to a 'random' value
khuint32_t h = SEED ^ 4;
//handle 4 bytes:
k *= M_32;
k ^= k >> R_32;
k *= M_32;
h *= M_32;
h ^= k;
// Do a few final mixes of the hash to ensure the "last few
// bytes" are well-incorporated. (Really needed here?)
h ^= h >> 13;
h *= M_32;
h ^= h >> 15;
return h;
}
// it is possible to have a special x64-version, which would need less operations, but
// using 32bit version always has also some benefits:
// - one code for 32bit and 64bit builds
// - the same case for 32bit and 64bit builds
// - no performance difference could be measured compared to a possible x64-version
khuint32_t PANDAS_INLINE murmur2_32_32to32(khuint32_t k1, khuint32_t k2){
const khuint32_t SEED = 0xc70f6907UL;
// 'm' and 'r' are mixing constants generated offline.
// They're not really 'magic', they just happen to work well.
const khuint32_t M_32 = 0x5bd1e995;
const int R_32 = 24;
// Initialize the hash to a 'random' value
khuint32_t h = SEED ^ 4;
//handle first 4 bytes:
k1 *= M_32;
k1 ^= k1 >> R_32;
k1 *= M_32;
h *= M_32;
h ^= k1;
//handle second 4 bytes:
k2 *= M_32;
k2 ^= k2 >> R_32;
k2 *= M_32;
h *= M_32;
h ^= k2;
// Do a few final mixes of the hash to ensure the "last few
// bytes" are well-incorporated.
h ^= h >> 13;
h *= M_32;
h ^= h >> 15;
return h;
}
khuint32_t PANDAS_INLINE murmur2_64to32(khuint64_t k){
khuint32_t k1 = (khuint32_t)k;
khuint32_t k2 = (khuint32_t)(k >> 32);
return murmur2_32_32to32(k1, k2);
}
#ifdef KHASH_LINEAR
#define __ac_inc(k, m) 1
#else
#define __ac_inc(k, m) (murmur2_32to32(k) | 1) & (m)
#endif
#define __ac_fsize(m) ((m) < 32? 1 : (m)>>5)
#ifndef kroundup32
#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
#endif
static const double __ac_HASH_UPPER = 0.77;
#define KHASH_DECLARE(name, khkey_t, khval_t) \
typedef struct { \
khuint_t n_buckets, size, n_occupied, upper_bound; \
khuint32_t *flags; \
khkey_t *keys; \
khval_t *vals; \
} kh_##name##_t; \
extern kh_##name##_t *kh_init_##name(); \
extern void kh_destroy_##name(kh_##name##_t *h); \
extern void kh_clear_##name(kh_##name##_t *h); \
extern khuint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
extern void kh_resize_##name(kh_##name##_t *h, khuint_t new_n_buckets); \
extern khuint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
extern void kh_del_##name(kh_##name##_t *h, khuint_t x);
#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
typedef struct { \
khuint_t n_buckets, size, n_occupied, upper_bound; \
khuint32_t *flags; \
khkey_t *keys; \
khval_t *vals; \
} kh_##name##_t; \
SCOPE kh_##name##_t *kh_init_##name(void) { \
return (kh_##name##_t*)KHASH_CALLOC(1, sizeof(kh_##name##_t)); \
} \
SCOPE void kh_destroy_##name(kh_##name##_t *h) \
{ \
if (h) { \
KHASH_FREE(h->keys); KHASH_FREE(h->flags); \
KHASH_FREE(h->vals); \
KHASH_FREE(h); \
} \
} \
SCOPE void kh_clear_##name(kh_##name##_t *h) \
{ \
if (h && h->flags) { \
memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khuint32_t)); \
h->size = h->n_occupied = 0; \
} \
} \
SCOPE khuint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \
{ \
if (h->n_buckets) { \
khuint_t inc, k, i, last, mask; \
mask = h->n_buckets - 1; \
k = __hash_func(key); i = k & mask; \
inc = __ac_inc(k, mask); last = i; /* inc==1 for linear probing */ \
while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
i = (i + inc) & mask; \
if (i == last) return h->n_buckets; \
} \
return __ac_iseither(h->flags, i)? h->n_buckets : i; \
} else return 0; \
} \
SCOPE void kh_resize_##name(kh_##name##_t *h, khuint_t new_n_buckets) \
{ /* This function uses 0.25*n_bucktes bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
khuint32_t *new_flags = 0; \
khuint_t j = 1; \
{ \
kroundup32(new_n_buckets); \
if (new_n_buckets < 4) new_n_buckets = 4; \
if (h->size >= (khuint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \
else { /* hash table size to be changed (shrink or expand); rehash */ \
new_flags = (khuint32_t*)KHASH_MALLOC(__ac_fsize(new_n_buckets) * sizeof(khuint32_t)); \
memset(new_flags, 0xff, __ac_fsize(new_n_buckets) * sizeof(khuint32_t)); \
if (h->n_buckets < new_n_buckets) { /* expand */ \
h->keys = (khkey_t*)KHASH_REALLOC(h->keys, new_n_buckets * sizeof(khkey_t)); \
if (kh_is_map) h->vals = (khval_t*)KHASH_REALLOC(h->vals, new_n_buckets * sizeof(khval_t)); \
} /* otherwise shrink */ \
} \
} \
if (j) { /* rehashing is needed */ \
for (j = 0; j != h->n_buckets; ++j) { \
if (__ac_iseither(h->flags, j) == 0) { \
khkey_t key = h->keys[j]; \
khval_t val; \
khuint_t new_mask; \
new_mask = new_n_buckets - 1; \
if (kh_is_map) val = h->vals[j]; \
__ac_set_isempty_true(h->flags, j); \
while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
khuint_t inc, k, i; \
k = __hash_func(key); \
i = k & new_mask; \
inc = __ac_inc(k, new_mask); \
while (!__ac_isempty(new_flags, i)) i = (i + inc) & new_mask; \
__ac_set_isempty_false(new_flags, i); \
if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
__ac_set_isempty_true(h->flags, i); /* mark it as deleted in the old hash table */ \
} else { /* write the element and jump out of the loop */ \
h->keys[i] = key; \
if (kh_is_map) h->vals[i] = val; \
break; \
} \
} \
} \
} \
if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
h->keys = (khkey_t*)KHASH_REALLOC(h->keys, new_n_buckets * sizeof(khkey_t)); \
if (kh_is_map) h->vals = (khval_t*)KHASH_REALLOC(h->vals, new_n_buckets * sizeof(khval_t)); \
} \
KHASH_FREE(h->flags); /* free the working space */ \
h->flags = new_flags; \
h->n_buckets = new_n_buckets; \
h->n_occupied = h->size; \
h->upper_bound = (khuint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
} \
} \
SCOPE khuint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
{ \
khuint_t x; \
if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
if (h->n_buckets > (h->size<<1)) kh_resize_##name(h, h->n_buckets - 1); /* clear "deleted" elements */ \
else kh_resize_##name(h, h->n_buckets + 1); /* expand the hash table */ \
} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
{ \
khuint_t inc, k, i, site, last, mask = h->n_buckets - 1; \
x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \
else { \
inc = __ac_inc(k, mask); last = i; \
while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
if (__ac_isdel(h->flags, i)) site = i; \
i = (i + inc) & mask; \
if (i == last) { x = site; break; } \
} \
if (x == h->n_buckets) { \
if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
else x = i; \
} \
} \
} \
if (__ac_isempty(h->flags, x)) { /* not present at all */ \
h->keys[x] = key; \
__ac_set_isboth_false(h->flags, x); \
++h->size; ++h->n_occupied; \
*ret = 1; \
} else if (__ac_isdel(h->flags, x)) { /* deleted */ \
h->keys[x] = key; \
__ac_set_isboth_false(h->flags, x); \
++h->size; \
*ret = 2; \
} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
return x; \
} \
SCOPE void kh_del_##name(kh_##name##_t *h, khuint_t x) \
{ \
if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \
__ac_set_isdel_true(h->flags, x); \
--h->size; \
} \
}
#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
KHASH_INIT2(name, PANDAS_INLINE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
/* --- BEGIN OF HASH FUNCTIONS --- */
/*! @function
@abstract Integer hash function
@param key The integer [khuint32_t]
@return The hash value [khuint_t]
*/
#define kh_int_hash_func(key) (khuint32_t)(key)
/*! @function
@abstract Integer comparison function
*/
#define kh_int_hash_equal(a, b) ((a) == (b))
/*! @function
@abstract 64-bit integer hash function
@param key The integer [khuint64_t]
@return The hash value [khuint_t]
*/
PANDAS_INLINE khuint_t kh_int64_hash_func(khuint64_t key)
{
return (khuint_t)((key)>>33^(key)^(key)<<11);
}
/*! @function
@abstract 64-bit integer comparison function
*/
#define kh_int64_hash_equal(a, b) ((a) == (b))
/*! @function
@abstract const char* hash function
@param s Pointer to a null terminated string
@return The hash value
*/
PANDAS_INLINE khuint_t __ac_X31_hash_string(const char *s)
{
khuint_t h = *s;
if (h) for (++s ; *s; ++s) h = (h << 5) - h + *s;
return h;
}
/*! @function
@abstract Another interface to const char* hash function
@param key Pointer to a null terminated string [const char*]
@return The hash value [khuint_t]
*/
#define kh_str_hash_func(key) __ac_X31_hash_string(key)
/*! @function
@abstract Const char* comparison function
*/
#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
PANDAS_INLINE khuint_t __ac_Wang_hash(khuint_t key)
{
key += ~(key << 15);
key ^= (key >> 10);
key += (key << 3);
key ^= (key >> 6);
key += ~(key << 11);
key ^= (key >> 16);
return key;
}
#define kh_int_hash_func2(k) __ac_Wang_hash((khuint_t)key)
/* --- END OF HASH FUNCTIONS --- */
/* Other convenient macros... */
/*!
@abstract Type of the hash table.
@param name Name of the hash table [symbol]
*/
#define khash_t(name) kh_##name##_t
/*! @function
@abstract Initiate a hash table.
@param name Name of the hash table [symbol]
@return Pointer to the hash table [khash_t(name)*]
*/
#define kh_init(name) kh_init_##name(void)
/*! @function
@abstract Destroy a hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
*/
#define kh_destroy(name, h) kh_destroy_##name(h)
/*! @function
@abstract Reset a hash table without deallocating memory.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
*/
#define kh_clear(name, h) kh_clear_##name(h)
/*! @function
@abstract Resize a hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
@param s New size [khuint_t]
*/
#define kh_resize(name, h, s) kh_resize_##name(h, s)
/*! @function
@abstract Insert a key to the hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
@param k Key [type of keys]
@param r Extra return code: 0 if the key is present in the hash table;
1 if the bucket is empty (never used); 2 if the element in
the bucket has been deleted [int*]
@return Iterator to the inserted element [khuint_t]
*/
#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
/*! @function
@abstract Retrieve a key from the hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
@param k Key [type of keys]
@return Iterator to the found element, or kh_end(h) is the element is absent [khuint_t]
*/
#define kh_get(name, h, k) kh_get_##name(h, k)
/*! @function
@abstract Remove a key from the hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
@param k Iterator to the element to be deleted [khuint_t]
*/
#define kh_del(name, h, k) kh_del_##name(h, k)
/*! @function
@abstract Test whether a bucket contains data.
@param h Pointer to the hash table [khash_t(name)*]
@param x Iterator to the bucket [khuint_t]
@return 1 if containing data; 0 otherwise [int]
*/
#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
/*! @function
@abstract Get key given an iterator
@param h Pointer to the hash table [khash_t(name)*]
@param x Iterator to the bucket [khuint_t]
@return Key [type of keys]
*/
#define kh_key(h, x) ((h)->keys[x])
/*! @function
@abstract Get value given an iterator
@param h Pointer to the hash table [khash_t(name)*]
@param x Iterator to the bucket [khuint_t]
@return Value [type of values]
@discussion For hash sets, calling this results in segfault.
*/
#define kh_val(h, x) ((h)->vals[x])
/*! @function
@abstract Alias of kh_val()
*/
#define kh_value(h, x) ((h)->vals[x])
/*! @function
@abstract Get the start iterator
@param h Pointer to the hash table [khash_t(name)*]
@return The start iterator [khuint_t]
*/
#define kh_begin(h) (khuint_t)(0)
/*! @function
@abstract Get the end iterator
@param h Pointer to the hash table [khash_t(name)*]
@return The end iterator [khuint_t]
*/
#define kh_end(h) ((h)->n_buckets)
/*! @function
@abstract Get the number of elements in the hash table
@param h Pointer to the hash table [khash_t(name)*]
@return Number of elements in the hash table [khuint_t]
*/
#define kh_size(h) ((h)->size)
/*! @function
@abstract Get the number of buckets in the hash table
@param h Pointer to the hash table [khash_t(name)*]
@return Number of buckets in the hash table [khuint_t]
*/
#define kh_n_buckets(h) ((h)->n_buckets)
/* More convenient interfaces */
/*! @function
@abstract Instantiate a hash set containing integer keys
@param name Name of the hash table [symbol]
*/
#define KHASH_SET_INIT_INT(name) \
KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
/*! @function
@abstract Instantiate a hash map containing integer keys
@param name Name of the hash table [symbol]
@param khval_t Type of values [type]
*/
#define KHASH_MAP_INIT_INT(name, khval_t) \
KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
#define KHASH_MAP_INIT_UINT(name, khval_t) \
KHASH_INIT(name, khuint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
/*! @function
@abstract Instantiate a hash map containing 64-bit integer keys
@param name Name of the hash table [symbol]
*/
#define KHASH_SET_INIT_UINT64(name) \
KHASH_INIT(name, khuint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
#define KHASH_SET_INIT_INT64(name) \
KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
/*! @function
@abstract Instantiate a hash map containing 64-bit integer keys
@param name Name of the hash table [symbol]
@param khval_t Type of values [type]
*/
#define KHASH_MAP_INIT_UINT64(name, khval_t) \
KHASH_INIT(name, khuint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
#define KHASH_MAP_INIT_INT64(name, khval_t) \
KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
/*! @function
@abstract Instantiate a hash map containing 16bit-integer keys
@param name Name of the hash table [symbol]
@param khval_t Type of values [type]
*/
#define KHASH_MAP_INIT_INT16(name, khval_t) \
KHASH_INIT(name, khint16_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
#define KHASH_MAP_INIT_UINT16(name, khval_t) \
KHASH_INIT(name, khuint16_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
/*! @function
@abstract Instantiate a hash map containing 8bit-integer keys
@param name Name of the hash table [symbol]
@param khval_t Type of values [type]
*/
#define KHASH_MAP_INIT_INT8(name, khval_t) \
KHASH_INIT(name, khint8_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
#define KHASH_MAP_INIT_UINT8(name, khval_t) \
KHASH_INIT(name, khuint8_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
typedef const char *kh_cstr_t;
/*! @function
@abstract Instantiate a hash map containing const char* keys
@param name Name of the hash table [symbol]
*/
#define KHASH_SET_INIT_STR(name) \
KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
/*! @function
@abstract Instantiate a hash map containing const char* keys
@param name Name of the hash table [symbol]
@param khval_t Type of values [type]
*/
#define KHASH_MAP_INIT_STR(name, khval_t) \
KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
#define kh_exist_str(h, k) (kh_exist(h, k))
#define kh_exist_float64(h, k) (kh_exist(h, k))
#define kh_exist_uint64(h, k) (kh_exist(h, k))
#define kh_exist_int64(h, k) (kh_exist(h, k))
#define kh_exist_float32(h, k) (kh_exist(h, k))
#define kh_exist_int32(h, k) (kh_exist(h, k))
#define kh_exist_uint32(h, k) (kh_exist(h, k))
#define kh_exist_int16(h, k) (kh_exist(h, k))
#define kh_exist_uint16(h, k) (kh_exist(h, k))
#define kh_exist_int8(h, k) (kh_exist(h, k))
#define kh_exist_uint8(h, k) (kh_exist(h, k))
KHASH_MAP_INIT_STR(str, size_t)
KHASH_MAP_INIT_INT(int32, size_t)
KHASH_MAP_INIT_UINT(uint32, size_t)
KHASH_MAP_INIT_INT64(int64, size_t)
KHASH_MAP_INIT_UINT64(uint64, size_t)
KHASH_MAP_INIT_INT16(int16, size_t)
KHASH_MAP_INIT_UINT16(uint16, size_t)
KHASH_MAP_INIT_INT8(int8, size_t)
KHASH_MAP_INIT_UINT8(uint8, size_t)
#endif /* __AC_KHASH_H */

View File

@@ -0,0 +1,450 @@
#include <string.h>
#include <Python.h>
typedef struct {
float real;
float imag;
} khcomplex64_t;
typedef struct {
double real;
double imag;
} khcomplex128_t;
// khash should report usage to tracemalloc
#if PY_VERSION_HEX >= 0x03060000
#include <pymem.h>
#if PY_VERSION_HEX < 0x03070000
#define PyTraceMalloc_Track _PyTraceMalloc_Track
#define PyTraceMalloc_Untrack _PyTraceMalloc_Untrack
#endif
#else
#define PyTraceMalloc_Track(...)
#define PyTraceMalloc_Untrack(...)
#endif
static const int KHASH_TRACE_DOMAIN = 424242;
void *traced_malloc(size_t size){
void * ptr = malloc(size);
if(ptr!=NULL){
PyTraceMalloc_Track(KHASH_TRACE_DOMAIN, (uintptr_t)ptr, size);
}
return ptr;
}
void *traced_calloc(size_t num, size_t size){
void * ptr = calloc(num, size);
if(ptr!=NULL){
PyTraceMalloc_Track(KHASH_TRACE_DOMAIN, (uintptr_t)ptr, num*size);
}
return ptr;
}
void *traced_realloc(void* old_ptr, size_t size){
void * ptr = realloc(old_ptr, size);
if(ptr!=NULL){
if(old_ptr != ptr){
PyTraceMalloc_Untrack(KHASH_TRACE_DOMAIN, (uintptr_t)old_ptr);
}
PyTraceMalloc_Track(KHASH_TRACE_DOMAIN, (uintptr_t)ptr, size);
}
return ptr;
}
void traced_free(void* ptr){
if(ptr!=NULL){
PyTraceMalloc_Untrack(KHASH_TRACE_DOMAIN, (uintptr_t)ptr);
}
free(ptr);
}
#define KHASH_MALLOC traced_malloc
#define KHASH_REALLOC traced_realloc
#define KHASH_CALLOC traced_calloc
#define KHASH_FREE traced_free
#include "khash.h"
// Previously we were using the built in cpython hash function for doubles
// python 2.7 https://github.com/python/cpython/blob/2.7/Objects/object.c#L1021
// python 3.5 https://github.com/python/cpython/blob/3.5/Python/pyhash.c#L85
// The python 3 hash function has the invariant hash(x) == hash(int(x)) == hash(decimal(x))
// and the size of hash may be different by platform / version (long in py2, Py_ssize_t in py3).
// We don't need those invariants because types will be cast before hashing, and if Py_ssize_t
// is 64 bits the truncation causes collision issues. Given all that, we use our own
// simple hash, viewing the double bytes as an int64 and using khash's default
// hash for 64 bit integers.
// GH 13436 showed that _Py_HashDouble doesn't work well with khash
// GH 28303 showed, that the simple xoring-version isn't good enough
// See GH 36729 for evaluation of the currently used murmur2-hash version
// An interesting alternative to expensive murmur2-hash would be to change
// the probing strategy and use e.g. the probing strategy from CPython's
// implementation of dicts, which shines for smaller sizes but is more
// predisposed to superlinear running times (see GH 36729 for comparison)
khuint64_t PANDAS_INLINE asuint64(double key) {
khuint64_t val;
memcpy(&val, &key, sizeof(double));
return val;
}
khuint32_t PANDAS_INLINE asuint32(float key) {
khuint32_t val;
memcpy(&val, &key, sizeof(float));
return val;
}
#define ZERO_HASH 0
#define NAN_HASH 0
khuint32_t PANDAS_INLINE kh_float64_hash_func(double val){
// 0.0 and -0.0 should have the same hash:
if (val == 0.0){
return ZERO_HASH;
}
// all nans should have the same hash:
if ( val!=val ){
return NAN_HASH;
}
khuint64_t as_int = asuint64(val);
return murmur2_64to32(as_int);
}
khuint32_t PANDAS_INLINE kh_float32_hash_func(float val){
// 0.0 and -0.0 should have the same hash:
if (val == 0.0f){
return ZERO_HASH;
}
// all nans should have the same hash:
if ( val!=val ){
return NAN_HASH;
}
khuint32_t as_int = asuint32(val);
return murmur2_32to32(as_int);
}
#define kh_floats_hash_equal(a, b) ((a) == (b) || ((b) != (b) && (a) != (a)))
#define KHASH_MAP_INIT_FLOAT64(name, khval_t) \
KHASH_INIT(name, khfloat64_t, khval_t, 1, kh_float64_hash_func, kh_floats_hash_equal)
KHASH_MAP_INIT_FLOAT64(float64, size_t)
#define KHASH_MAP_INIT_FLOAT32(name, khval_t) \
KHASH_INIT(name, khfloat32_t, khval_t, 1, kh_float32_hash_func, kh_floats_hash_equal)
KHASH_MAP_INIT_FLOAT32(float32, size_t)
khint32_t PANDAS_INLINE kh_complex128_hash_func(khcomplex128_t val){
return kh_float64_hash_func(val.real)^kh_float64_hash_func(val.imag);
}
khint32_t PANDAS_INLINE kh_complex64_hash_func(khcomplex64_t val){
return kh_float32_hash_func(val.real)^kh_float32_hash_func(val.imag);
}
#define kh_complex_hash_equal(a, b) \
(kh_floats_hash_equal(a.real, b.real) && kh_floats_hash_equal(a.imag, b.imag))
#define KHASH_MAP_INIT_COMPLEX64(name, khval_t) \
KHASH_INIT(name, khcomplex64_t, khval_t, 1, kh_complex64_hash_func, kh_complex_hash_equal)
KHASH_MAP_INIT_COMPLEX64(complex64, size_t)
#define KHASH_MAP_INIT_COMPLEX128(name, khval_t) \
KHASH_INIT(name, khcomplex128_t, khval_t, 1, kh_complex128_hash_func, kh_complex_hash_equal)
KHASH_MAP_INIT_COMPLEX128(complex128, size_t)
#define kh_exist_complex64(h, k) (kh_exist(h, k))
#define kh_exist_complex128(h, k) (kh_exist(h, k))
// NaN-floats should be in the same equivalency class, see GH 22119
int PANDAS_INLINE floatobject_cmp(PyFloatObject* a, PyFloatObject* b){
return (
Py_IS_NAN(PyFloat_AS_DOUBLE(a)) &&
Py_IS_NAN(PyFloat_AS_DOUBLE(b))
)
||
( PyFloat_AS_DOUBLE(a) == PyFloat_AS_DOUBLE(b) );
}
// NaNs should be in the same equivalency class, see GH 41836
// PyObject_RichCompareBool for complexobjects has a different behavior
// needs to be replaced
int PANDAS_INLINE complexobject_cmp(PyComplexObject* a, PyComplexObject* b){
return (
Py_IS_NAN(a->cval.real) &&
Py_IS_NAN(b->cval.real) &&
Py_IS_NAN(a->cval.imag) &&
Py_IS_NAN(b->cval.imag)
)
||
(
Py_IS_NAN(a->cval.real) &&
Py_IS_NAN(b->cval.real) &&
a->cval.imag == b->cval.imag
)
||
(
a->cval.real == b->cval.real &&
Py_IS_NAN(a->cval.imag) &&
Py_IS_NAN(b->cval.imag)
)
||
(
a->cval.real == b->cval.real &&
a->cval.imag == b->cval.imag
);
}
int PANDAS_INLINE pyobject_cmp(PyObject* a, PyObject* b);
// replacing PyObject_RichCompareBool (NaN!=NaN) with pyobject_cmp (NaN==NaN),
// which treats NaNs as equivalent
// see GH 41836
int PANDAS_INLINE tupleobject_cmp(PyTupleObject* a, PyTupleObject* b){
Py_ssize_t i;
if (Py_SIZE(a) != Py_SIZE(b)) {
return 0;
}
for (i = 0; i < Py_SIZE(a); ++i) {
if (!pyobject_cmp(PyTuple_GET_ITEM(a, i), PyTuple_GET_ITEM(b, i))) {
return 0;
}
}
return 1;
}
int PANDAS_INLINE pyobject_cmp(PyObject* a, PyObject* b) {
if (a == b) {
return 1;
}
if (Py_TYPE(a) == Py_TYPE(b)) {
// special handling for some built-in types which could have NaNs
// as we would like to have them equivalent, but the usual
// PyObject_RichCompareBool would return False
if (PyFloat_CheckExact(a)) {
return floatobject_cmp((PyFloatObject*)a, (PyFloatObject*)b);
}
if (PyComplex_CheckExact(a)) {
return complexobject_cmp((PyComplexObject*)a, (PyComplexObject*)b);
}
if (PyTuple_CheckExact(a)) {
return tupleobject_cmp((PyTupleObject*)a, (PyTupleObject*)b);
}
// frozenset isn't yet supported
}
int result = PyObject_RichCompareBool(a, b, Py_EQ);
if (result < 0) {
PyErr_Clear();
return 0;
}
return result;
}
Py_hash_t PANDAS_INLINE _Pandas_HashDouble(double val) {
//Since Python3.10, nan is no longer has hash 0
if (Py_IS_NAN(val)) {
return 0;
}
#if PY_VERSION_HEX < 0x030A0000
return _Py_HashDouble(val);
#else
return _Py_HashDouble(NULL, val);
#endif
}
Py_hash_t PANDAS_INLINE floatobject_hash(PyFloatObject* key) {
return _Pandas_HashDouble(PyFloat_AS_DOUBLE(key));
}
#define _PandasHASH_IMAG 1000003UL
// replaces _Py_HashDouble with _Pandas_HashDouble
Py_hash_t PANDAS_INLINE complexobject_hash(PyComplexObject* key) {
Py_uhash_t realhash = (Py_uhash_t)_Pandas_HashDouble(key->cval.real);
Py_uhash_t imaghash = (Py_uhash_t)_Pandas_HashDouble(key->cval.imag);
if (realhash == (Py_uhash_t)-1 || imaghash == (Py_uhash_t)-1) {
return -1;
}
Py_uhash_t combined = realhash + _PandasHASH_IMAG * imaghash;
if (combined == (Py_uhash_t)-1) {
return -2;
}
return (Py_hash_t)combined;
}
khuint32_t PANDAS_INLINE kh_python_hash_func(PyObject* key);
//we could use any hashing algorithm, this is the original CPython's for tuples
#if SIZEOF_PY_UHASH_T > 4
#define _PandasHASH_XXPRIME_1 ((Py_uhash_t)11400714785074694791ULL)
#define _PandasHASH_XXPRIME_2 ((Py_uhash_t)14029467366897019727ULL)
#define _PandasHASH_XXPRIME_5 ((Py_uhash_t)2870177450012600261ULL)
#define _PandasHASH_XXROTATE(x) ((x << 31) | (x >> 33)) /* Rotate left 31 bits */
#else
#define _PandasHASH_XXPRIME_1 ((Py_uhash_t)2654435761UL)
#define _PandasHASH_XXPRIME_2 ((Py_uhash_t)2246822519UL)
#define _PandasHASH_XXPRIME_5 ((Py_uhash_t)374761393UL)
#define _PandasHASH_XXROTATE(x) ((x << 13) | (x >> 19)) /* Rotate left 13 bits */
#endif
Py_hash_t PANDAS_INLINE tupleobject_hash(PyTupleObject* key) {
Py_ssize_t i, len = Py_SIZE(key);
PyObject **item = key->ob_item;
Py_uhash_t acc = _PandasHASH_XXPRIME_5;
for (i = 0; i < len; i++) {
Py_uhash_t lane = kh_python_hash_func(item[i]);
if (lane == (Py_uhash_t)-1) {
return -1;
}
acc += lane * _PandasHASH_XXPRIME_2;
acc = _PandasHASH_XXROTATE(acc);
acc *= _PandasHASH_XXPRIME_1;
}
/* Add input length, mangled to keep the historical value of hash(()). */
acc += len ^ (_PandasHASH_XXPRIME_5 ^ 3527539UL);
if (acc == (Py_uhash_t)-1) {
return 1546275796;
}
return acc;
}
khuint32_t PANDAS_INLINE kh_python_hash_func(PyObject* key) {
Py_hash_t hash;
// For PyObject_Hash holds:
// hash(0.0) == 0 == hash(-0.0)
// yet for different nan-objects different hash-values
// are possible
if (PyFloat_CheckExact(key)) {
// we cannot use kh_float64_hash_func
// because float(k) == k holds for any int-object k
// and kh_float64_hash_func doesn't respect it
hash = floatobject_hash((PyFloatObject*)key);
}
else if (PyComplex_CheckExact(key)) {
// we cannot use kh_complex128_hash_func
// because complex(k,0) == k holds for any int-object k
// and kh_complex128_hash_func doesn't respect it
hash = complexobject_hash((PyComplexObject*)key);
}
else if (PyTuple_CheckExact(key)) {
hash = tupleobject_hash((PyTupleObject*)key);
}
else {
hash = PyObject_Hash(key);
}
if (hash == -1) {
PyErr_Clear();
return 0;
}
#if SIZEOF_PY_HASH_T == 4
// it is already 32bit value
return hash;
#else
// for 64bit builds,
// we need information of the upper 32bits as well
// see GH 37615
khuint64_t as_uint = (khuint64_t) hash;
// uints avoid undefined behavior of signed ints
return (as_uint>>32)^as_uint;
#endif
}
#define kh_python_hash_equal(a, b) (pyobject_cmp(a, b))
// Python object
typedef PyObject* kh_pyobject_t;
#define KHASH_MAP_INIT_PYOBJECT(name, khval_t) \
KHASH_INIT(name, kh_pyobject_t, khval_t, 1, \
kh_python_hash_func, kh_python_hash_equal)
KHASH_MAP_INIT_PYOBJECT(pymap, Py_ssize_t)
#define KHASH_SET_INIT_PYOBJECT(name) \
KHASH_INIT(name, kh_pyobject_t, char, 0, \
kh_python_hash_func, kh_python_hash_equal)
KHASH_SET_INIT_PYOBJECT(pyset)
#define kh_exist_pymap(h, k) (kh_exist(h, k))
#define kh_exist_pyset(h, k) (kh_exist(h, k))
KHASH_MAP_INIT_STR(strbox, kh_pyobject_t)
typedef struct {
kh_str_t *table;
int starts[256];
} kh_str_starts_t;
typedef kh_str_starts_t* p_kh_str_starts_t;
p_kh_str_starts_t PANDAS_INLINE kh_init_str_starts(void) {
kh_str_starts_t *result = (kh_str_starts_t*)KHASH_CALLOC(1, sizeof(kh_str_starts_t));
result->table = kh_init_str();
return result;
}
khuint_t PANDAS_INLINE kh_put_str_starts_item(kh_str_starts_t* table, char* key, int* ret) {
khuint_t result = kh_put_str(table->table, key, ret);
if (*ret != 0) {
table->starts[(unsigned char)key[0]] = 1;
}
return result;
}
khuint_t PANDAS_INLINE kh_get_str_starts_item(const kh_str_starts_t* table, const char* key) {
unsigned char ch = *key;
if (table->starts[ch]) {
if (ch == '\0' || kh_get_str(table->table, key) != table->table->n_buckets) return 1;
}
return 0;
}
void PANDAS_INLINE kh_destroy_str_starts(kh_str_starts_t* table) {
kh_destroy_str(table->table);
KHASH_FREE(table);
}
void PANDAS_INLINE kh_resize_str_starts(kh_str_starts_t* table, khuint_t val) {
kh_resize_str(table->table, val);
}
// utility function: given the number of elements
// returns number of necessary buckets
khuint_t PANDAS_INLINE kh_needed_n_buckets(khuint_t n_elements){
khuint_t candidate = n_elements;
kroundup32(candidate);
khuint_t upper_bound = (khuint_t)(candidate * __ac_HASH_UPPER + 0.5);
return (upper_bound < n_elements) ? 2*candidate : candidate;
}

View File

@@ -0,0 +1,112 @@
/*
Copyright (c) 2016, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
The full license is in the LICENSE file, distributed with this software.
Copyright (c) 2005-2011, NumPy Developers
All rights reserved.
This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
*/
#pragma once
#ifndef NPY_NO_DEPRECATED_API
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#endif // NPY_NO_DEPRECATED_API
#include <numpy/ndarraytypes.h>
typedef struct {
npy_int64 days;
npy_int32 hrs, min, sec, ms, us, ns, seconds, microseconds, nanoseconds;
} pandas_timedeltastruct;
static const npy_datetimestruct _AS_MIN_DTS = {
1969, 12, 31, 23, 59, 50, 776627, 963145, 224193};
static const npy_datetimestruct _FS_MIN_DTS = {
1969, 12, 31, 21, 26, 16, 627963, 145224, 193000};
static const npy_datetimestruct _PS_MIN_DTS = {
1969, 9, 16, 5, 57, 7, 963145, 224193, 0};
static const npy_datetimestruct _NS_MIN_DTS = {
1677, 9, 21, 0, 12, 43, 145224, 193000, 0};
static const npy_datetimestruct _US_MIN_DTS = {
-290308, 12, 21, 19, 59, 05, 224193, 0, 0};
static const npy_datetimestruct _MS_MIN_DTS = {
-292275055, 5, 16, 16, 47, 4, 193000, 0, 0};
static const npy_datetimestruct _S_MIN_DTS = {
-292277022657, 1, 27, 8, 29, 53, 0, 0, 0};
static const npy_datetimestruct _M_MIN_DTS = {
-17536621475646, 5, 4, 5, 53, 0, 0, 0, 0};
static const npy_datetimestruct _AS_MAX_DTS = {
1970, 1, 1, 0, 0, 9, 223372, 36854, 775807};
static const npy_datetimestruct _FS_MAX_DTS = {
1970, 1, 1, 2, 33, 43, 372036, 854775, 807000};
static const npy_datetimestruct _PS_MAX_DTS = {
1970, 4, 17, 18, 2, 52, 36854, 775807, 0};
static const npy_datetimestruct _NS_MAX_DTS = {
2262, 4, 11, 23, 47, 16, 854775, 807000, 0};
static const npy_datetimestruct _US_MAX_DTS = {
294247, 1, 10, 4, 0, 54, 775807, 0, 0};
static const npy_datetimestruct _MS_MAX_DTS = {
292278994, 8, 17, 7, 12, 55, 807000, 0, 0};
static const npy_datetimestruct _S_MAX_DTS = {
292277026596, 12, 4, 15, 30, 7, 0, 0, 0};
static const npy_datetimestruct _M_MAX_DTS = {
17536621479585, 8, 30, 18, 7, 0, 0, 0, 0};
// stuff pandas needs
// ----------------------------------------------------------------------------
PyObject *extract_utc_offset(PyObject *obj);
npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT base,
const npy_datetimestruct *dts);
void pandas_datetime_to_datetimestruct(npy_datetime val, NPY_DATETIMEUNIT fr,
npy_datetimestruct *result);
void pandas_timedelta_to_timedeltastruct(npy_timedelta val,
NPY_DATETIMEUNIT fr,
pandas_timedeltastruct *result);
extern const int days_per_month_table[2][12];
// stuff numpy-derived code needs in header
// ----------------------------------------------------------------------------
int is_leapyear(npy_int64 year);
/*
* Calculates the days offset from the 1970 epoch.
*/
npy_int64
get_datetimestruct_days(const npy_datetimestruct *dts);
/*
* Compares two npy_datetimestruct objects chronologically
*/
int cmp_npy_datetimestruct(const npy_datetimestruct *a,
const npy_datetimestruct *b);
/*
* Adjusts a datetimestruct based on a minutes offset. Assumes
* the current values are valid.
*/
void
add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes);
/*
* This function returns the DateTimeMetaData
* contained within the provided datetime dtype.
*/
PyArray_DatetimeMetaData get_datetime_metadata_from_dtype(
PyArray_Descr *dtype);

View File

@@ -0,0 +1,109 @@
/*
Copyright (c) 2016, PyData Development Team
All rights reserved.
Distributed under the terms of the BSD Simplified License.
The full license is in the LICENSE file, distributed with this software.
Written by Mark Wiebe (mwwiebe@gmail.com)
Copyright (c) 2011 by Enthought, Inc.
Copyright (c) 2005-2011, NumPy Developers
All rights reserved.
See NUMPY_LICENSE.txt for the license.
This file implements string parsing and creation for NumPy datetime.
*/
#pragma once
#ifndef NPY_NO_DEPRECATED_API
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#endif // NPY_NO_DEPRECATED_API
/* 'format_requirement' can be one of three values:
* * PARTIAL_MATCH : Only require a partial match with 'format'.
* For example, if the string is '2020-01-01 05:00:00' and
* 'format' is '%Y-%m-%d', then parse '2020-01-01';
* * EXACT_MATCH : require an exact match with 'format'. If the
* string is '2020-01-01', then the only format which will
* be able to parse it without error is '%Y-%m-%d';
* * INFER_FORMAT: parse without comparing 'format' (i.e. infer it).
*/
typedef enum {
PARTIAL_MATCH,
EXACT_MATCH,
INFER_FORMAT
} FormatRequirement;
/*
* Parses (almost) standard ISO 8601 date strings. The differences are:
*
* + The date "20100312" is parsed as the year 20100312, not as
* equivalent to "2010-03-12". The '-' in the dates are not optional.
* + Only seconds may have a decimal point, with up to 18 digits after it
* (maximum attoseconds precision).
* + Either a 'T' as in ISO 8601 or a ' ' may be used to separate
* the date and the time. Both are treated equivalently.
* + Doesn't (yet) handle the "YYYY-DDD" or "YYYY-Www" formats.
* + Doesn't handle leap seconds (seconds value has 60 in these cases).
* + Doesn't handle 24:00:00 as synonym for midnight (00:00:00) tomorrow
* + Accepts special values "NaT" (not a time), "Today", (current
* day according to local time) and "Now" (current time in UTC).
*
* 'str' must be a NULL-terminated string, and 'len' must be its length.
*
* 'out' gets filled with the parsed date-time.
* 'out_local' gets whether returned value contains timezone. 0 for UTC, 1 for local time.
* 'out_tzoffset' gets set to timezone offset by minutes
* if the parsed time was in local time,
* to 0 otherwise. The values 'now' and 'today' don't get counted
* as local, and neither do UTC +/-#### timezone offsets, because
* they aren't using the computer's local timezone offset.
*
* Returns 0 on success, -1 on failure.
*/
int
parse_iso_8601_datetime(const char *str, int len, int want_exc,
npy_datetimestruct *out,
NPY_DATETIMEUNIT *out_bestunit,
int *out_local,
int *out_tzoffset,
const char* format,
int format_len,
FormatRequirement format_requirement);
/*
* Provides a string length to use for converting datetime
* objects with the given local and unit settings.
*/
int
get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base);
/*
* Converts an npy_datetimestruct to an (almost) ISO 8601
* NULL-terminated string using timezone Z (UTC).
*
* 'base' restricts the output to that unit. Set 'base' to
* -1 to auto-detect a base after which all the values are zero.
*
* Returns 0 on success, -1 on failure (for example if the output
* string was too short).
*/
int
make_iso_8601_datetime(npy_datetimestruct *dts, char *outstr, int outlen,
int utc, NPY_DATETIMEUNIT base);
/*
* Converts an pandas_timedeltastruct to an ISO 8601 string.
*
* Mutates outlen to provide size of (non-NULL terminated) string.
*
* Currently has no error handling
*/
int make_iso_8601_timedelta(pandas_timedeltastruct *tds, char *outstr,
size_t *outlen);

View File

@@ -0,0 +1,314 @@
/*
Copyright (c) 2011-2013, ESN Social Software AB and Jonas Tarnstrom
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the ESN Social Software AB nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL ESN SOCIAL SOFTWARE AB OR JONAS TARNSTROM BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
https://github.com/client9/stringencoders
Copyright (c) 2007 Nick Galbreath -- nickg [at] modp [dot] com. All rights reserved.
Numeric decoder derived from TCL library
https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
* Copyright (c) 1988-1993 The Regents of the University of California.
* Copyright (c) 1994 Sun Microsystems, Inc.
*/
/*
Ultra fast JSON encoder and decoder
Developed by Jonas Tarnstrom (jonas@esn.me).
Encoder notes:
------------------
:: Cyclic references ::
Cyclic referenced objects are not detected.
Set JSONObjectEncoder.recursionMax to suitable value or make sure input object
tree doesn't have cyclic references.
*/
#pragma once
#include <stdio.h>
#include <wchar.h>
#include "pandas/portable.h"
// Don't output any extra whitespaces when encoding
#define JSON_NO_EXTRA_WHITESPACE
// Max decimals to encode double floating point numbers with
#ifndef JSON_DOUBLE_MAX_DECIMALS
#define JSON_DOUBLE_MAX_DECIMALS 15
#endif
// Max recursion depth, default for encoder
#ifndef JSON_MAX_RECURSION_DEPTH
#define JSON_MAX_RECURSION_DEPTH 1024
#endif
// Max recursion depth, default for decoder
#ifndef JSON_MAX_OBJECT_DEPTH
#define JSON_MAX_OBJECT_DEPTH 1024
#endif
/*
Dictates and limits how much stack space for buffers UltraJSON will use before resorting to provided heap functions */
#ifndef JSON_MAX_STACK_BUFFER_SIZE
#define JSON_MAX_STACK_BUFFER_SIZE 131072
#endif
#ifdef _WIN32
typedef __int64 JSINT64;
typedef unsigned __int64 JSUINT64;
typedef __int32 JSINT32;
typedef unsigned __int32 JSUINT32;
typedef unsigned __int8 JSUINT8;
typedef unsigned __int16 JSUTF16;
typedef unsigned __int32 JSUTF32;
typedef __int64 JSLONG;
#define EXPORTFUNCTION __declspec(dllexport)
#define FASTCALL_MSVC __fastcall
#define INLINE_PREFIX static __inline
#else
#include <stdint.h>
typedef int64_t JSINT64;
typedef uint64_t JSUINT64;
typedef int32_t JSINT32;
typedef uint32_t JSUINT32;
#define FASTCALL_MSVC
#define INLINE_PREFIX static inline
typedef uint8_t JSUINT8;
typedef uint16_t JSUTF16;
typedef uint32_t JSUTF32;
typedef int64_t JSLONG;
#define EXPORTFUNCTION
#endif
#if !(defined(__LITTLE_ENDIAN__) || defined(__BIG_ENDIAN__))
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define __LITTLE_ENDIAN__
#else
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define __BIG_ENDIAN__
#endif
#endif
#endif
#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
#error "Endianness not supported"
#endif
enum JSTYPES {
JT_NULL, // NULL
JT_TRUE, // boolean true
JT_FALSE, // boolean false
JT_INT, // (JSINT32 (signed 32-bit))
JT_LONG, // (JSINT64 (signed 64-bit))
JT_DOUBLE, // (double)
JT_BIGNUM, // integer larger than sys.maxsize
JT_UTF8, // (char 8-bit)
JT_ARRAY, // Array structure
JT_OBJECT, // Key/Value structure
JT_INVALID, // Internal, do not return nor expect
JT_POS_INF, // Positive infinity
JT_NEG_INF, // Negative infinity
};
typedef void * JSOBJ;
typedef void * JSITER;
typedef struct __JSONTypeContext {
int type;
void *encoder;
void *prv;
} JSONTypeContext;
/*
Function pointer declarations, suitable for implementing UltraJSON */
typedef void (*JSPFN_ITERBEGIN)(JSOBJ obj, JSONTypeContext *tc);
typedef int (*JSPFN_ITERNEXT)(JSOBJ obj, JSONTypeContext *tc);
typedef void (*JSPFN_ITEREND)(JSOBJ obj, JSONTypeContext *tc);
typedef JSOBJ (*JSPFN_ITERGETVALUE)(JSOBJ obj, JSONTypeContext *tc);
typedef char *(*JSPFN_ITERGETNAME)(JSOBJ obj, JSONTypeContext *tc,
size_t *outLen);
typedef void *(*JSPFN_MALLOC)(size_t size);
typedef void (*JSPFN_FREE)(void *pptr);
typedef void *(*JSPFN_REALLOC)(void *base, size_t size);
typedef struct __JSONObjectEncoder {
void (*beginTypeContext)(JSOBJ obj, JSONTypeContext *tc);
void (*endTypeContext)(JSOBJ obj, JSONTypeContext *tc);
const char *(*getStringValue)(JSOBJ obj, JSONTypeContext *tc,
size_t *_outLen);
JSINT64 (*getLongValue)(JSOBJ obj, JSONTypeContext *tc);
JSINT32 (*getIntValue)(JSOBJ obj, JSONTypeContext *tc);
double (*getDoubleValue)(JSOBJ obj, JSONTypeContext *tc);
const char *(*getBigNumStringValue)(JSOBJ obj, JSONTypeContext *tc,
size_t *_outLen);
/*
Begin iteration of an iterable object (JS_ARRAY or JS_OBJECT)
Implementor should setup iteration state in ti->prv
*/
JSPFN_ITERBEGIN iterBegin;
/*
Retrieve next object in an iteration. Should return 0 to indicate iteration has reached end or 1 if there are more items.
Implementor is responsible for keeping state of the iteration. Use ti->prv fields for this
*/
JSPFN_ITERNEXT iterNext;
/*
Ends the iteration of an iterable object.
Any iteration state stored in ti->prv can be freed here
*/
JSPFN_ITEREND iterEnd;
/*
Returns a reference to the value object of an iterator
The is responsible for the life-cycle of the returned string. Use iterNext/iterEnd and ti->prv to keep track of current object
*/
JSPFN_ITERGETVALUE iterGetValue;
/*
Return name of iterator.
The is responsible for the life-cycle of the returned string. Use iterNext/iterEnd and ti->prv to keep track of current object
*/
JSPFN_ITERGETNAME iterGetName;
/*
Release a value as indicated by setting ti->release = 1 in the previous getValue call.
The ti->prv array should contain the necessary context to release the value
*/
void (*releaseObject)(JSOBJ obj);
/* Library functions
Set to NULL to use STDLIB malloc,realloc,free */
JSPFN_MALLOC malloc;
JSPFN_REALLOC realloc;
JSPFN_FREE free;
/*
Configuration for max recursion, set to 0 to use default (see JSON_MAX_RECURSION_DEPTH)*/
int recursionMax;
/*
Configuration for max decimals of double floating point numbers to encode (0-9) */
int doublePrecision;
/*
If true output will be ASCII with all characters above 127 encoded as \uXXXX. If false output will be UTF-8 or what ever charset strings are brought as */
int forceASCII;
/*
If true, '<', '>', and '&' characters will be encoded as \u003c, \u003e, and \u0026, respectively. If false, no special encoding will be used. */
int encodeHTMLChars;
/*
Configuration for spaces of indent */
int indent;
/*
Set to an error message if error occurred */
const char *errorMsg;
JSOBJ errorObj;
/* Buffer stuff */
char *start;
char *offset;
char *end;
int heap;
int level;
} JSONObjectEncoder;
/*
Encode an object structure into JSON.
Arguments:
obj - An anonymous type representing the object
enc - Function definitions for querying JSOBJ type
buffer - Preallocated buffer to store result in. If NULL function allocates own buffer
cbBuffer - Length of buffer (ignored if buffer is NULL)
Returns:
Encoded JSON object as a null terminated char string.
NOTE:
If the supplied buffer wasn't enough to hold the result the function will allocate a new buffer.
Life cycle of the provided buffer must still be handled by caller.
If the return value doesn't equal the specified buffer caller must release the memory using
JSONObjectEncoder.free or free() as specified when calling this function.
*/
EXPORTFUNCTION char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc,
char *buffer, size_t cbBuffer);
typedef struct __JSONObjectDecoder {
JSOBJ (*newString)(void *prv, wchar_t *start, wchar_t *end);
int (*objectAddKey)(void *prv, JSOBJ obj, JSOBJ name, JSOBJ value);
int (*arrayAddItem)(void *prv, JSOBJ obj, JSOBJ value);
JSOBJ (*newTrue)(void *prv);
JSOBJ (*newFalse)(void *prv);
JSOBJ (*newNull)(void *prv);
JSOBJ (*newPosInf)(void *prv);
JSOBJ (*newNegInf)(void *prv);
JSOBJ (*newObject)(void *prv, void *decoder);
JSOBJ (*endObject)(void *prv, JSOBJ obj);
JSOBJ (*newArray)(void *prv, void *decoder);
JSOBJ (*endArray)(void *prv, JSOBJ obj);
JSOBJ (*newInt)(void *prv, JSINT32 value);
JSOBJ (*newLong)(void *prv, JSINT64 value);
JSOBJ (*newUnsignedLong)(void *prv, JSUINT64 value);
JSOBJ (*newDouble)(void *prv, double value);
void (*releaseObject)(void *prv, JSOBJ obj, void *decoder);
JSPFN_MALLOC malloc;
JSPFN_FREE free;
JSPFN_REALLOC realloc;
char *errorStr;
char *errorOffset;
int preciseFloat;
void *prv;
} JSONObjectDecoder;
EXPORTFUNCTION JSOBJ JSON_DecodeObject(JSONObjectDecoder *dec,
const char *buffer, size_t cbBuffer);
EXPORTFUNCTION void encode(JSOBJ, JSONObjectEncoder *, const char *, size_t);

View File

@@ -0,0 +1,40 @@
/*
Copyright (c) 2011-2013, ESN Social Software AB and Jonas Tarnstrom
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the ESN Social Software AB nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL ESN SOCIAL SOFTWARE AB OR JONAS TARNSTROM BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
https://github.com/client9/stringencoders
Copyright (c) 2007 Nick Galbreath -- nickg [at] modp [dot] com. All rights reserved.
Numeric decoder derived from TCL library
https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
* Copyright (c) 1988-1993 The Regents of the University of California.
* Copyright (c) 1994 Sun Microsystems, Inc.
*/
#pragma once
#define UJSON_VERSION "1.33"