You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
465 lines
14 KiB
465 lines
14 KiB
1 year ago
|
import operator
|
||
|
|
import re
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from pandas import option_context
|
||
|
|
import pandas._testing as tm
|
||
|
|
from pandas.core.api import (
|
||
|
|
DataFrame,
|
||
|
|
Index,
|
||
|
|
Series,
|
||
|
|
)
|
||
|
|
from pandas.core.computation import expressions as expr
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _frame():
|
||
|
|
return DataFrame(
|
||
|
|
np.random.default_rng(2).standard_normal((10001, 4)),
|
||
|
|
columns=list("ABCD"),
|
||
|
|
dtype="float64",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _frame2():
|
||
|
|
return DataFrame(
|
||
|
|
np.random.default_rng(2).standard_normal((100, 4)),
|
||
|
|
columns=list("ABCD"),
|
||
|
|
dtype="float64",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _mixed(_frame):
|
||
|
|
return DataFrame(
|
||
|
|
{
|
||
|
|
"A": _frame["A"].copy(),
|
||
|
|
"B": _frame["B"].astype("float32"),
|
||
|
|
"C": _frame["C"].astype("int64"),
|
||
|
|
"D": _frame["D"].astype("int32"),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _mixed2(_frame2):
|
||
|
|
return DataFrame(
|
||
|
|
{
|
||
|
|
"A": _frame2["A"].copy(),
|
||
|
|
"B": _frame2["B"].astype("float32"),
|
||
|
|
"C": _frame2["C"].astype("int64"),
|
||
|
|
"D": _frame2["D"].astype("int32"),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _integer():
|
||
|
|
return DataFrame(
|
||
|
|
np.random.default_rng(2).integers(1, 100, size=(10001, 4)),
|
||
|
|
columns=list("ABCD"),
|
||
|
|
dtype="int64",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _integer_integers(_integer):
|
||
|
|
# integers to get a case with zeros
|
||
|
|
return _integer * np.random.default_rng(2).integers(0, 2, size=np.shape(_integer))
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _integer2():
|
||
|
|
return DataFrame(
|
||
|
|
np.random.default_rng(2).integers(1, 100, size=(101, 4)),
|
||
|
|
columns=list("ABCD"),
|
||
|
|
dtype="int64",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _array(_frame):
|
||
|
|
return _frame["A"].values.copy()
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _array2(_frame2):
|
||
|
|
return _frame2["A"].values.copy()
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _array_mixed(_mixed):
|
||
|
|
return _mixed["D"].values.copy()
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def _array_mixed2(_mixed2):
|
||
|
|
return _mixed2["D"].values.copy()
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.skipif(not expr.USE_NUMEXPR, reason="not using numexpr")
|
||
|
|
class TestExpressions:
|
||
|
|
@pytest.fixture(autouse=True)
|
||
|
|
def save_min_elements(self):
|
||
|
|
min_elements = expr._MIN_ELEMENTS
|
||
|
|
yield
|
||
|
|
expr._MIN_ELEMENTS = min_elements
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def call_op(df, other, flex: bool, opname: str):
|
||
|
|
if flex:
|
||
|
|
op = lambda x, y: getattr(x, opname)(y)
|
||
|
|
op.__name__ = opname
|
||
|
|
else:
|
||
|
|
op = getattr(operator, opname)
|
||
|
|
|
||
|
|
with option_context("compute.use_numexpr", False):
|
||
|
|
expected = op(df, other)
|
||
|
|
|
||
|
|
expr.get_test_result()
|
||
|
|
|
||
|
|
result = op(df, other)
|
||
|
|
return result, expected
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"fixture",
|
||
|
|
[
|
||
|
|
"_integer",
|
||
|
|
"_integer2",
|
||
|
|
"_integer_integers",
|
||
|
|
"_frame",
|
||
|
|
"_frame2",
|
||
|
|
"_mixed",
|
||
|
|
"_mixed2",
|
||
|
|
],
|
||
|
|
)
|
||
|
|
@pytest.mark.parametrize("flex", [True, False])
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"arith", ["add", "sub", "mul", "mod", "truediv", "floordiv"]
|
||
|
|
)
|
||
|
|
def test_run_arithmetic(self, request, fixture, flex, arith):
|
||
|
|
df = request.getfixturevalue(fixture)
|
||
|
|
expr._MIN_ELEMENTS = 0
|
||
|
|
result, expected = self.call_op(df, df, flex, arith)
|
||
|
|
|
||
|
|
if arith == "truediv":
|
||
|
|
assert all(x.kind == "f" for x in expected.dtypes.values)
|
||
|
|
tm.assert_equal(expected, result)
|
||
|
|
|
||
|
|
for i in range(len(df.columns)):
|
||
|
|
result, expected = self.call_op(df.iloc[:, i], df.iloc[:, i], flex, arith)
|
||
|
|
if arith == "truediv":
|
||
|
|
assert expected.dtype.kind == "f"
|
||
|
|
tm.assert_equal(expected, result)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"fixture",
|
||
|
|
[
|
||
|
|
"_integer",
|
||
|
|
"_integer2",
|
||
|
|
"_integer_integers",
|
||
|
|
"_frame",
|
||
|
|
"_frame2",
|
||
|
|
"_mixed",
|
||
|
|
"_mixed2",
|
||
|
|
],
|
||
|
|
)
|
||
|
|
@pytest.mark.parametrize("flex", [True, False])
|
||
|
|
def test_run_binary(self, request, fixture, flex, comparison_op):
|
||
|
|
"""
|
||
|
|
tests solely that the result is the same whether or not numexpr is
|
||
|
|
enabled. Need to test whether the function does the correct thing
|
||
|
|
elsewhere.
|
||
|
|
"""
|
||
|
|
df = request.getfixturevalue(fixture)
|
||
|
|
arith = comparison_op.__name__
|
||
|
|
with option_context("compute.use_numexpr", False):
|
||
|
|
other = df.copy() + 1
|
||
|
|
|
||
|
|
expr._MIN_ELEMENTS = 0
|
||
|
|
expr.set_test_mode(True)
|
||
|
|
|
||
|
|
result, expected = self.call_op(df, other, flex, arith)
|
||
|
|
|
||
|
|
used_numexpr = expr.get_test_result()
|
||
|
|
assert used_numexpr, "Did not use numexpr as expected."
|
||
|
|
tm.assert_equal(expected, result)
|
||
|
|
|
||
|
|
for i in range(len(df.columns)):
|
||
|
|
binary_comp = other.iloc[:, i] + 1
|
||
|
|
self.call_op(df.iloc[:, i], binary_comp, flex, "add")
|
||
|
|
|
||
|
|
def test_invalid(self):
|
||
|
|
array = np.random.default_rng(2).standard_normal(1_000_001)
|
||
|
|
array2 = np.random.default_rng(2).standard_normal(100)
|
||
|
|
|
||
|
|
# no op
|
||
|
|
result = expr._can_use_numexpr(operator.add, None, array, array, "evaluate")
|
||
|
|
assert not result
|
||
|
|
|
||
|
|
# min elements
|
||
|
|
result = expr._can_use_numexpr(operator.add, "+", array2, array2, "evaluate")
|
||
|
|
assert not result
|
||
|
|
|
||
|
|
# ok, we only check on first part of expression
|
||
|
|
result = expr._can_use_numexpr(operator.add, "+", array, array2, "evaluate")
|
||
|
|
assert result
|
||
|
|
|
||
|
|
@pytest.mark.filterwarnings("ignore:invalid value encountered in:RuntimeWarning")
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"opname,op_str",
|
||
|
|
[("add", "+"), ("sub", "-"), ("mul", "*"), ("truediv", "/"), ("pow", "**")],
|
||
|
|
)
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"left_fix,right_fix", [("_array", "_array2"), ("_array_mixed", "_array_mixed2")]
|
||
|
|
)
|
||
|
|
def test_binary_ops(self, request, opname, op_str, left_fix, right_fix):
|
||
|
|
left = request.getfixturevalue(left_fix)
|
||
|
|
right = request.getfixturevalue(right_fix)
|
||
|
|
|
||
|
|
def testit(left, right, opname, op_str):
|
||
|
|
if opname == "pow":
|
||
|
|
left = np.abs(left)
|
||
|
|
|
||
|
|
op = getattr(operator, opname)
|
||
|
|
|
||
|
|
# array has 0s
|
||
|
|
result = expr.evaluate(op, left, left, use_numexpr=True)
|
||
|
|
expected = expr.evaluate(op, left, left, use_numexpr=False)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = expr._can_use_numexpr(op, op_str, right, right, "evaluate")
|
||
|
|
assert not result
|
||
|
|
|
||
|
|
with option_context("compute.use_numexpr", False):
|
||
|
|
testit(left, right, opname, op_str)
|
||
|
|
|
||
|
|
expr.set_numexpr_threads(1)
|
||
|
|
testit(left, right, opname, op_str)
|
||
|
|
expr.set_numexpr_threads()
|
||
|
|
testit(left, right, opname, op_str)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"left_fix,right_fix", [("_array", "_array2"), ("_array_mixed", "_array_mixed2")]
|
||
|
|
)
|
||
|
|
def test_comparison_ops(self, request, comparison_op, left_fix, right_fix):
|
||
|
|
left = request.getfixturevalue(left_fix)
|
||
|
|
right = request.getfixturevalue(right_fix)
|
||
|
|
|
||
|
|
def testit():
|
||
|
|
f12 = left + 1
|
||
|
|
f22 = right + 1
|
||
|
|
|
||
|
|
op = comparison_op
|
||
|
|
|
||
|
|
result = expr.evaluate(op, left, f12, use_numexpr=True)
|
||
|
|
expected = expr.evaluate(op, left, f12, use_numexpr=False)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = expr._can_use_numexpr(op, op, right, f22, "evaluate")
|
||
|
|
assert not result
|
||
|
|
|
||
|
|
with option_context("compute.use_numexpr", False):
|
||
|
|
testit()
|
||
|
|
|
||
|
|
expr.set_numexpr_threads(1)
|
||
|
|
testit()
|
||
|
|
expr.set_numexpr_threads()
|
||
|
|
testit()
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("cond", [True, False])
|
||
|
|
@pytest.mark.parametrize("fixture", ["_frame", "_frame2", "_mixed", "_mixed2"])
|
||
|
|
def test_where(self, request, cond, fixture):
|
||
|
|
df = request.getfixturevalue(fixture)
|
||
|
|
|
||
|
|
def testit():
|
||
|
|
c = np.empty(df.shape, dtype=np.bool_)
|
||
|
|
c.fill(cond)
|
||
|
|
result = expr.where(c, df.values, df.values + 1)
|
||
|
|
expected = np.where(c, df.values, df.values + 1)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
with option_context("compute.use_numexpr", False):
|
||
|
|
testit()
|
||
|
|
|
||
|
|
expr.set_numexpr_threads(1)
|
||
|
|
testit()
|
||
|
|
expr.set_numexpr_threads()
|
||
|
|
testit()
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"op_str,opname", [("/", "truediv"), ("//", "floordiv"), ("**", "pow")]
|
||
|
|
)
|
||
|
|
def test_bool_ops_raise_on_arithmetic(self, op_str, opname):
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"a": np.random.default_rng(2).random(10) > 0.5,
|
||
|
|
"b": np.random.default_rng(2).random(10) > 0.5,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
msg = f"operator '{opname}' not implemented for bool dtypes"
|
||
|
|
f = getattr(operator, opname)
|
||
|
|
err_msg = re.escape(msg)
|
||
|
|
|
||
|
|
with pytest.raises(NotImplementedError, match=err_msg):
|
||
|
|
f(df, df)
|
||
|
|
|
||
|
|
with pytest.raises(NotImplementedError, match=err_msg):
|
||
|
|
f(df.a, df.b)
|
||
|
|
|
||
|
|
with pytest.raises(NotImplementedError, match=err_msg):
|
||
|
|
f(df.a, True)
|
||
|
|
|
||
|
|
with pytest.raises(NotImplementedError, match=err_msg):
|
||
|
|
f(False, df.a)
|
||
|
|
|
||
|
|
with pytest.raises(NotImplementedError, match=err_msg):
|
||
|
|
f(False, df)
|
||
|
|
|
||
|
|
with pytest.raises(NotImplementedError, match=err_msg):
|
||
|
|
f(df, True)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"op_str,opname", [("+", "add"), ("*", "mul"), ("-", "sub")]
|
||
|
|
)
|
||
|
|
def test_bool_ops_warn_on_arithmetic(self, op_str, opname):
|
||
|
|
n = 10
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"a": np.random.default_rng(2).random(n) > 0.5,
|
||
|
|
"b": np.random.default_rng(2).random(n) > 0.5,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
subs = {"+": "|", "*": "&", "-": "^"}
|
||
|
|
sub_funcs = {"|": "or_", "&": "and_", "^": "xor"}
|
||
|
|
|
||
|
|
f = getattr(operator, opname)
|
||
|
|
fe = getattr(operator, sub_funcs[subs[op_str]])
|
||
|
|
|
||
|
|
if op_str == "-":
|
||
|
|
# raises TypeError
|
||
|
|
return
|
||
|
|
|
||
|
|
with tm.use_numexpr(True, min_elements=5):
|
||
|
|
with tm.assert_produces_warning():
|
||
|
|
r = f(df, df)
|
||
|
|
e = fe(df, df)
|
||
|
|
tm.assert_frame_equal(r, e)
|
||
|
|
|
||
|
|
with tm.assert_produces_warning():
|
||
|
|
r = f(df.a, df.b)
|
||
|
|
e = fe(df.a, df.b)
|
||
|
|
tm.assert_series_equal(r, e)
|
||
|
|
|
||
|
|
with tm.assert_produces_warning():
|
||
|
|
r = f(df.a, True)
|
||
|
|
e = fe(df.a, True)
|
||
|
|
tm.assert_series_equal(r, e)
|
||
|
|
|
||
|
|
with tm.assert_produces_warning():
|
||
|
|
r = f(False, df.a)
|
||
|
|
e = fe(False, df.a)
|
||
|
|
tm.assert_series_equal(r, e)
|
||
|
|
|
||
|
|
with tm.assert_produces_warning():
|
||
|
|
r = f(False, df)
|
||
|
|
e = fe(False, df)
|
||
|
|
tm.assert_frame_equal(r, e)
|
||
|
|
|
||
|
|
with tm.assert_produces_warning():
|
||
|
|
r = f(df, True)
|
||
|
|
e = fe(df, True)
|
||
|
|
tm.assert_frame_equal(r, e)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"test_input,expected",
|
||
|
|
[
|
||
|
|
(
|
||
|
|
DataFrame(
|
||
|
|
[[0, 1, 2, "aa"], [0, 1, 2, "aa"]], columns=["a", "b", "c", "dtype"]
|
||
|
|
),
|
||
|
|
DataFrame([[False, False], [False, False]], columns=["a", "dtype"]),
|
||
|
|
),
|
||
|
|
(
|
||
|
|
DataFrame(
|
||
|
|
[[0, 3, 2, "aa"], [0, 4, 2, "aa"], [0, 1, 1, "bb"]],
|
||
|
|
columns=["a", "b", "c", "dtype"],
|
||
|
|
),
|
||
|
|
DataFrame(
|
||
|
|
[[False, False], [False, False], [False, False]],
|
||
|
|
columns=["a", "dtype"],
|
||
|
|
),
|
||
|
|
),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_bool_ops_column_name_dtype(self, test_input, expected):
|
||
|
|
# GH 22383 - .ne fails if columns containing column name 'dtype'
|
||
|
|
result = test_input.loc[:, ["a", "dtype"]].ne(test_input.loc[:, ["a", "dtype"]])
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"arith", ("add", "sub", "mul", "mod", "truediv", "floordiv")
|
||
|
|
)
|
||
|
|
@pytest.mark.parametrize("axis", (0, 1))
|
||
|
|
def test_frame_series_axis(self, axis, arith, _frame):
|
||
|
|
# GH#26736 Dataframe.floordiv(Series, axis=1) fails
|
||
|
|
|
||
|
|
df = _frame
|
||
|
|
if axis == 1:
|
||
|
|
other = df.iloc[0, :]
|
||
|
|
else:
|
||
|
|
other = df.iloc[:, 0]
|
||
|
|
|
||
|
|
expr._MIN_ELEMENTS = 0
|
||
|
|
|
||
|
|
op_func = getattr(df, arith)
|
||
|
|
|
||
|
|
with option_context("compute.use_numexpr", False):
|
||
|
|
expected = op_func(other, axis=axis)
|
||
|
|
|
||
|
|
result = op_func(other, axis=axis)
|
||
|
|
tm.assert_frame_equal(expected, result)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"op",
|
||
|
|
[
|
||
|
|
"__mod__",
|
||
|
|
"__rmod__",
|
||
|
|
"__floordiv__",
|
||
|
|
"__rfloordiv__",
|
||
|
|
],
|
||
|
|
)
|
||
|
|
@pytest.mark.parametrize("box", [DataFrame, Series, Index])
|
||
|
|
@pytest.mark.parametrize("scalar", [-5, 5])
|
||
|
|
def test_python_semantics_with_numexpr_installed(self, op, box, scalar):
|
||
|
|
# https://github.com/pandas-dev/pandas/issues/36047
|
||
|
|
expr._MIN_ELEMENTS = 0
|
||
|
|
data = np.arange(-50, 50)
|
||
|
|
obj = box(data)
|
||
|
|
method = getattr(obj, op)
|
||
|
|
result = method(scalar)
|
||
|
|
|
||
|
|
# compare result with numpy
|
||
|
|
with option_context("compute.use_numexpr", False):
|
||
|
|
expected = method(scalar)
|
||
|
|
|
||
|
|
tm.assert_equal(result, expected)
|
||
|
|
|
||
|
|
# compare result element-wise with Python
|
||
|
|
for i, elem in enumerate(data):
|
||
|
|
if box == DataFrame:
|
||
|
|
scalar_result = result.iloc[i, 0]
|
||
|
|
else:
|
||
|
|
scalar_result = result[i]
|
||
|
|
try:
|
||
|
|
expected = getattr(int(elem), op)(scalar)
|
||
|
|
except ZeroDivisionError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
assert scalar_result == expected
|