numdaplot 0.1.0__py3-none-any.whl

numdaplot/tables/frames/frame_normalization.py

@@ -0,0 +1,459 @@
+"""
+Normalization of a pandas DataFrame into the storage of a Table.
+
+.. versionadded:: 0.1.0
+"""
+
+from datetime import datetime
+from typing import Any, cast
+
+import numpy
+import pandas
+from pandas.api.types import infer_dtype
+
+from numdaplot.data_types.numpy_mapping import from_numpy_dtype
+from numdaplot.data_types.pandas_mapping import TYPES_BY_PANDAS_NAME, from_pandas_dtype, to_pandas_dtype
+from numdaplot.data_types.time_of_day import holds_only_times
+from numdaplot.data_types.type import Type
+from numdaplot.errors.unsupported_index_error import UnsupportedIndexError
+from numdaplot.tables.columns.column_names import require_column_names
+from numdaplot.tables.frames.column import Column
+from numdaplot.tables.frames.time_columns import is_time_column, read_object_values, store_time_column
+from numdaplot.tables.frames.time_range_errors import TimeRangeErrors
+from numdaplot.tables.frames.time_zones import normalize_time_zone_column
+
+INFERRED_TYPES = {
+	"integer": Type.INT64,
+	"floating": Type.FLOAT64,
+	"mixed-integer-float": Type.FLOAT64,
+	"decimal": Type.FLOAT64,
+	"boolean": Type.BOOL,
+	"timedelta": Type.DURATION,
+}
+MOMENT_KINDS = frozenset({"date", "datetime"})
+INTEGER_KINDS = frozenset({"i", "u"})
+BOOLEAN_KIND = "b"
+DECIMAL_KIND = "f"
+DATETIME_KIND = "M"
+DURATION_KIND = "m"
+TEXT_INFERRED_KINDS = frozenset({"string", "empty"})
+DATETIME_DTYPE = "datetime64[ns]"
+DURATION_DTYPE = "timedelta64[ns]"
+PYTHON_OBJECT_DTYPE = numpy.dtype(object)
+TABLE_CREATION_OPERATION = "Table creation"
+
+
+def normalize_frame(frame: pandas.DataFrame) -> pandas.DataFrame:
+	"""
+	Return a DataFrame stored the way a Table stores its columns.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	frame
+		Any DataFrame, read from a file or given by the user. It is not changed.
+
+	Returns
+	-------
+	pandas.DataFrame
+		A new DataFrame numbered from 0, whose columns use the dtypes given by ``to_pandas_dtype``.
+
+	Raises
+	------
+	InvalidColumnNameError
+		When a column name is not a non-empty text.
+	DuplicateColumnError
+		When a column name appears twice.
+	UnsupportedIndexError
+		When the index holds anything else than the row numbers 0, 1, 2...
+	UnsupportedTypeError
+		When a column has a type numdaplot does not support, such as complex numbers.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import normalize_frame
+	>>> normalize_frame(pandas.DataFrame({"count": [1, 2]})).dtypes.tolist()
+	[Int64Dtype()]
+	"""
+	raw_names = list(frame.columns)
+	require_column_names(raw_names)
+	require_plain_index(frame.index)
+	column_names = [str(raw_name) for raw_name in raw_names]
+	normalized_columns = dict[str, Column]()
+
+	for column_name in column_names:
+		column = frame[column_name]
+		with TimeRangeErrors(TABLE_CREATION_OPERATION, column_name):
+			normalized_column = normalize_column(column)
+
+		normalized_columns[column_name] = normalized_column
+
+	row_count = len(frame)
+	row_numbers = pandas.RangeIndex(row_count)
+	normalized_frame = pandas.DataFrame(normalized_columns, index=row_numbers)
+
+	return normalized_frame
+
+
+def require_plain_index(index: pandas.Index) -> None:
+	"""
+	Check that an index only holds the row numbers 0, 1, 2...
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	index
+		The index of a DataFrame.
+
+	Raises
+	------
+	UnsupportedIndexError
+		When the index holds other values, such as city names.
+
+	Examples
+	--------
+	>>> import numdaplot as ndp
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import require_plain_index
+	>>> try:
+	... 	require_plain_index(pandas.Index(["Paris", "Lyon"]))
+	... except ndp.UnsupportedIndexError as error:
+	... 	print(error)
+	A Table has no index, call reset_index() on the pandas DataFrame first to keep its index as a column.
+	"""
+	row_count = len(index)
+	row_numbers = pandas.RangeIndex(row_count)
+	is_plain = index.equals(row_numbers)
+	if not is_plain:
+		raise UnsupportedIndexError()
+
+
+def normalize_column(column: Column) -> Column:
+	"""
+	Return a column stored with the pandas dtype of its numdaplot type.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		Any pandas column.
+
+	Returns
+	-------
+	pandas.Series
+		The column itself when its dtype is already one a Table uses, otherwise a converted copy.
+		Python objects become integers, decimals or booleans when they all are, and texts otherwise.
+
+	Raises
+	------
+	UnsupportedTypeError
+		When the column has a type numdaplot does not support, such as complex numbers or
+		datetimes in a fixed offset instead of a time zone.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import normalize_column
+	>>> text_column = normalize_column(pandas.Series(["a", None], dtype=object))
+	>>> str(text_column.dtype)
+	'string'
+	"""
+	dtype_name = str(column.dtype)
+	if dtype_name in TYPES_BY_PANDAS_NAME:
+		stored_column = normalize_categories(column)
+		return stored_column
+
+	converted_column = convert_column_dtype(column)
+	from_pandas_dtype(converted_column.dtype)
+
+	return converted_column
+
+
+def convert_column_dtype(column: Column) -> Column:
+	"""
+	Return a column converted to the dtype a Table uses for its kind of values.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		A column whose dtype is not one a Table uses.
+
+	Returns
+	-------
+	pandas.Series
+		The converted column, or the column itself when no conversion applies.
+
+	Raises
+	------
+	UnsupportedTypeError
+		When the column holds datetimes in a fixed offset instead of a time zone.
+
+	Examples
+	--------
+	>>> import numpy
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import convert_column_dtype
+	>>> convert_column_dtype(pandas.Series([1, 2], dtype=numpy.int32)).dtype
+	Int32Dtype()
+	"""
+	column_dtype = column.dtype
+	dtype_kind = column_dtype.kind
+	if isinstance(column_dtype, pandas.DatetimeTZDtype):
+		time_zone_column = normalize_time_zone_column(column)
+		return time_zone_column
+
+	if isinstance(column_dtype, pandas.StringDtype):
+		text_column = convert_to_type(column, Type.TEXT)
+		return text_column
+
+	if dtype_kind == DECIMAL_KIND:
+		decimal_column = convert_decimals(column)
+		return decimal_column
+
+	if dtype_kind == DATETIME_KIND:
+		datetime_column = column.astype(DATETIME_DTYPE)
+		return datetime_column
+
+	if is_time_column(column):
+		return column
+
+	if dtype_kind == DURATION_KIND:
+		duration_column = column.astype(DURATION_DTYPE)
+		return duration_column
+
+	is_numpy_dtype = isinstance(column_dtype, numpy.dtype)
+	if not is_numpy_dtype:
+		return column
+
+	converted_column = convert_numpy_column(column)
+
+	return converted_column
+
+
+def convert_decimals(column: Column) -> Column:
+	"""
+	Return a decimal column stored as numpy decimals, with NaN as missing value.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		A decimal column, such as the pandas ``Float64`` dtype that marks missing values with ``pd.NA``.
+
+	Returns
+	-------
+	pandas.Series
+		The column with the numpy decimal dtype of the same size.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import convert_decimals
+	>>> convert_decimals(pandas.Series([1.5, None], dtype="Float64")).tolist()
+	[1.5, nan]
+	"""
+	dtype_name = str(column.dtype)
+	numpy_dtype_name = dtype_name.lower()
+	decimal_values = column.to_numpy(dtype=numpy_dtype_name, na_value=numpy.nan)
+	decimal_column = pandas.Series(decimal_values, index=column.index)
+
+	return decimal_column
+
+
+def convert_numpy_column(column: Column) -> Column:
+	"""
+	Return a numpy integer, boolean or object column stored with the dtype a Table uses.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		A column with a numpy dtype.
+
+	Returns
+	-------
+	pandas.Series
+		Integers and booleans in the pandas dtypes that accept missing values, Python objects
+		as guessed by ``infer_object_column``, any other column unchanged.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import convert_numpy_column
+	>>> convert_numpy_column(pandas.Series([True, False])).dtype
+	BooleanDtype
+	"""
+	dtype_kind = column.dtype.kind
+	if dtype_kind in INTEGER_KINDS:
+		integer_dtype = cast(numpy.dtype[Any], column.dtype)
+		integer_type = from_numpy_dtype(integer_dtype)
+		integer_column = convert_to_type(column, integer_type)
+		return integer_column
+
+	if dtype_kind == BOOLEAN_KIND:
+		boolean_column = convert_to_type(column, Type.BOOL)
+		return boolean_column
+
+	if column.dtype == PYTHON_OBJECT_DTYPE:
+		object_column = infer_object_column(column)
+		return object_column
+
+	return column
+
+
+def infer_object_column(column: Column) -> Column:
+	"""
+	Return a column of Python objects stored with the type its present values share.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		A column of Python objects.
+
+	Returns
+	-------
+	pandas.Series
+		``INT64`` when every present value is an integer, ``FLOAT64`` for decimals or integers and decimals,
+		``BOOL`` for booleans, ``DATE`` for dates, ``DATETIME`` for datetimes or dates and datetimes, ``DURATION``
+		for durations, and ``TEXT`` for anything else, each value then written as a text.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import infer_object_column
+	>>> infer_object_column(pandas.Series([1, None, 3], dtype=object)).tolist()
+	[1, <NA>, 3]
+	"""
+	object_values = read_object_values(column)
+	if holds_only_times(object_values):
+		time_column = store_time_column(column)
+		return time_column
+
+	inferred_kind = infer_dtype(column, skipna=True)
+	target_type = INFERRED_TYPES.get(inferred_kind, Type.TEXT)
+	if inferred_kind in MOMENT_KINDS:
+		target_type = find_moment_type(column)
+
+	converted_column = convert_to_type(column, target_type)
+
+	return converted_column
+
+
+def find_moment_type(column: Column) -> Type:
+	"""
+	Return the type of a column of Python dates and datetimes.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		A column of Python objects, each present value being a date or a datetime.
+
+	Returns
+	-------
+	Type
+		``DATE`` for dates only, ``DATETIME`` as soon as a value is a datetime, a date then counting as midnight,
+		and ``TEXT`` as soon as a datetime has a time zone, as a column holds a single zone.
+
+	Examples
+	--------
+	>>> from datetime import date, datetime
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import find_moment_type
+	>>> find_moment_type(pandas.Series([date(2026, 1, 31)], dtype=object))
+	<Type.DATE: 14>
+	>>> find_moment_type(pandas.Series([date(2026, 1, 31), datetime(2026, 1, 31, 8)], dtype=object))
+	<Type.DATETIME: 15>
+	"""
+	present_column = column.dropna()
+	moments = present_column.tolist()
+	moment_type = Type.DATE
+
+	for moment in moments:
+		if isinstance(moment, datetime):
+			moment_type = Type.DATETIME
+			if moment.tzinfo is not None:
+				return Type.TEXT
+
+	return moment_type
+
+
+def normalize_categories(column: Column) -> Column:
+	"""
+	Return a column whose categories, if it has any, are texts.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		A column stored with a dtype a Table uses.
+
+	Returns
+	-------
+	pandas.Series
+		The column itself, or a category column of texts when its categories are other values.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.frames.frame_normalization import normalize_categories
+	>>> categories = normalize_categories(pandas.Series([1, 2], dtype="category"))
+	>>> categories.tolist()
+	['1', '2']
+	"""
+	if not isinstance(column.dtype, pandas.CategoricalDtype):
+		return column
+
+	categories = column.cat.categories
+	categories_kind = infer_dtype(categories, skipna=True)
+	if categories_kind in TEXT_INFERRED_KINDS:
+		return column
+
+	text_column = convert_to_type(column, Type.TEXT)
+	category_column = convert_to_type(text_column, Type.CATEGORY)
+
+	return category_column
+
+
+def convert_to_type(column: Column, data_type: Type) -> Column:
+	"""
+	Return a column converted to the pandas dtype of a numdaplot type.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		Any column.
+	data_type
+		The numdaplot type to store the column as.
+
+	Returns
+	-------
+	pandas.Series
+		The converted column.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> import numdaplot as ndp
+	>>> from numdaplot.tables.frames.frame_normalization import convert_to_type
+	>>> convert_to_type(pandas.Series([1, 2]), ndp.Type.TEXT).tolist()
+	['1', '2']
+	"""
+	pandas_dtype = to_pandas_dtype(data_type)
+	converted_column = column.astype(pandas_dtype)
+
+	return converted_column

numdaplot/tables/frames/named_column_errors.py

@@ -0,0 +1,85 @@
+"""
+Name of the Table column in the type errors of a block.
+
+.. versionadded:: 0.1.0
+"""
+
+from types import TracebackType
+
+from numdaplot.errors.incompatible_types_error import IncompatibleTypesError
+from numdaplot.errors.time_overflow_error import TimeOverflowError
+from numdaplot.errors.value_conversion_error import ValueConversionError
+from numdaplot.tables.frames.column import Column
+
+
+class NamedColumnErrors:
+	"""
+	Context adding the name of a Table column to the type errors raised by a block.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	column
+		The column the block works on. A column without name leaves the errors as they are.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> import numdaplot as ndp
+	>>> from numdaplot.tables.frames.named_column_errors import NamedColumnErrors
+	>>> try:
+	... 	with NamedColumnErrors(pandas.Series([1.0], name="price")):
+	... 		raise ndp.IncompatibleTypesError("Contains", ["FLOAT64"])
+	... except ndp.IncompatibleTypesError as error:
+	... 	print(error)
+	Cannot apply Contains to the FLOAT64 column 'price'.
+	"""
+
+	def __init__(self, column: Column) -> None:
+		"""
+		Keep the column of the block.
+
+		.. versionadded:: 0.1.0
+
+		Parameters
+		----------
+		column
+			The column the block works on.
+		"""
+		self.column = column
+
+	def __enter__(self) -> None:
+		"""
+		Start the block.
+
+		.. versionadded:: 0.1.0
+		"""
+
+	def __exit__(
+		self,
+		error_type: type[BaseException] | None,
+		error: BaseException | None,
+		traceback: TracebackType | None,
+	) -> None:
+		"""
+		Name the column in a type error of the block, which then goes on.
+
+		.. versionadded:: 0.1.0
+
+		Parameters
+		----------
+		error_type
+			The type of the error raised by the block, or None.
+		error
+			The error raised by the block, or None.
+		traceback
+			Its traceback, or None.
+		"""
+		column_name = self.column.name
+		if column_name is None:
+			return
+
+		if isinstance(error, IncompatibleTypesError | TimeOverflowError | ValueConversionError):
+			column_text = str(column_name)
+			error.name_column(column_text)

numdaplot/tables/frames/pandas_decimals.py

@@ -0,0 +1,113 @@
+"""
+Exact reading of decimals written as texts, which pandas only gives by default from version 3.1.
+
+.. versionadded:: 0.1.0
+"""
+
+import re
+from typing import Any, cast
+
+import numpy
+import pandas
+from numpy.typing import NDArray
+
+VERSION_PATTERN = r"(\d+)\.(\d+)"
+FIRST_EXACT_VERSION = (3, 1)
+FLOAT_PRECISION_SETTING = "float_precision"
+ROUND_TRIP_PRECISION = "round_trip"
+DECIMAL_KIND = "f"
+
+
+def reads_exact_decimals(pandas_version: str) -> bool:
+	"""
+	Return whether a version of pandas reads the exact value of a decimal written as a text.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	pandas_version
+		The version, such as ``"3.1.0"``.
+
+	Returns
+	-------
+	bool
+		True from version 3.1.
+
+	Examples
+	--------
+	>>> from numdaplot.tables.frames.pandas_decimals import reads_exact_decimals
+	>>> reads_exact_decimals("2.3.0"), reads_exact_decimals("3.1.0rc0")
+	(False, True)
+	"""
+	version_match = cast(re.Match[str], re.match(VERSION_PATTERN, pandas_version))
+	major_text, minor_text = version_match.groups()
+	major = int(major_text)
+	minor = int(minor_text)
+	is_exact = (major, minor) >= FIRST_EXACT_VERSION
+
+	return is_exact
+
+
+READS_EXACT_DECIMALS = reads_exact_decimals(pandas.__version__)
+
+
+def find_exact_csv_settings() -> dict[str, str]:
+	"""
+	Return the settings ``pandas.read_csv`` needs to read the exact value of each decimal.
+
+	.. versionadded:: 0.1.0
+
+	Returns
+	-------
+	dict[str, str]
+		The precision of the slower exact reading before pandas 3.1, nothing from then on as the setting is
+		deprecated there.
+
+	Examples
+	--------
+	>>> from numdaplot.tables.frames.pandas_decimals import find_exact_csv_settings
+	>>> settings = find_exact_csv_settings()
+	>>> settings in ({}, {"float_precision": "round_trip"})
+	True
+	"""
+	if READS_EXACT_DECIMALS:
+		return {}
+
+	return {FLOAT_PRECISION_SETTING: ROUND_TRIP_PRECISION}
+
+
+def restore_exact_decimals(texts: NDArray[Any], numbers: NDArray[Any]) -> NDArray[Any]:
+	"""
+	Return numbers read by ``pandas.to_numeric``, with the exact value of their texts before pandas 3.1.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	texts
+		The texts holding the numbers, each one read by pandas without failure.
+	numbers
+		The numbers pandas read.
+
+	Returns
+	-------
+	NDArray
+		The exact decimals, or the numbers themselves when pandas read them exactly or read integers.
+
+	Examples
+	--------
+	>>> import numpy
+	>>> from numdaplot.tables.frames.pandas_decimals import restore_exact_decimals
+	>>> texts = numpy.array(["0.5"], dtype=object)
+	>>> restore_exact_decimals(texts, numpy.array([0.5]))
+	array([0.5])
+	"""
+	if READS_EXACT_DECIMALS or numbers.dtype.kind != DECIMAL_KIND:
+		return numbers
+
+	text_list = cast(list[str], texts.tolist())
+	exact_list = [float(text) for text in text_list]
+	exact_numbers = numpy.array(exact_list, dtype=numbers.dtype)
+
+	return exact_numbers