numdaplot 0.1.0__py3-none-any.whl

numdaplot/tables/missing/missing_removal.py

@@ -0,0 +1,257 @@
+"""
+Removal of the rows or the columns of a Table holding missing values.
+
+.. versionadded:: 0.1.0
+"""
+
+import numpy
+import pandas
+from numpy.typing import NDArray
+
+from numdaplot.positions.axis import Axis
+from numdaplot.tables.columns.column_names import require_known_columns
+from numdaplot.tables.missing.missing_counts import (
+	COLUMN_SUM_AXIS,
+	ROW_SUM_AXIS,
+	divide_counts,
+	find_frame_missing,
+)
+from numdaplot.tables.missing.missing_drop import MissingDrop
+from numdaplot.tables.missing.missing_mode import MissingMode
+from numdaplot.tables.missing.missing_ratio import MissingRatio
+from numdaplot.tables.missing.ratio_comparison import RatioComparison
+
+
+def drop_frame_missing(frame: pandas.DataFrame, missing_drop: MissingDrop, axis: Axis) -> pandas.DataFrame:
+	"""
+	Return a Table without its rows or its columns holding too many missing values.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	frame
+		The columns of a Table.
+	missing_drop
+		The columns looked at and the rule deciding what is removed.
+	axis
+		``EACH_ROW`` to remove rows, ``EACH_COLUMN`` to remove columns.
+
+	Returns
+	-------
+	pandas.DataFrame
+		The remaining rows, in their order and numbered from 0, or the remaining columns, in their order.
+
+	Raises
+	------
+	ColumnNotFoundError
+		When a looked at column does not exist.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> import numdaplot as ndp
+	>>> from numdaplot.tables.missing.missing_removal import drop_frame_missing
+	>>> frame = pandas.DataFrame({"price": [1.5, None, 3.0]})
+	>>> drop_frame_missing(frame, ndp.MissingDrop(), ndp.Axis.EACH_ROW)
+	   price
+	0    1.5
+	1    3.0
+	"""
+	looked_names = [str(column_name) for column_name in frame.columns]
+	if missing_drop.columns is not None:
+		require_known_columns(missing_drop.columns, looked_names)
+		looked_names = missing_drop.columns
+
+	looked_frame = frame[looked_names]
+	if axis == Axis.EACH_COLUMN:
+		remaining_columns = drop_missing_columns(frame, looked_frame, missing_drop.rule)
+		return remaining_columns
+
+	remaining_rows = drop_missing_rows(frame, looked_frame, missing_drop.rule)
+
+	return remaining_rows
+
+
+def drop_missing_rows(
+	frame: pandas.DataFrame,
+	looked_frame: pandas.DataFrame,
+	rule: MissingMode | MissingRatio,
+) -> pandas.DataFrame:
+	"""
+	Return a Table without the rows holding too many missing values in some of its columns.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	frame
+		The columns of a Table.
+	looked_frame
+		The columns whose missing values are counted on each row.
+	rule
+		How many missing values make a row removed.
+
+	Returns
+	-------
+	pandas.DataFrame
+		The remaining rows, in their order, numbered from 0.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> import numdaplot as ndp
+	>>> from numdaplot.tables.missing.missing_removal import drop_missing_rows
+	>>> frame = pandas.DataFrame({"price": [1.5, None], "stock": [None, None]}, dtype="float64")
+	>>> drop_missing_rows(frame, frame, ndp.MissingMode.ALL)
+	   price  stock
+	0    1.5    NaN
+	"""
+	missing_grid = find_frame_missing(looked_frame)
+	row_counts = missing_grid.sum(axis=ROW_SUM_AXIS, dtype=numpy.int64)
+	looked_column_count = len(looked_frame.columns)
+	dropped = find_dropped(row_counts, looked_column_count, rule)
+	kept = ~dropped
+	kept_frame = frame[kept]
+	numbered_frame = kept_frame.reset_index(drop=True)
+
+	return numbered_frame
+
+
+def drop_missing_columns(
+	frame: pandas.DataFrame,
+	looked_frame: pandas.DataFrame,
+	rule: MissingMode | MissingRatio,
+) -> pandas.DataFrame:
+	"""
+	Return a Table without the columns holding too many missing values, among some of its columns.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	frame
+		The columns of a Table.
+	looked_frame
+		The columns that may be removed.
+	rule
+		How many missing values make a column removed.
+
+	Returns
+	-------
+	pandas.DataFrame
+		The remaining columns, in their order.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> import numdaplot as ndp
+	>>> from numdaplot.tables.missing.missing_removal import drop_missing_columns
+	>>> frame = pandas.DataFrame({"price": [1.5, None], "stock": [None, None]}, dtype="float64")
+	>>> drop_missing_columns(frame, frame, ndp.MissingMode.ALL)
+	   price
+	0    1.5
+	1    NaN
+	"""
+	missing_grid = find_frame_missing(looked_frame)
+	column_counts = missing_grid.sum(axis=COLUMN_SUM_AXIS, dtype=numpy.int64)
+	row_count = len(looked_frame)
+	dropped = find_dropped(column_counts, row_count, rule)
+	looked_names = [str(column_name) for column_name in looked_frame.columns]
+	dropped_names = [name for name, is_dropped in zip(looked_names, dropped, strict=True) if is_dropped]
+	remaining_frame = frame.drop(columns=dropped_names)
+
+	return remaining_frame
+
+
+def find_dropped(
+	counts: NDArray[numpy.int64],
+	value_count: int,
+	rule: MissingMode | MissingRatio,
+) -> NDArray[numpy.bool_]:
+	"""
+	Return which rows or columns hold too many missing values.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	counts
+		The number of missing values of each row or column.
+	value_count
+		The number of values of each row or column.
+	rule
+		How many missing values make a row or a column removed.
+
+	Returns
+	-------
+	NDArray[numpy.bool_]
+		True at each row or column to remove, never True when there is no value.
+
+	Examples
+	--------
+	>>> import numpy
+	>>> import numdaplot as ndp
+	>>> from numdaplot.tables.missing.missing_removal import find_dropped
+	>>> find_dropped(numpy.array([0, 1, 2]), 2, ndp.MissingMode.ALL)
+	array([False, False,  True])
+	>>> find_dropped(numpy.array([0, 1, 2]), 2, ndp.MissingRatio(0.5, comparison=ndp.RatioComparison.ABOVE))
+	array([False, False,  True])
+	"""
+	if value_count == 0:
+		nothing_dropped = numpy.zeros(counts.shape, dtype=numpy.bool_)
+		return nothing_dropped
+
+	if isinstance(rule, MissingRatio):
+		ratio_dropped = find_ratio_dropped(counts, value_count, rule)
+		return ratio_dropped
+
+	if rule == MissingMode.ALL:
+		all_missing = counts == value_count
+		return all_missing
+
+	any_missing = counts > 0
+
+	return any_missing
+
+
+def find_ratio_dropped(
+	counts: NDArray[numpy.int64],
+	value_count: int,
+	missing_ratio: MissingRatio,
+) -> NDArray[numpy.bool_]:
+	"""
+	Return which rows or columns reach a ratio of missing values.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	counts
+		The number of missing values of each row or column.
+	value_count
+		The number of values of each row or column, at least 1.
+	missing_ratio
+		The threshold and how a ratio is compared with it.
+
+	Returns
+	-------
+	NDArray[numpy.bool_]
+		True at each row or column whose ratio reaches the threshold.
+
+	Examples
+	--------
+	>>> import numpy
+	>>> import numdaplot as ndp
+	>>> from numdaplot.tables.missing.missing_removal import find_ratio_dropped
+	>>> find_ratio_dropped(numpy.array([0, 1, 2]), 2, ndp.MissingRatio(0.5))
+	array([False,  True,  True])
+	"""
+	ratios = divide_counts(counts, value_count)
+	if missing_ratio.comparison == RatioComparison.ABOVE:
+		above_ratio = ratios > missing_ratio.ratio
+		return above_ratio
+
+	at_least_ratio = ratios >= missing_ratio.ratio
+
+	return at_least_ratio

numdaplot/tables/missing/missing_summary.py

@@ -0,0 +1,174 @@
+"""
+Summary of the missing values of each column of a Table.
+
+.. versionadded:: 0.1.0
+"""
+
+import numpy
+import pandas
+
+from numdaplot.tables.frames.column_values import find_column_type
+from numdaplot.tables.frames.time_zones import name_column_type
+from numdaplot.tables.missing.missing_counts import COLUMN_SUM_AXIS, divide_counts, find_frame_missing
+
+COLUMN_NAMES_COLUMN = "column"
+TYPE_NAMES_COLUMN = "type"
+COUNTS_COLUMN = "missing_count"
+RATIOS_COLUMN = "missing_ratio"
+TOTAL_LABEL = "(all columns)"
+TOTAL_TYPE_NAME = ""
+SUMMARY_TEXT_DTYPE = "string"
+PERCENT_FORMAT = "{percent:.1f} %"
+PERCENT_FACTOR = 100
+
+
+def summarize_missing(frame: pandas.DataFrame) -> pandas.DataFrame:
+	"""
+	Return the number and the percentage of missing values of each column of a Table, then of the whole Table.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	frame
+		The columns of a Table.
+
+	Returns
+	-------
+	pandas.DataFrame
+		One row per column, in order, then an ``(all columns)`` row, with the columns ``column``, ``type``,
+		``missing_count`` and ``missing_ratio``. The ratios are texts such as ``"12.5 %"``,
+		missing when there is no value.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.missing.missing_summary import summarize_missing
+	>>> summarize_missing(pandas.DataFrame({"price": [1.5, None, 3.0], "stock": [2.0, 4.0, 1.0]}))
+	          column     type  missing_count missing_ratio
+	0          price  FLOAT64              1        33.3 %
+	1          stock  FLOAT64              0         0.0 %
+	2  (all columns)                       1        16.7 %
+	"""
+	missing_grid = find_frame_missing(frame)
+	row_count = len(frame)
+	column_counts = missing_grid.sum(axis=COLUMN_SUM_AXIS, dtype=numpy.int64)
+	column_ratios = divide_counts(column_counts, row_count)
+	missing_total = column_counts.sum(dtype=numpy.int64)
+	total_counts = numpy.array([missing_total], dtype=numpy.int64)
+	total_ratios = divide_counts(total_counts, missing_grid.size)
+	counts = numpy.concatenate([column_counts, total_counts])
+	ratios = numpy.concatenate([column_ratios, total_ratios])
+	ratio_texts = [format_ratio(ratio) for ratio in ratios]
+	summary_names = list_summary_names(frame)
+	type_names = list_summary_types(frame)
+	summary_columns = {
+		COLUMN_NAMES_COLUMN: summary_names,
+		TYPE_NAMES_COLUMN: type_names,
+		COUNTS_COLUMN: counts,
+		RATIOS_COLUMN: ratio_texts,
+	}
+	raw_summary = pandas.DataFrame(summary_columns)
+	summary_dtypes = dict.fromkeys([COLUMN_NAMES_COLUMN, TYPE_NAMES_COLUMN, RATIOS_COLUMN], SUMMARY_TEXT_DTYPE)
+	summary = raw_summary.astype(summary_dtypes)
+
+	return summary
+
+
+def list_summary_names(frame: pandas.DataFrame) -> list[str]:
+	"""
+	Return the labels of the rows of a summary: the names of the columns, then ``(all columns)``.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	frame
+		The columns of a Table.
+
+	Returns
+	-------
+	list[str]
+		One label per column, in order, then the label of the total row.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.missing.missing_summary import list_summary_names
+	>>> list_summary_names(pandas.DataFrame({"price": [1.5]}))
+	['price', '(all columns)']
+	"""
+	column_names = [str(column_name) for column_name in frame.columns]
+	summary_names = [*column_names, TOTAL_LABEL]
+
+	return summary_names
+
+
+def list_summary_types(frame: pandas.DataFrame) -> list[str]:
+	"""
+	Return the type names of the rows of a summary: the type of each column, then an empty text.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	frame
+		The columns of a Table.
+
+	Returns
+	-------
+	list[str]
+		The ``ndp.Type`` name of each column, with its time zone when it has one, in order, then an empty
+		text for the total row.
+
+	Examples
+	--------
+	>>> import pandas
+	>>> from numdaplot.tables.missing.missing_summary import list_summary_types
+	>>> list_summary_types(pandas.DataFrame({"price": [1.5]}))
+	['FLOAT64', '']
+	"""
+	type_names = list[str]()
+
+	for column_name in frame.columns:
+		column = frame[column_name]
+		column_type = find_column_type(column)
+		type_name = name_column_type(column, column_type)
+		type_names.append(type_name)
+
+	type_names.append(TOTAL_TYPE_NAME)
+
+	return type_names
+
+
+def format_ratio(ratio: numpy.float64) -> str | None:
+	"""
+	Return a fraction written as a percentage with one decimal.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	ratio
+		A fraction between 0 and 1, or NaN when there is no value.
+
+	Returns
+	-------
+	str or None
+		The percentage, such as ``"12.5 %"``, or ``None`` for NaN.
+
+	Examples
+	--------
+	>>> import numpy
+	>>> from numdaplot.tables.missing.missing_summary import format_ratio
+	>>> format_ratio(numpy.float64(0.125))
+	'12.5 %'
+	"""
+	is_missing = bool(numpy.isnan(ratio))
+	if is_missing:
+		return None
+
+	percent = ratio * PERCENT_FACTOR
+	ratio_text = PERCENT_FORMAT.format(percent=percent)
+
+	return ratio_text

numdaplot/tables/missing/ratio_comparison.py

@@ -0,0 +1,70 @@
+"""
+How a ratio of missing values is compared with its threshold.
+
+.. versionadded:: 0.1.0
+"""
+
+from enum import Enum, auto
+
+
+class RatioComparison(Enum):
+	"""
+	How a ratio of missing values is compared with its threshold, used by ``MissingRatio``.
+
+	.. versionadded:: 0.1.0
+
+	Examples
+	--------
+	>>> import numdaplot as ndp
+	>>> prices = ndp.Array.from_list([120.0, None, 80.0, None])
+	>>> stocks = ndp.Array.from_list([3.0, 5.0, None, 2.0])
+	>>> table = ndp.Table.from_arrays({"price": prices, "stock": stocks})
+	>>> half = ndp.MissingRatio(0.5, comparison=ndp.RatioComparison.ABOVE)
+	>>> table.drop_missing(ndp.MissingDrop(rule=half), axis=ndp.Axis.EACH_COLUMN)
+	    price   stock
+	  FLOAT64 FLOAT64
+	0   120.0     3.0
+	1     NaN     5.0
+	2    80.0     NaN
+	3     NaN     2.0
+	"""
+
+	AT_LEAST = auto()
+	"""
+	The ratio reaches the threshold or goes beyond it. Default value.
+
+	Examples
+	--------
+	>>> import numdaplot as ndp
+	>>> prices = ndp.Array.from_list([120.0, None, 80.0, None])
+	>>> stocks = ndp.Array.from_list([3.0, 5.0, None, 2.0])
+	>>> table = ndp.Table.from_arrays({"price": prices, "stock": stocks})
+	>>> half = ndp.MissingRatio(0.5, comparison=ndp.RatioComparison.AT_LEAST)
+	>>> table.drop_missing(ndp.MissingDrop(rule=half), axis=ndp.Axis.EACH_COLUMN)
+	    stock
+	  FLOAT64
+	0     3.0
+	1     5.0
+	2     NaN
+	3     2.0
+	"""
+
+	ABOVE = auto()
+	"""
+	The ratio goes beyond the threshold.
+
+	Examples
+	--------
+	>>> import numdaplot as ndp
+	>>> prices = ndp.Array.from_list([120.0, None, 80.0, None])
+	>>> stocks = ndp.Array.from_list([3.0, 5.0, None, 2.0])
+	>>> table = ndp.Table.from_arrays({"price": prices, "stock": stocks})
+	>>> half = ndp.MissingRatio(0.5, comparison=ndp.RatioComparison.ABOVE)
+	>>> table.drop_missing(ndp.MissingDrop(rule=half), axis=ndp.Axis.EACH_COLUMN)
+	    price   stock
+	  FLOAT64 FLOAT64
+	0   120.0     3.0
+	1     NaN     5.0
+	2    80.0     NaN
+	3     NaN     2.0
+	"""

numdaplot/tables/pivot/__init__.py

@@ -0,0 +1,5 @@
+"""
+Pivot tables: one column per value of a column of a Table.
+
+.. versionadded:: 0.1.0
+"""

numdaplot/tables/pivot/pivot_table.py

@@ -0,0 +1,104 @@
+"""
+Settings of a pivot table.
+
+.. versionadded:: 0.1.0
+"""
+
+from dataclasses import dataclass
+
+from numdaplot.errors.empty_list_error import EmptyListError
+from numdaplot.errors.refused_setting_error import RefusedSettingError
+from numdaplot.settings.field_types import require_field_types
+from numdaplot.tables.aggregation.calculation import Calculation
+from numdaplot.tables.columns.column_names import require_column_name, require_column_names, require_unique_names
+
+ROW_COLUMN_ITEM_NAME = "row column"
+RESULT_NAME_SETTING = "the result_name of its calculation"
+PIVOT_TABLE_NAME = "PivotTable"
+
+
+@dataclass(frozen=True)
+class PivotTable:
+	"""
+	One row per group of rows and one column per value of a column, used by ``Table.pivot_table``.
+
+	.. versionadded:: 0.1.0
+
+	Parameters
+	----------
+	rows
+		The names of the columns whose values form the rows of the result: one row per combination of values, sorted,
+		with missing values as a group of their own, last. The pivot table keeps its own copy of the list.
+	columns
+		The name of the column whose values become the columns of the result, sorted. It cannot hold missing values.
+		The values are written as texts: ``"Paris"``, ``"8"`` for 8 and 8.0, ``"2.5"``, ``"2026-12-31"``.
+	calculation
+		The statistic of each cell. A combination without any row gives what the statistic gives without value: 0
+		for ``Count`` and ``Sum``, a missing value for ``Mean`` or ``Max``. Its ``result_name`` is refused, since each
+		result column is named after a value.
+
+	Raises
+	------
+	InvalidTypeError
+		When a setting is not of the type its annotation names.
+	EmptyListError
+		When ``rows`` is an empty list.
+	InvalidColumnNameError
+		When ``columns`` or a name in ``rows`` is an empty text.
+	DuplicateColumnError
+		When a name appears twice among ``rows`` and ``columns``.
+	RefusedSettingError
+		When the calculation has a ``result_name``.
+
+	Examples
+	--------
+	>>> import numdaplot as ndp
+	>>> hours = ndp.Array.from_list([8, 8, 9])
+	>>> weekdays = ndp.Array.from_list(["Mon", "Tue", "Mon"])
+	>>> fares = ndp.Array.from_list([7.0, 9.0, 5.0])
+	>>> trips = ndp.Table.from_arrays({"hour": hours, "weekday": weekdays, "fare": fares})
+	>>> total_fare = ndp.Calculation(column="fare", function=ndp.Sum())
+	>>> trips.pivot_table(ndp.PivotTable(rows=["hour"], columns="weekday", calculation=total_fare))
+	   hour     Mon     Tue
+	  INT64 FLOAT64 FLOAT64
+	0     8     7.0     9.0
+	1     9     5.0     0.0
+	"""
+
+	rows: list[str]
+	columns: str
+	calculation: Calculation
+
+	def __post_init__(self) -> None:
+		"""
+		Check the names and the calculation, and replace ``rows`` with a copy that only the pivot table holds.
+
+		.. versionadded:: 0.1.0
+
+		Raises
+		------
+		InvalidTypeError
+			When a setting is not of the type its annotation names.
+		EmptyListError
+			When ``rows`` is an empty list.
+		InvalidColumnNameError
+			When ``columns`` or a name in ``rows`` is an empty text.
+		DuplicateColumnError
+			When a name appears twice among ``rows`` and ``columns``.
+		RefusedSettingError
+			When the calculation has a ``result_name``.
+		"""
+		require_field_types(self)
+
+		if self.rows == []:
+			raise EmptyListError(ROW_COLUMN_ITEM_NAME)
+
+		if self.calculation.result_name is not None:
+			raise RefusedSettingError(RESULT_NAME_SETTING, PIVOT_TABLE_NAME)
+
+		require_column_names(self.rows)
+		require_column_name(self.columns)
+		rows_copy = list(self.rows)
+		object.__setattr__(self, "rows", rows_copy)
+		grouping_names = [*rows_copy, self.columns]
+		require_unique_names(grouping_names)