dukit 0.2.5__tar.gz → 0.2.7__tar.gz

{dukit-0.2.5 → dukit-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dukit
-Version: 0.2.5
+Version: 0.2.7
 Summary: dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas
 Keywords: pandas,query,utility,data,diff
 Author: MartinVoelkl94
@@ -89,8 +89,8 @@ these functions offer various ways to deal with non-unique keys/ids.
 primarily used when merging 2 dfs with a one-to-many relationship.
 
 available functions:
-- df.dk.flatten()
-- df.dk.stagger()
+- df.dk.stack()
+- df.dk.unstack()
 - df.dk.embed()
 - df.dk.collapse()
 <br>

{dukit-0.2.5 → dukit-0.2.7}/README.md

@@ -72,8 +72,8 @@ these functions offer various ways to deal with non-unique keys/ids.
 primarily used when merging 2 dfs with a one-to-many relationship.
 
 available functions:
-- df.dk.flatten()
-- df.dk.stagger()
+- df.dk.stack()
+- df.dk.unstack()
 - df.dk.embed()
 - df.dk.collapse()
 <br>

{dukit-0.2.5 → dukit-0.2.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dukit"
-version = "0.2.5"
+version = "0.2.7"
 description = "dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas"
 readme = "README.md"
 authors = [{ name = "MartinVoelkl94", email = "martinv16248@gmail.com" }]

{dukit-0.2.5 → dukit-0.2.7}/src/dukit/__init__.py

@@ -24,16 +24,24 @@ from .excel import (
     save,
     )
 from .pandas import (
-    collapse,
+    compose,
+    decompose,
+    stack,
+    unstack,
+    transpose,
     date_delta,
     date_table,
     deduplicate,
-    embed,
-    flatten,
     get_df,
-    get_dfs,
+    get_df_patients,
+    get_df_medications,
+    get_df_old,
+    get_df_new,
+    split_col,
+    embed,
+    collapse,
     stagger,
-    transpose,
+    flatten,
     )
 from .os import (
     cd,
@@ -85,16 +93,24 @@ __all__ = (
     'format',
     'save',
 
-    'collapse',
+    'compose',
+    'decompose',
+    'stack',
+    'unstack',
+    'transpose',
     'date_delta',
     'date_table',
     'deduplicate',
-    'embed',
-    'flatten',
     'get_df',
-    'get_dfs',
+    'get_df_patients',
+    'get_df_medications',
+    'get_df_old',
+    'get_df_new',
+    'split_col',
+    'embed',
+    'collapse',
     'stagger',
-    'transpose',
+    'flatten',
 
     'cd',
     'cp',

{dukit-0.2.5 → dukit-0.2.7}/src/dukit/_test_utils.py

@@ -33,51 +33,6 @@ def _setup_xlsx(df_old, df_new, tmpdir):
 
 
 
-def _get_dfs():
-
-    df_old = pd.DataFrame(
-        columns=['a', 'b', 'c'],
-        index=['x', 'y', 'z', ],
-        )
-
-    df_old.insert(0, 'uid', df_old.index)
-
-    df_old.loc['x', 'a'] = 1
-    df_old.loc['x', 'b'] = 1
-    df_old.loc['x', 'c'] = 1
-
-    df_old.loc['y', 'a'] = 2
-    df_old.loc['y', 'b'] = 2
-    df_old.loc['y', 'c'] = 2
-
-    df_old.loc['z', 'a'] = 3
-    df_old.loc['z', 'b'] = None
-    df_old.loc['z', 'c'] = 3
-
-
-    df_new = pd.DataFrame(
-        columns=['d', 'b', 'a'],
-        index=['y', 'x2', 'z', ],
-        )
-
-    df_new.insert(0, 'uid', df_new.index)
-
-    df_new.loc['y', 'd'] = 2
-    df_new.loc['y', 'b'] = 2
-    df_new.loc['y', 'a'] = 0
-
-    df_new.loc['x2', 'd'] = 1
-    df_new.loc['x2', 'b'] = 1
-    df_new.loc['x2', 'a'] = 1
-
-    df_new.loc['z', 'd'] = 3
-    df_new.loc['z', 'b'] = 3
-    df_new.loc['z', 'a'] = pd.NA
-
-    return df_old, df_new
-
-
-
 def _get_expected_new():
 
     uid = pd.Series(
@@ -86,7 +41,7 @@ def _get_expected_new():
         name='uid',
         )
     expected = pd.DataFrame(
-        columns=['uid', 'diff', 'd', 'b', 'a'],
+        columns=['diff', 'uid', 'd', 'b', 'a'],
         index=uid,
         )
 
@@ -121,7 +76,7 @@ def _get_expected_newplus():
         name='uid',
         )
     expected = pd.DataFrame(
-        columns=['uid', 'diff', 'd', 'b', 'b *old', 'a', 'a *old'],
+        columns=['diff', 'uid', 'd', 'b', 'b *old', 'a', 'a *old'],
         index=uid,
         )
 
@@ -165,7 +120,7 @@ def _get_expected_old():
         name='uid',
         )
     expected = pd.DataFrame(
-        columns=['uid', 'diff', 'a', 'b', 'c'],
+        columns=['diff', 'uid', 'a', 'b', 'c'],
         index=uid,
         )
 
@@ -200,7 +155,7 @@ def _get_expected_mix():
         name='uid',
         )
     expected = pd.DataFrame(
-        columns=['uid', 'diff', 'd', 'b', 'a', 'c'],
+        columns=['diff', 'uid', 'd', 'b', 'a', 'c'],
         index=uid,
         )
 

{dukit-0.2.5 → dukit-0.2.7}/src/dukit/diffing.py

@@ -27,6 +27,7 @@ def diff(
         remove_cols: list | str = None,
         remove_cols_by_suffix='',
         retain_cols: list | str = None,
+        retain_rows: list | str = None,
         ignore_cols: list | str = None,
         remove_sheets: list | str = None,
         name='data',
@@ -50,11 +51,11 @@ def diff(
         controls the values included in each result.
         - 'mix': show rows and cols from old and new df.
         additions and deletions are highlighted.
-        - 'old': show only rows and cols from the old df.
-        deletions are highlighted.
         - 'new': show only rows and cols from the new df.
         additions are highlighted.
         - 'new+': also preserve changed old values in parallel metadata cols.
+        - 'old': show only rows and cols from the old df.
+        deletions are highlighted.
     rename_cols : dict, optional
         mapping of original col names to names used during comparison.
     remove_cols : list or str, optional
@@ -62,7 +63,9 @@ def diff(
     remove_cols_by_suffix : str, default ''
         remove cols whose names end with this suffix.
     retain_cols : list or str, optional
-        keep only the cols from the old df.
+        keep these cols from the old df.
+    retain_rows : list or str, optional
+        keep these rows from the old df.
     ignore_cols : list or str, optional
         cols excluded from comparison.
     remove_sheets : list or str, optional
@@ -111,6 +114,7 @@ def diff(
         remove_cols=remove_cols,
         remove_cols_by_suffix=remove_cols_by_suffix,
         retain_cols=retain_cols,
+        retain_rows=retain_rows,
         ignore_cols=ignore_cols,
         remove_sheets=remove_sheets,
         name=name,
@@ -132,6 +136,7 @@ def rediff(
         remove_cols: list | str = 'diff',
         remove_cols_by_suffix=' *old',
         retain_cols: list | str = 'notes',
+        retain_rows: list | str = 'notes',
         ignore_cols: list | str = None,
         remove_sheets=['info', 'summary', 'details'],
         name='data',
@@ -154,6 +159,7 @@ def rediff(
         remove_cols=remove_cols,
         remove_cols_by_suffix=remove_cols_by_suffix,
         retain_cols=retain_cols,
+        retain_rows=retain_rows,
         ignore_cols=ignore_cols,
         remove_sheets=remove_sheets,
         name=name,
@@ -182,6 +188,7 @@ class Diffs:
             remove_cols: list | str = None,
             remove_cols_by_suffix='',
             retain_cols: list | str = None,
+            retain_rows: list | str = None,
             ignore_cols: list | str = None,
             remove_sheets: list | str = None,
             name='data',
@@ -232,6 +239,7 @@ class Diffs:
             remove_cols=remove_cols,
             remove_cols_by_suffix=remove_cols_by_suffix,
             retain_cols=retain_cols,
+            retain_rows=retain_rows,
             ignore_cols=ignore_cols,
             linebreak=linebreak,
             suffix_old=suffix_old,
@@ -641,6 +649,7 @@ def _get_diffs(
         remove_cols: list | str = None,
         remove_cols_by_suffix='',
         retain_cols: list | str = None,
+        retain_rows: list | str = None,
         ignore_cols: list | str = None,
         linebreak='<br>',
         suffix_old=' *old',
@@ -660,6 +669,7 @@ def _get_diffs(
                 mode=mode,
                 rename_cols=rename_cols,
                 retain_cols=retain_cols,
+                retain_rows=retain_rows,
                 remove_cols=remove_cols,
                 remove_cols_by_suffix=remove_cols_by_suffix,
                 ignore_cols=ignore_cols,
@@ -677,6 +687,7 @@ def _get_diffs(
                 mode=mode,
                 rename_cols=rename_cols,
                 retain_cols=retain_cols,
+                retain_rows=retain_rows,
                 remove_cols=remove_cols,
                 remove_cols_by_suffix=remove_cols_by_suffix,
                 ignore_cols=ignore_cols,
@@ -694,6 +705,7 @@ def _get_diffs(
                 mode=mode,
                 rename_cols=rename_cols,
                 retain_cols=retain_cols,
+                retain_rows=retain_rows,
                 remove_cols=remove_cols,
                 remove_cols_by_suffix=remove_cols_by_suffix,
                 ignore_cols=ignore_cols,
@@ -719,6 +731,7 @@ def _get_single_diff(
         remove_cols: list | str = None,
         remove_cols_by_suffix='',
         retain_cols: list | str = None,
+        retain_rows: list | str = None,
         ignore_cols: list | str = None,
         name='data',
         linebreak='<br>',
@@ -752,6 +765,7 @@ def _get_single_diff(
 
     d = _set_uid(d, uid)
     d = _retain_cols(d, list_(retain_cols))  #depends on _set_uid()
+    d = _retain_rows(d, list_(retain_rows))  #depends on _set_uid()
     d = _ignore_cols(d, list_(ignore_cols))
 
     d = _create_working_values(d)
@@ -769,8 +783,10 @@ def _get_single_diff(
     d = _populate_diff_col(d)  #depends on _get_row_col_diffs() and _get_val_diffs()
 
     d = _add_retained_cols(d)
-    d = _add_diff_col(d)
+    d = _add_retained_rows(d)
     d = _add_uid_col(d)
+    d = _add_diff_col(d)
+    d = _reset_index(d)
     d = _apply_style(d)
 
     return d
@@ -822,7 +838,8 @@ class Diff:
         self.vals_changed: pd.Series = pd.Series(dtype='int')
 
         #helper attributes for internal use
-        self._df_retained: pd.DataFrame
+        self._cols_retained: pd.DataFrame
+        self._rows_retained: pd.DataFrame
         self._cols_ignore: list[str]
         self._metadata_col_mapping: dict[str, str]
         self._mask_added: pd.DataFrame
@@ -863,9 +880,10 @@ class Diff:
                 f'vals changed: {sum(self.vals_changed)}\n'
                 )
         txt += (
-            '>>>d.result\n'
-            '>>>d.summary\n'
-            '>>>d.details\n'
+            '>>>d.show()\n'
+            '>>>d.info()\n'
+            '>>>d.summary()\n'
+            '>>>d.details()\n'
             )
         txt += '----------------Diff object end----------------\n'
 
@@ -919,7 +937,8 @@ def _handle_edgecases(d: Diff) -> Diff:
     d._values = values
     d._style = style
 
-    d._df_retained = pd.DataFrame()
+    d._cols_retained = pd.DataFrame()
+    d._rows_retained = pd.DataFrame()
     d._cols_ignore = []
     d._metadata_col_mapping = {}
     d._mask_added = pd.DataFrame(dtype='bool')
@@ -1117,17 +1136,41 @@ def _retain_cols(d: Diff, cols: list) -> Diff:
     """
 
     if not cols:
-        d._df_retained = pd.DataFrame()
+        d._cols_retained = pd.DataFrame()
         return d
 
     cols_retain_old = d.old.columns.intersection(cols)
     cols_retain_new = d.new.columns.intersection(cols)
 
-    df_retained = d.old[cols_retain_old].copy()
+    cols_retained = d.old[cols_retain_old].copy()
 
     d.old = d.old.drop(columns=cols_retain_old)
     d.new = d.new.drop(columns=cols_retain_new)
-    d._df_retained = df_retained
+    d._cols_retained = cols_retained
+
+    return d
+
+
+
+def _retain_rows(d: Diff, rows: list) -> Diff:
+    """
+    remove rows from both dfs before diffing,
+    then readd the ones from the old df to
+    the diff result later.
+    """
+
+    if not rows:
+        d._rows_retained = pd.DataFrame()
+        return d
+
+    rows_retain_old = d.old.index.intersection(rows)
+    rows_retain_new = d.new.index.intersection(rows)
+
+    rows_retained = d.old.loc[rows_retain_old].copy()
+
+    d.old = d.old.drop(index=rows_retain_old)
+    d.new = d.new.drop(index=rows_retain_new)
+    d._rows_retained = rows_retained
 
     return d
 
@@ -1158,7 +1201,7 @@ def _create_working_values(d: Diff) -> Diff:
     if d.mode == 'mix':
         rows_old_only = d.old.index.difference(d.new.index)
         cols_old_only = d.old.columns.difference(d.new.columns)
-        d._values = pd.concat([d.new, d.old.loc[:, cols_old_only]], axis=1)
+        d._values = pd.concat([d.new, d.old.loc[:, cols_old_only]], axis='columns')
         d._values.loc[rows_old_only, :] = d.old.loc[rows_old_only, :]
 
     elif d.mode == 'old':
@@ -1266,11 +1309,11 @@ def _create_meta_cols(d: Diff) -> Diff:
 
     values = pd.concat(
         [values, df_meta],
-        axis=1,
+        axis='columns',
         )
     style = pd.concat(
         [style, df_meta_style],
-        axis=1,
+        axis='columns',
         )
 
     d._metadata_col_mapping = col_mapping
@@ -1396,9 +1439,9 @@ def _populate_val_styles(d: Diff) -> Diff:
 
 def _populate_diff_col(d: Diff) -> Diff:
 
-    sum_added = d._mask_added.sum(axis=1)
-    sum_removed = d._mask_removed.sum(axis=1)
-    sum_changed = d._mask_changed.sum(axis=1)
+    sum_added = d._mask_added.sum(axis='columns')
+    sum_removed = d._mask_removed.sum(axis='columns')
+    sum_changed = d._mask_changed.sum(axis='columns')
 
     rows_added = sum_added[sum_added > 0].index
     rows_removed = sum_removed[sum_removed > 0].index
@@ -1450,21 +1493,60 @@ def _populate_meta_cols(d: Diff) -> Diff:
 
 def _add_retained_cols(d: Diff) -> Diff:
 
-    if d._df_retained.empty:
+    if d._cols_retained.empty:
         return d
 
-    idx_shared = d._values.index.intersection(d._df_retained.index)
-    df_retained = d._df_retained.loc[idx_shared, :]
-    d._values = pd.concat([df_retained, d._values], axis=1)
+    idx_shared = d._values.index.intersection(d._cols_retained.index)
+    df_retained = d._cols_retained.loc[idx_shared, :]
+    d._values = pd.merge(
+        df_retained,
+        d._values,
+        left_index=True,
+        right_index=True,
+        how='right',
+        )
 
+    df_style = pd.DataFrame(
+        data='',
+        index=df_retained.index,
+        columns=df_retained.columns,
+        )
+    d._style = pd.merge(
+        df_style,
+        d._style,
+        left_index=True,
+        right_index=True,
+        how='right',
+        )
     return d
 
 
 
-def _add_diff_col(d: Diff) -> Diff:
-    colname = ensure_unique_string('diff', d._values.columns)
-    d._diff_col.name = colname
-    d._values = pd.concat([d._diff_col, d._values], axis=1)
+def _add_retained_rows(d: Diff) -> Diff:
+
+    if d._rows_retained.empty:
+        return d
+
+    cols_shared = d._values.columns.intersection(d._rows_retained.columns)
+    df_retained = d._rows_retained.loc[:, cols_shared]
+    cols_ordered = d._values.columns
+    d._values = pd.concat([df_retained, d._values], axis='index')
+    d._values = d._values[cols_ordered]
+
+    df_style = pd.DataFrame(
+        data='',
+        index=df_retained.index,
+        columns=cols_shared,
+        )
+    d._style = pd.concat([df_style, d._style], axis='index')
+
+    blank_retained = pd.Series(
+        '',
+        index=df_retained.index,
+        name=d._diff_col.name,
+        )
+    d._diff_col = pd.concat([blank_retained, d._diff_col], axis='index')
+
     return d
 
 
@@ -1477,7 +1559,21 @@ def _add_uid_col(d: Diff) -> Diff:
         index=d._values.index,
         name=colname_uid,
         )
-    d._values = pd.concat([uid_col, d._values], axis=1)
+    d._values = pd.concat([uid_col, d._values], axis='columns')
+
+    return d
+
+
+
+def _add_diff_col(d: Diff) -> Diff:
+    colname = ensure_unique_string('diff', d._values.columns)
+    d._diff_col.name = colname
+    d._values = pd.concat([d._diff_col, d._values], axis='columns')
+    return d
+
+
+
+def _reset_index(d: Diff) -> Diff:
 
     index_diff = d._values.index.difference(d._style.index)
     if index_diff.empty:
@@ -1489,7 +1585,7 @@ def _add_uid_col(d: Diff) -> Diff:
             'This should never happen. Please report this issue.'
             f'Index difference: {index_diff}'
             )
-        log(msg, 'dk.diffing._add_uid_col', d.verbosity)
+        log(msg, 'dk.diffing._reset_index', d.verbosity)
 
     return d
 
@@ -1506,6 +1602,7 @@ def _apply_style(d: Diff) -> Diff:
             )
         log(msg, context, d.verbosity)
 
+    d._style.fillna('', inplace=True)
     d.result = (
         d._values
         .style

{dukit-0.2.5 → dukit-0.2.7}/src/dukit/excel.py

@@ -136,6 +136,14 @@ def save(
         sheet_name='df',
         index=False,
         format_excel=True,
+        freeze_panes='B2',
+        hide_sheets=None,
+        hide_cols=None,
+        col_width_max=70,
+        col_width_padding=2,
+        align_vertical='top',
+        align_horizontal='left',
+        wrap_text=True,
         **kwargs,
         ):
     """
@@ -184,4 +192,15 @@ def save(
             )
 
     if format_excel:
-        format(path, sheet=sheet_name)
+        format(
+            path,
+            sheet=sheet_name,
+            freeze_panes=freeze_panes,
+            hide_sheets=hide_sheets,
+            hide_cols=hide_cols,
+            col_width_max=col_width_max,
+            col_width_padding=col_width_padding,
+            align_vertical=align_vertical,
+            align_horizontal=align_horizontal,
+            wrap_text=wrap_text,
+            )