dukit 0.2.5__tar.gz → 0.2.6__tar.gz

{dukit-0.2.5 → dukit-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dukit
-Version: 0.2.5
+Version: 0.2.6
 Summary: dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas
 Keywords: pandas,query,utility,data,diff
 Author: MartinVoelkl94

{dukit-0.2.5 → dukit-0.2.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dukit"
-version = "0.2.5"
+version = "0.2.6"
 description = "dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas"
 readme = "README.md"
 authors = [{ name = "MartinVoelkl94", email = "martinv16248@gmail.com" }]

{dukit-0.2.5 → dukit-0.2.6}/src/dukit/__init__.py

@@ -31,9 +31,13 @@ from .pandas import (
     embed,
     flatten,
     get_df,
-    get_dfs,
+    get_df_patients,
+    get_df_medications,
+    get_df_old,
+    get_df_new,
     stagger,
     transpose,
+    split_col,
     )
 from .os import (
     cd,
@@ -92,9 +96,13 @@ __all__ = (
     'embed',
     'flatten',
     'get_df',
-    'get_dfs',
+    'get_df_patients',
+    'get_df_medications',
+    'get_df_old',
+    'get_df_new',
     'stagger',
     'transpose',
+    'split_col',
 
     'cd',
     'cp',

{dukit-0.2.5 → dukit-0.2.6}/src/dukit/_test_utils.py

@@ -33,51 +33,6 @@ def _setup_xlsx(df_old, df_new, tmpdir):
 
 
 
-def _get_dfs():
-
-    df_old = pd.DataFrame(
-        columns=['a', 'b', 'c'],
-        index=['x', 'y', 'z', ],
-        )
-
-    df_old.insert(0, 'uid', df_old.index)
-
-    df_old.loc['x', 'a'] = 1
-    df_old.loc['x', 'b'] = 1
-    df_old.loc['x', 'c'] = 1
-
-    df_old.loc['y', 'a'] = 2
-    df_old.loc['y', 'b'] = 2
-    df_old.loc['y', 'c'] = 2
-
-    df_old.loc['z', 'a'] = 3
-    df_old.loc['z', 'b'] = None
-    df_old.loc['z', 'c'] = 3
-
-
-    df_new = pd.DataFrame(
-        columns=['d', 'b', 'a'],
-        index=['y', 'x2', 'z', ],
-        )
-
-    df_new.insert(0, 'uid', df_new.index)
-
-    df_new.loc['y', 'd'] = 2
-    df_new.loc['y', 'b'] = 2
-    df_new.loc['y', 'a'] = 0
-
-    df_new.loc['x2', 'd'] = 1
-    df_new.loc['x2', 'b'] = 1
-    df_new.loc['x2', 'a'] = 1
-
-    df_new.loc['z', 'd'] = 3
-    df_new.loc['z', 'b'] = 3
-    df_new.loc['z', 'a'] = pd.NA
-
-    return df_old, df_new
-
-
-
 def _get_expected_new():
 
     uid = pd.Series(
@@ -86,7 +41,7 @@ def _get_expected_new():
         name='uid',
         )
     expected = pd.DataFrame(
-        columns=['uid', 'diff', 'd', 'b', 'a'],
+        columns=['diff', 'uid', 'd', 'b', 'a'],
         index=uid,
         )
 
@@ -121,7 +76,7 @@ def _get_expected_newplus():
         name='uid',
         )
     expected = pd.DataFrame(
-        columns=['uid', 'diff', 'd', 'b', 'b *old', 'a', 'a *old'],
+        columns=['diff', 'uid', 'd', 'b', 'b *old', 'a', 'a *old'],
         index=uid,
         )
 
@@ -165,7 +120,7 @@ def _get_expected_old():
         name='uid',
         )
     expected = pd.DataFrame(
-        columns=['uid', 'diff', 'a', 'b', 'c'],
+        columns=['diff', 'uid', 'a', 'b', 'c'],
         index=uid,
         )
 
@@ -200,7 +155,7 @@ def _get_expected_mix():
         name='uid',
         )
     expected = pd.DataFrame(
-        columns=['uid', 'diff', 'd', 'b', 'a', 'c'],
+        columns=['diff', 'uid', 'd', 'b', 'a', 'c'],
         index=uid,
         )
 

{dukit-0.2.5 → dukit-0.2.6}/src/dukit/diffing.py

@@ -27,6 +27,7 @@ def diff(
         remove_cols: list | str = None,
         remove_cols_by_suffix='',
         retain_cols: list | str = None,
+        retain_rows: list | str = None,
         ignore_cols: list | str = None,
         remove_sheets: list | str = None,
         name='data',
@@ -50,11 +51,11 @@ def diff(
         controls the values included in each result.
         - 'mix': show rows and cols from old and new df.
         additions and deletions are highlighted.
-        - 'old': show only rows and cols from the old df.
-        deletions are highlighted.
         - 'new': show only rows and cols from the new df.
         additions are highlighted.
         - 'new+': also preserve changed old values in parallel metadata cols.
+        - 'old': show only rows and cols from the old df.
+        deletions are highlighted.
     rename_cols : dict, optional
         mapping of original col names to names used during comparison.
     remove_cols : list or str, optional
@@ -62,7 +63,9 @@ def diff(
     remove_cols_by_suffix : str, default ''
         remove cols whose names end with this suffix.
     retain_cols : list or str, optional
-        keep only the cols from the old df.
+        keep these cols from the old df.
+    retain_rows : list or str, optional
+        keep these rows from the old df.
     ignore_cols : list or str, optional
         cols excluded from comparison.
     remove_sheets : list or str, optional
@@ -111,6 +114,7 @@ def diff(
         remove_cols=remove_cols,
         remove_cols_by_suffix=remove_cols_by_suffix,
         retain_cols=retain_cols,
+        retain_rows=retain_rows,
         ignore_cols=ignore_cols,
         remove_sheets=remove_sheets,
         name=name,
@@ -132,6 +136,7 @@ def rediff(
         remove_cols: list | str = 'diff',
         remove_cols_by_suffix=' *old',
         retain_cols: list | str = 'notes',
+        retain_rows: list | str = 'notes',
         ignore_cols: list | str = None,
         remove_sheets=['info', 'summary', 'details'],
         name='data',
@@ -154,6 +159,7 @@ def rediff(
         remove_cols=remove_cols,
         remove_cols_by_suffix=remove_cols_by_suffix,
         retain_cols=retain_cols,
+        retain_rows=retain_rows,
         ignore_cols=ignore_cols,
         remove_sheets=remove_sheets,
         name=name,
@@ -182,6 +188,7 @@ class Diffs:
             remove_cols: list | str = None,
             remove_cols_by_suffix='',
             retain_cols: list | str = None,
+            retain_rows: list | str = None,
             ignore_cols: list | str = None,
             remove_sheets: list | str = None,
             name='data',
@@ -232,6 +239,7 @@ class Diffs:
             remove_cols=remove_cols,
             remove_cols_by_suffix=remove_cols_by_suffix,
             retain_cols=retain_cols,
+            retain_rows=retain_rows,
             ignore_cols=ignore_cols,
             linebreak=linebreak,
             suffix_old=suffix_old,
@@ -641,6 +649,7 @@ def _get_diffs(
         remove_cols: list | str = None,
         remove_cols_by_suffix='',
         retain_cols: list | str = None,
+        retain_rows: list | str = None,
         ignore_cols: list | str = None,
         linebreak='<br>',
         suffix_old=' *old',
@@ -660,6 +669,7 @@ def _get_diffs(
                 mode=mode,
                 rename_cols=rename_cols,
                 retain_cols=retain_cols,
+                retain_rows=retain_rows,
                 remove_cols=remove_cols,
                 remove_cols_by_suffix=remove_cols_by_suffix,
                 ignore_cols=ignore_cols,
@@ -677,6 +687,7 @@ def _get_diffs(
                 mode=mode,
                 rename_cols=rename_cols,
                 retain_cols=retain_cols,
+                retain_rows=retain_rows,
                 remove_cols=remove_cols,
                 remove_cols_by_suffix=remove_cols_by_suffix,
                 ignore_cols=ignore_cols,
@@ -694,6 +705,7 @@ def _get_diffs(
                 mode=mode,
                 rename_cols=rename_cols,
                 retain_cols=retain_cols,
+                retain_rows=retain_rows,
                 remove_cols=remove_cols,
                 remove_cols_by_suffix=remove_cols_by_suffix,
                 ignore_cols=ignore_cols,
@@ -719,6 +731,7 @@ def _get_single_diff(
         remove_cols: list | str = None,
         remove_cols_by_suffix='',
         retain_cols: list | str = None,
+        retain_rows: list | str = None,
         ignore_cols: list | str = None,
         name='data',
         linebreak='<br>',
@@ -752,6 +765,7 @@ def _get_single_diff(
 
     d = _set_uid(d, uid)
     d = _retain_cols(d, list_(retain_cols))  #depends on _set_uid()
+    d = _retain_rows(d, list_(retain_rows))  #depends on _set_uid()
     d = _ignore_cols(d, list_(ignore_cols))
 
     d = _create_working_values(d)
@@ -769,8 +783,10 @@ def _get_single_diff(
     d = _populate_diff_col(d)  #depends on _get_row_col_diffs() and _get_val_diffs()
 
     d = _add_retained_cols(d)
-    d = _add_diff_col(d)
+    d = _add_retained_rows(d)
     d = _add_uid_col(d)
+    d = _add_diff_col(d)
+    d = _reset_index(d)
     d = _apply_style(d)
 
     return d
@@ -822,7 +838,8 @@ class Diff:
         self.vals_changed: pd.Series = pd.Series(dtype='int')
 
         #helper attributes for internal use
-        self._df_retained: pd.DataFrame
+        self._cols_retained: pd.DataFrame
+        self._rows_retained: pd.DataFrame
         self._cols_ignore: list[str]
         self._metadata_col_mapping: dict[str, str]
         self._mask_added: pd.DataFrame
@@ -863,9 +880,10 @@ class Diff:
                 f'vals changed: {sum(self.vals_changed)}\n'
                 )
         txt += (
-            '>>>d.result\n'
-            '>>>d.summary\n'
-            '>>>d.details\n'
+            '>>>d.show()\n'
+            '>>>d.info()\n'
+            '>>>d.summary()\n'
+            '>>>d.details()\n'
             )
         txt += '----------------Diff object end----------------\n'
 
@@ -919,7 +937,8 @@ def _handle_edgecases(d: Diff) -> Diff:
     d._values = values
     d._style = style
 
-    d._df_retained = pd.DataFrame()
+    d._cols_retained = pd.DataFrame()
+    d._rows_retained = pd.DataFrame()
     d._cols_ignore = []
     d._metadata_col_mapping = {}
     d._mask_added = pd.DataFrame(dtype='bool')
@@ -1117,17 +1136,41 @@ def _retain_cols(d: Diff, cols: list) -> Diff:
     """
 
     if not cols:
-        d._df_retained = pd.DataFrame()
+        d._cols_retained = pd.DataFrame()
         return d
 
     cols_retain_old = d.old.columns.intersection(cols)
     cols_retain_new = d.new.columns.intersection(cols)
 
-    df_retained = d.old[cols_retain_old].copy()
+    cols_retained = d.old[cols_retain_old].copy()
 
     d.old = d.old.drop(columns=cols_retain_old)
     d.new = d.new.drop(columns=cols_retain_new)
-    d._df_retained = df_retained
+    d._cols_retained = cols_retained
+
+    return d
+
+
+
+def _retain_rows(d: Diff, rows: list) -> Diff:
+    """
+    remove rows from both dfs before diffing,
+    then readd the ones from the old df to
+    the diff result later.
+    """
+
+    if not rows:
+        d._rows_retained = pd.DataFrame()
+        return d
+
+    rows_retain_old = d.old.index.intersection(rows)
+    rows_retain_new = d.new.index.intersection(rows)
+
+    rows_retained = d.old.loc[rows_retain_old].copy()
+
+    d.old = d.old.drop(index=rows_retain_old)
+    d.new = d.new.drop(index=rows_retain_new)
+    d._rows_retained = rows_retained
 
     return d
 
@@ -1158,7 +1201,7 @@ def _create_working_values(d: Diff) -> Diff:
     if d.mode == 'mix':
         rows_old_only = d.old.index.difference(d.new.index)
         cols_old_only = d.old.columns.difference(d.new.columns)
-        d._values = pd.concat([d.new, d.old.loc[:, cols_old_only]], axis=1)
+        d._values = pd.concat([d.new, d.old.loc[:, cols_old_only]], axis='columns')
         d._values.loc[rows_old_only, :] = d.old.loc[rows_old_only, :]
 
     elif d.mode == 'old':
@@ -1266,11 +1309,11 @@ def _create_meta_cols(d: Diff) -> Diff:
 
     values = pd.concat(
         [values, df_meta],
-        axis=1,
+        axis='columns',
         )
     style = pd.concat(
         [style, df_meta_style],
-        axis=1,
+        axis='columns',
         )
 
     d._metadata_col_mapping = col_mapping
@@ -1396,9 +1439,9 @@ def _populate_val_styles(d: Diff) -> Diff:
 
 def _populate_diff_col(d: Diff) -> Diff:
 
-    sum_added = d._mask_added.sum(axis=1)
-    sum_removed = d._mask_removed.sum(axis=1)
-    sum_changed = d._mask_changed.sum(axis=1)
+    sum_added = d._mask_added.sum(axis='columns')
+    sum_removed = d._mask_removed.sum(axis='columns')
+    sum_changed = d._mask_changed.sum(axis='columns')
 
     rows_added = sum_added[sum_added > 0].index
     rows_removed = sum_removed[sum_removed > 0].index
@@ -1450,21 +1493,49 @@ def _populate_meta_cols(d: Diff) -> Diff:
 
 def _add_retained_cols(d: Diff) -> Diff:
 
-    if d._df_retained.empty:
+    if d._cols_retained.empty:
         return d
 
-    idx_shared = d._values.index.intersection(d._df_retained.index)
-    df_retained = d._df_retained.loc[idx_shared, :]
-    d._values = pd.concat([df_retained, d._values], axis=1)
+    idx_shared = d._values.index.intersection(d._cols_retained.index)
+    df_retained = d._cols_retained.loc[idx_shared, :]
+    d._values = pd.concat([df_retained, d._values], axis='columns')
+
+    df_style = pd.DataFrame(
+        data='',
+        index=df_retained.index,
+        columns=df_retained.columns,
+        )
+    d._style = pd.concat([df_style, d._style], axis='columns')
 
     return d
 
 
 
-def _add_diff_col(d: Diff) -> Diff:
-    colname = ensure_unique_string('diff', d._values.columns)
-    d._diff_col.name = colname
-    d._values = pd.concat([d._diff_col, d._values], axis=1)
+def _add_retained_rows(d: Diff) -> Diff:
+
+    if d._rows_retained.empty:
+        return d
+
+    cols_shared = d._values.columns.intersection(d._rows_retained.columns)
+    df_retained = d._rows_retained.loc[:, cols_shared]
+    cols_ordered = d._values.columns
+    d._values = pd.concat([df_retained, d._values], axis='index')
+    d._values = d._values[cols_ordered]
+
+    df_style = pd.DataFrame(
+        data='',
+        index=df_retained.index,
+        columns=cols_shared,
+        )
+    d._style = pd.concat([df_style, d._style], axis='index')
+
+    blank_retained = pd.Series(
+        '',
+        index=df_retained.index,
+        name=d._diff_col.name,
+        )
+    d._diff_col = pd.concat([blank_retained, d._diff_col], axis='index')
+
     return d
 
 
@@ -1477,7 +1548,21 @@ def _add_uid_col(d: Diff) -> Diff:
         index=d._values.index,
         name=colname_uid,
         )
-    d._values = pd.concat([uid_col, d._values], axis=1)
+    d._values = pd.concat([uid_col, d._values], axis='columns')
+
+    return d
+
+
+
+def _add_diff_col(d: Diff) -> Diff:
+    colname = ensure_unique_string('diff', d._values.columns)
+    d._diff_col.name = colname
+    d._values = pd.concat([d._diff_col, d._values], axis='columns')
+    return d
+
+
+
+def _reset_index(d: Diff) -> Diff:
 
     index_diff = d._values.index.difference(d._style.index)
     if index_diff.empty:
@@ -1489,7 +1574,7 @@ def _add_uid_col(d: Diff) -> Diff:
             'This should never happen. Please report this issue.'
             f'Index difference: {index_diff}'
             )
-        log(msg, 'dk.diffing._add_uid_col', d.verbosity)
+        log(msg, 'dk.diffing._reset_index', d.verbosity)
 
     return d
 
@@ -1506,6 +1591,7 @@ def _apply_style(d: Diff) -> Diff:
             )
         log(msg, context, d.verbosity)
 
+    d._style.fillna('', inplace=True)
     d.result = (
         d._values
         .style

{dukit-0.2.5 → dukit-0.2.6}/src/dukit/excel.py

@@ -136,6 +136,14 @@ def save(
         sheet_name='df',
         index=False,
         format_excel=True,
+        freeze_panes='B2',
+        hide_sheets=None,
+        hide_cols=None,
+        col_width_max=70,
+        col_width_padding=2,
+        align_vertical='top',
+        align_horizontal='left',
+        wrap_text=True,
         **kwargs,
         ):
     """
@@ -184,4 +192,15 @@ def save(
             )
 
     if format_excel:
-        format(path, sheet=sheet_name)
+        format(
+            path,
+            sheet=sheet_name,
+            freeze_panes=freeze_panes,
+            hide_sheets=hide_sheets,
+            hide_cols=hide_cols,
+            col_width_max=col_width_max,
+            col_width_padding=col_width_padding,
+            align_vertical=align_vertical,
+            align_horizontal=align_horizontal,
+            wrap_text=wrap_text,
+            )

{dukit-0.2.5 → dukit-0.2.6}/src/dukit/interfaces.py

@@ -16,6 +16,7 @@ from .pandas import (
     flatten,
     stagger,
     transpose,
+    split_col,
     )
 
 
@@ -76,6 +77,9 @@ class DukitAccessor():
             uid: str = None,
             upper: int = None,
             lower: int = None,
+            start_at_day1: bool = True,
+            schedule: dict[int, str] = None,
+            filler: str = '.',
             linebreak: str = '<br>',
             verbosity: int = 3,
             ):
@@ -85,6 +89,9 @@ class DukitAccessor():
             uid=uid,
             upper=upper,
             lower=lower,
+            start_at_day1=start_at_day1,
+            schedule=schedule,
+            filler=filler,
             linebreak=linebreak,
             verbosity=verbosity,
             )
@@ -167,12 +174,37 @@ class DukitAccessor():
         return df_new
 
 
+    def split_col(
+            self,
+            col: str = 'uid',
+            sep: str = '::\n',
+            new1: str = None,
+            new2: str = 'label',
+            ):
+        df_new = split_col(
+            self.df,
+            col=col,
+            sep=sep,
+            new1=new1,
+            new2=new2,
+            )
+        return df_new
+
+
     def save(
             self,
             path: str,
             sheet_name='df',
             index=False,
             format_excel=True,
+            freeze_panes='B2',
+            hide_sheets=None,
+            hide_cols=None,
+            col_width_max=70,
+            col_width_padding=2,
+            align_vertical='top',
+            align_horizontal='left',
+            wrap_text=True,
             **kwargs,
             ):
         save(
@@ -181,6 +213,14 @@ class DukitAccessor():
             sheet_name=sheet_name,
             index=index,
             format_excel=format_excel,
+            freeze_panes=freeze_panes,
+            hide_sheets=hide_sheets,
+            hide_cols=hide_cols,
+            col_width_max=col_width_max,
+            col_width_padding=col_width_padding,
+            align_vertical=align_vertical,
+            align_horizontal=align_horizontal,
+            wrap_text=wrap_text,
             **kwargs,
             )
         return None

{dukit-0.2.5 → dukit-0.2.6}/src/dukit/pandas.py

@@ -4,7 +4,7 @@ import datetime
 import numpy as np
 import pandas as pd
 
-from .utils import log, GREEN_LIGHT
+from .utils import log, ensure_unique_string, GREEN_LIGHT
 from .typing import date_
 
 
@@ -183,8 +183,8 @@ def get_df() -> pd.DataFrame:
 
 
 
-def get_dfs():
-    df1 = pd.DataFrame({
+def get_df_patients() -> pd.DataFrame:
+    patients = pd.DataFrame({
         'id': [
             10001,
             10002,
@@ -204,7 +204,12 @@ def get_dfs():
             40,
             ],
         })
-    df2 = pd.DataFrame({
+    return patients
+
+
+
+def get_df_medications() -> pd.DataFrame:
+    medications = pd.DataFrame({
         'id': [
             10002,
             20001,
@@ -238,7 +243,55 @@ def get_dfs():
             'mg',
             ]
         })
-    return df1, df2
+    return medications
+
+
+def get_df_old() -> pd.DataFrame:
+
+    old = pd.DataFrame(
+        columns=['a', 'b', 'c'],
+        index=['x', 'y', 'z', ],
+        )
+
+    old.insert(0, 'uid', old.index)
+
+    old.loc['x', 'a'] = 1
+    old.loc['x', 'b'] = 1
+    old.loc['x', 'c'] = 1
+
+    old.loc['y', 'a'] = 2
+    old.loc['y', 'b'] = 2
+    old.loc['y', 'c'] = 2
+
+    old.loc['z', 'a'] = 3
+    old.loc['z', 'b'] = None
+    old.loc['z', 'c'] = 3
+
+    return old
+
+
+def get_df_new() -> pd.DataFrame:
+
+    new = pd.DataFrame(
+        columns=['d', 'b', 'a'],
+        index=['y', 'x2', 'z', ],
+        )
+
+    new.insert(0, 'uid', new.index)
+
+    new.loc['y', 'd'] = 2
+    new.loc['y', 'b'] = 2
+    new.loc['y', 'a'] = 0
+
+    new.loc['x2', 'd'] = 1
+    new.loc['x2', 'b'] = 1
+    new.loc['x2', 'a'] = 1
+
+    new.loc['z', 'd'] = 3
+    new.loc['z', 'b'] = 3
+    new.loc['z', 'a'] = pd.NA
+
+    return new
 
 
 
@@ -342,6 +395,7 @@ def date_table(
         lower: int = None,
         start_at_day1: bool = True,
         schedule: dict[int, str] = None,
+        filler: str = '.',
         linebreak: str = '\n',
         verbosity: int = 3,
         ) -> pd.DataFrame | pd.io.formats.style.Styler:
@@ -359,19 +413,17 @@ def date_table(
     df = df.copy()
     if uid:
         df.index = df[uid]
-    reference = df[reference_col].apply(date_)
-
-    df = (
-        df
-        .copy()
-        .map(date_)
-        .convert_dtypes()
-        )
+    df = df.map(date_).convert_dtypes()
 
     for col in df.columns:
         if df[col].isna().all():
-            df.drop(columns=col, inplace=True)
+            if col == reference_col:
+                msg = f'WARNING: no dates found in reference col "{reference_col}"'
+                log(msg, 'dk.date_delta', verbosity)
+            else:
+                df.drop(columns=col, inplace=True)
 
+    reference = df[reference_col].apply(date_).apply(_fix_nas)
 
     blanks = {col: pd.NaT for col in df.columns}
     deltas = pd.DataFrame(
@@ -393,11 +445,10 @@ def date_table(
 
     deltas = deltas.T
 
-
     if upper is None:
-        upper = int(deltas.max().max()) + 1
+        upper = int(deltas.fillna(0).max().max()) + 1
     if lower is None:
-        lower = int(deltas.min().min())
+        lower = int(deltas.fillna(0).min().min())
 
     days = list(range(lower, upper + 1))
     if start_at_day1:
@@ -427,7 +478,7 @@ def date_table(
         if start_at_day1:
             events = _shift_day0(events)
 
-        merged = days_df.copy().merge(events, on='days', how='left').fillna('')
+        merged = days_df.copy().merge(events, on='days', how='left').fillna(filler)
         collapsed = collapse(
             merged,
             on='days',
@@ -439,6 +490,23 @@ def date_table(
             how='left',
             )
 
+    cols_records = [
+        col for col
+        in df_timeline.columns
+        if col not in ('_meta', 'days', 'planned')
+        ]
+    df_timeline.insert(
+        loc=0,
+        column='_meta',
+        value=''
+        )
+    has_event = df_timeline[cols_records] != filler
+    records_per_row = has_event.sum(axis='columns')
+    rows_with_records = records_per_row > 0
+    summary = 'records: ' + records_per_row.astype(str)
+    df_timeline.loc[rows_with_records, '_meta'] = summary
+    df_timeline['_meta'] = df_timeline['_meta'].fillna('')
+
     if schedule is not None:
         df_timeline = _highlight_schedule(df_timeline, schedule)
 
@@ -819,3 +887,39 @@ def transpose(
     df.reset_index(drop=True, inplace=True)
 
     return df
+
+
+
+def split_col(
+        df: pd.DataFrame,
+        col: str = 'uid',
+        sep: str = '::\n',
+        new1: str = None,
+        new2: str = 'label',
+        ):
+
+    if col not in df.columns:
+        log(f'ERROR: no col named "{col}" found')
+        return df
+    elif df.columns.tolist().count(col) > 1:
+        log(f'ERROR: multiple cols named "{col}" found')
+        return df
+
+    df = df.copy()
+    pos = df.columns.get_loc(col)
+    splits = df[col].str.partition(sep)
+
+    df.drop(columns=[col], inplace=True)
+    if new1 is None:
+        new1 = col
+    elif new1 in df.columns:
+        log(f'WARNING: Col {new1} already exists, choosing new name.')
+        new1 = ensure_unique_string(new1, df.columns.tolist())
+    if new2 in df.columns:
+        log(f'WARNING: Col {new2} already exists, choosing new name.')
+        new2 = ensure_unique_string(new2, df.columns.tolist() + [new1])
+
+    df.insert(pos, new1, splits[0])  #type: ignore
+    df.insert(pos + 1, new2, splits[2])  #type: ignore
+
+    return df

{dukit-0.2.5 → dukit-0.2.6}/src/dukit/qlang/symbols.py

@@ -1413,8 +1413,8 @@ class GetNotEquals(Symbol):
 
 class GetContains(Symbol):
     """
-    get cols/rows/vals if
-    they contain an arg.
+    get cols/rows/vals if they
+    contain a given string.
 
     Examples
     --------
@@ -1495,6 +1495,184 @@ class GetContains(Symbol):
 
 
 
+class GetStartsWith(Symbol):
+    """
+    get cols/rows/vals if they
+    start with a given string.
+
+    Examples
+    --------
+    >>> df.dk.qs(r'name  :start(john)')
+    """
+
+    #symbol attributes
+    name = 'GetStartsWith'
+    category = 'getter'
+    regex = (
+        r':startswith',
+        r':starts',
+        r':start',
+        )
+
+    #used to validate the current op
+    connectors_allowed = {
+        'new': 'start a new op',
+        'and': 'combine getter selection with current selection using logical AND',
+        'or': 'combine getter selection with current selection using logical OR',
+        }
+    scopes_allowed = {
+        'cols': 'get or set cols/headers',
+        'rows': 'get or set rows/index',
+        'vals': 'get or set vals within current row and col selection',
+        }
+    flags_allowed = {
+        'negate': 'negate the condition',
+        'regex': 'parse arg as regex',
+
+        'index': 'condition is applied to index instead of vals',
+        'any': 'condition must apply to any args',
+        'all': 'condition must apply to all args',
+        'allcols': 'get rows where condition applies in all selected cols',
+
+        'strict': 'strict type comparison/conversion',
+        }
+    args_allowed = {}
+    args_min = 1
+    args_max = 1_000_000
+
+
+    def parse(self, q: Query) -> Query:
+        q = _preparse_for_getter(q)
+        q = _parse_op_symbol(self, q)
+        return q
+
+
+    def run(self, q: Query) -> Query:
+        if self.scope == 'cols':
+            q = _get_cols(self, q)
+        elif self.scope == 'rows':
+            q = _get_rows(self, q)
+        elif self.scope == 'vals':
+            q = _get_vals(self, q)
+        return q
+
+
+    def getter(
+            self,
+            series: pd.Series,
+            mask: pd.Series[bool],
+            arg: str,
+            q: Query,
+            ) -> pd.Series[bool]:
+
+        series = series.astype('string')
+
+        if 'regex' in self.flags:
+            mask = series.str.match(arg)
+
+        elif 'strict' in self.flags:
+            mask = series.str.startswith(arg)
+
+        else:
+            series = series.str.lower()
+            arg = arg.lower()
+            mask = series.str.startswith(arg)
+
+        return mask
+
+
+
+
+class GetEndsWith(Symbol):
+    """
+    get cols/rows/vals if they
+    end with a given string.
+
+    Examples
+    --------
+    >>> df.dk.qs(r'name  :end(doe)')
+    """
+
+    #symbol attributes
+    name = 'GetEndsWith'
+    category = 'getter'
+    regex = (
+        r':endswith',
+        r':ends',
+        r':end',
+        )
+
+    #used to validate the current op
+    connectors_allowed = {
+        'new': 'start a new op',
+        'and': 'combine getter selection with current selection using logical AND',
+        'or': 'combine getter selection with current selection using logical OR',
+        }
+    scopes_allowed = {
+        'cols': 'get or set cols/headers',
+        'rows': 'get or set rows/index',
+        'vals': 'get or set vals within current row and col selection',
+        }
+    flags_allowed = {
+        'negate': 'negate the condition',
+        'regex': 'parse arg as regex',
+
+        'index': 'condition is applied to index instead of vals',
+        'any': 'condition must apply to any args',
+        'all': 'condition must apply to all args',
+        'allcols': 'get rows where condition applies in all selected cols',
+
+        'strict': 'strict type comparison/conversion',
+        }
+    args_allowed = {}
+    args_min = 1
+    args_max = 1_000_000
+
+
+    def parse(self, q: Query) -> Query:
+        q = _preparse_for_getter(q)
+        q = _parse_op_symbol(self, q)
+        return q
+
+
+    def run(self, q: Query) -> Query:
+        if self.scope == 'cols':
+            q = _get_cols(self, q)
+        elif self.scope == 'rows':
+            q = _get_rows(self, q)
+        elif self.scope == 'vals':
+            q = _get_vals(self, q)
+        return q
+
+
+    def getter(
+            self,
+            series: pd.Series,
+            mask: pd.Series[bool],
+            arg: str,
+            q: Query,
+            ) -> pd.Series[bool]:
+
+        series = series.astype('string')
+
+        if 'regex' in self.flags:
+            if not arg.endswith('$'):
+                arg = arg + '$'
+            mask = series.str.contains(arg)
+
+        elif 'strict' in self.flags:
+            mask = series.str.endswith(arg)
+
+        else:
+            series = series.str.lower()
+            arg = arg.lower()
+            mask = series.str.endswith(arg)
+
+        return mask
+
+
+
+
 class GetGreaterEqual(Symbol):
     """
     get cols/rows/vals if

{dukit-0.2.5 → dukit-0.2.6}/src/dukit/typing.py

@@ -84,6 +84,10 @@ VALUES_NA = (
     'blank',
     'empty',
     )
+VALUES_NA_STRICT = (
+    '<na>',
+    'none',
+    )
 VALUES_NK = (
     'unk',
     'unknown',
@@ -487,6 +491,10 @@ def type_(x) -> str:
         return 'int'
     elif isinstance(x, TYPES_FLOAT):
         return 'float'
+    elif pd.isna(x):
+        return 'na'
+    elif x == '':
+        return 'str'
     elif datetime_(x) is not pd.NaT:
         x = str(x).strip()
         if re.fullmatch(r'\d*', x):
@@ -504,7 +512,7 @@ def type_(x) -> str:
             return 'int'
         elif re.fullmatch(r'-?\d+[\.,]\d+', x):
             return 'float'
-        elif str(x).lower() in VALUES_NA:
+        elif str(x).lower() in VALUES_NA_STRICT:
             return 'na'
         else:
             try:
@@ -513,9 +521,6 @@ def type_(x) -> str:
             except Exception:
                 return 'str'
 
-    elif pd.isna(x):
-        return 'na'
-
     else:
         return type(x).__name__