dukit 0.2.2__tar.gz → 0.2.3__tar.gz

{dukit-0.2.2 → dukit-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dukit
-Version: 0.2.2
+Version: 0.2.3
 Summary: dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas
 Keywords: pandas,query,utility,data,diff
 Author: MartinVoelkl94

{dukit-0.2.2 → dukit-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dukit"
-version = "0.2.2"
+version = "0.2.3"
 description = "dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas"
 readme = "README.md"
 authors = [{ name = "MartinVoelkl94", email = "martinv16248@gmail.com" }]

{dukit-0.2.2 → dukit-0.2.3}/src/dukit/__init__.py

@@ -25,6 +25,7 @@ from .pandas import (
     collapse,
     transpose,
     date_delta,
+    date_table,
     )
 from .excel import (
     format,
@@ -78,6 +79,7 @@ __all__ = (
     'collapse',
     'transpose',
     'date_delta',
+    'date_table',
     'format',
     'save',
     'Box',

{dukit-0.2.2 → dukit-0.2.3}/src/dukit/diffing.py

@@ -1,6 +1,5 @@
 
 import pandas as pd
-import openpyxl
 import typing
 import os
 
@@ -447,7 +446,6 @@ class Diffs:
 
         #format excel file
         if apply_format:
-            wb = openpyxl.load_workbook(path)
 
             #metadata sheets
             sheets_hide = []
@@ -462,7 +460,6 @@ class Diffs:
                 sheet=[sheet_info, sheet_summary, sheet_details],
                 hide_sheets=sheets_hide,
                 freeze_panes='B2',
-                openpyxl_workbook=wb,
                 )
             msg = 'DEBUG: info, summary, details sheets formatted'
             log(msg, context, self.verbosity)
@@ -478,7 +475,6 @@ class Diffs:
                     sheet=diff.name,
                     hide_cols=cols_hide,
                     freeze_panes=freeze_panes,
-                    openpyxl_workbook=wb,
                     )
                 msg = f'DEBUG: diff sheet formatted: "{diff.name}"'
                 log(msg, context, self.verbosity)

{dukit-0.2.2 → dukit-0.2.3}/src/dukit/excel.py

@@ -19,7 +19,6 @@ def format(
         align_vertical='top',
         align_horizontal='left',
         wrap_text=True,
-        openpyxl_workbook=None,
         verbosity=3,
         ):  #pragma: no cover (does not affect reading of xlsx files)
     """
@@ -29,10 +28,8 @@ def format(
     - hide specified columns
     """
 
-    if openpyxl_workbook:
-        wb = openpyxl_workbook
-    else:
-        wb = openpyxl.load_workbook(path)
+    wb = openpyxl.load_workbook(path)
+
     if sheet:
         sheetnames = list_(sheet)
     else:

{dukit-0.2.2 → dukit-0.2.3}/src/dukit/interfaces.py

@@ -1,5 +1,6 @@
 
 import pandas as pd
+import datetime
 from .excel import save
 from .qlang import (
     q,
@@ -12,6 +13,8 @@ from .pandas import (
     embed,
     collapse,
     transpose,
+    date_delta,
+    date_table,
     )
 
 
@@ -104,16 +107,14 @@ class DukitAccessor():
     def collapse(
             self,
             on: str,
-            template='{colname}',
-            line_start='#',
-            line_stop='\n',
+            template_col='{colname}',
+            template_item='#{counter}: {item}\n',
             ):
         df_new = collapse(
             self.df,
             on=on,
-            template=template,
-            line_start=line_start,
-            line_stop=line_stop,
+            template_col=template_col,
+            template_item=template_item,
             )
         return df_new
 
@@ -129,6 +130,44 @@ class DukitAccessor():
         return df_new
 
 
+    def date_delta(
+            self,
+            reference_date: str | datetime.date | pd.Timestamp = None,
+            reference_col: str = None,
+            linebreak: str = '<br>',
+            verbosity: int = 3,
+            ):
+        return date_delta(
+            self.df,
+            reference_date=reference_date,
+            reference_col=reference_col,
+            linebreak=linebreak,
+            verbosity=verbosity,
+            )
+
+
+    def date_table(
+            self,
+            reference_date: str | datetime.date | pd.Timestamp = None,
+            reference_col: str = None,
+            uid: str = None,
+            upper: int = None,
+            lower: int = None,
+            linebreak: str = '<br>',
+            verbosity: int = 3,
+            ):
+        return date_table(
+            df=self.df,
+            reference_date=reference_date,
+            reference_col=reference_col,
+            uid=uid,
+            upper=upper,
+            lower=lower,
+            linebreak=linebreak,
+            verbosity=verbosity,
+            )
+
+
     def save(
             self,
             path: str,

{dukit-0.2.2 → dukit-0.2.3}/src/dukit/pandas.py

@@ -3,7 +3,7 @@ import pandas as pd
 import numpy as np
 import datetime
 import copy
-from .utils import log
+from .utils import log, GREEN_LIGHT
 from .typing import date_
 
 def get_df() -> pd.DataFrame:
@@ -446,13 +446,12 @@ def embed(
 def collapse(
         df: pd.DataFrame,
         on: str,
-        template='{colname}',
-        line_start='#',
-        line_stop='\n',
+        template_col='{colname}',
+        template_item='#{counter}: {item}\n',
         ):
     df = df.groupby(on).agg(list)
-    df = df.map(lambda x: _to_lines(x, line_start, line_stop))
-    df.columns = [template.format(colname=col) for col in df.columns]
+    df = df.map(lambda x: _to_lines(x, template_item))
+    df.columns = [template_col.format(colname=col) for col in df.columns]
     df.insert(0, on, df.index)
     df.reset_index(drop=True, inplace=True)
     return df
@@ -460,8 +459,7 @@ def collapse(
 
 def _to_lines(
         x,
-        line_start='#',
-        line_stop='\n',
+        template_item='#{counter}: {item}\n',
         ):
     if not isinstance(x, list):
         return x
@@ -470,7 +468,7 @@ def _to_lines(
     else:
         x_str = ''
         for i, item in enumerate(x):
-            x_str += f'{line_start}{i + 1}: {item}{line_stop}'
+            x_str += template_item.format(counter=i + 1, item=item)
     return x_str
 
 
@@ -494,24 +492,18 @@ def transpose(
 
 
 def date_delta(
-        df,
-        reference_date=None,
-        reference_col=None,
-        verbosity=3,
+        df: pd.DataFrame,
+        reference_date: str | datetime.date | pd.Timestamp = None,
+        reference_col: str = None,
+        linebreak: str = '\n',
+        verbosity: int = 3,
         ):
     """
     Calculates the number of days
     between a reference date or col
-    and all other cols in the df.
+    and all other date cols in the df.
     """
 
-    df = df.copy().map(date_).convert_dtypes()
-
-    for col in df.columns:
-        if df[col].isna().all():
-            df.drop(columns=col, inplace=True)
-
-
     if reference_date is None and reference_col is None:
         msg = 'ERROR: no reference date or column provided'
         log(msg, 'dk.date_delta', verbosity)
@@ -522,6 +514,27 @@ def date_delta(
         log(msg, 'dk.date_delta', verbosity)
         return df
 
+
+    df = df.copy()
+    df_dates = (
+        df
+        .copy()
+        .map(date_)
+        .convert_dtypes()
+        )
+    cols_dates = []
+    cols_all = list(df.columns)
+
+    for col in df_dates.columns:
+        if df_dates[col].isna().all():
+            df_dates.drop(columns=col, inplace=True)
+        else:
+            cols_dates.append(col)
+            df.drop(columns=col, inplace=True)
+
+    df = pd.concat([df, df_dates], axis=1)[cols_all]
+
+
     if 'reference_date' in df.columns:
         msg = 'WARNING: column "reference_date" already exists, overwriting'
         log(msg, 'dk.date_delta', verbosity)
@@ -541,10 +554,12 @@ def date_delta(
     df = df[cols_reorder]
 
     for col in df.columns:
+        if col not in cols_dates:
+            continue
         if col == 'reference_date':
             continue
 
-        name = f'days from<br>{reference_col}<br>to<br>{col}'
+        name = f'days from{linebreak}{reference_col}{linebreak}to{linebreak}{col}'
         if name in df.columns:
             log(f'WARNING: column "{name}" already exists, overwriting',
                 'dk.date_delta', verbosity)
@@ -554,10 +569,173 @@ def date_delta(
         #_fix_nas converts accordingly
         col_formatted = df[col].apply(date_).apply(_fix_nas)
         col_diff = col_formatted - df['reference_date']
-        df[name] = col_diff.fillna(pd.NaT).apply(lambda x: x.days)
+        df[name] = (
+            col_diff
+            .fillna(pd.NaT)  #type:ignore (pylance doesnt know date_ enables this)
+            .apply(lambda x: x.days)
+            )
 
     return df
 
 
 def _fix_nas(x):
     return pd.NA if x is pd.NaT else x
+
+
+
+
+def date_table(
+        df: pd.DataFrame,
+        reference_date: str | datetime.date | pd.Timestamp = None,
+        reference_col: str = None,
+        uid: str = None,
+        upper: int = None,
+        lower: int = None,
+        start_at_day1: bool = True,
+        schedule: dict = None,
+        linebreak: str = '\n',
+        verbosity: int = 3,
+        ) -> pd.DataFrame | pd.io.formats.style.Styler:
+    """
+    Arranges all dates into a table,
+    with positions relative to a
+    reference date or column.
+    """
+
+    if reference_date is None and reference_col is None:
+        msg = 'ERROR: no reference date or column provided'
+        log(msg, 'dk.date_delta', verbosity)
+        return df
+
+    if reference_date is not None and reference_col is not None:
+        msg = 'ERROR: both reference date and column provided'
+        log(msg, 'dk.date_delta', verbosity)
+        return df
+
+    df = df.copy()
+
+    if uid:
+        df.index = df[uid]
+
+    if reference_date is not None:
+        reference = pd.Series(
+            date_(reference_date),
+            index=df.index
+            )
+    else:
+        reference = df[reference_col].apply(date_)
+
+    df = (
+        df
+        .copy()
+        .map(date_)
+        .convert_dtypes()
+        )
+
+    for col in df.columns:
+        if df[col].isna().all():
+            df.drop(columns=col, inplace=True)
+
+
+    blanks = {col: pd.NaT for col in df.columns}
+    deltas = pd.DataFrame(
+        blanks,
+        index=df.index,
+        )
+
+    for col in df.columns:
+        #difference only works with pd.NA,
+        #but x.days only works with pd.NaT.
+        #_fix_nas converts accordingly
+        col_formatted = df[col].apply(date_).apply(_fix_nas)
+        col_diff = col_formatted - reference
+        deltas[col] = (
+            col_diff
+            .fillna(pd.NaT)  #type:ignore (pylance doesnt know date_ enables this)
+            .apply(lambda x: x.days)
+            )
+
+    deltas = deltas.T
+
+
+    if upper is None:
+        upper = int(deltas.max().max())
+    if lower is None:
+        lower = int(deltas.min().min())
+
+    days = list(range(lower, upper + 1))
+    if start_at_day1:
+        days.remove(0)
+    days_df = pd.DataFrame({'days': days})
+
+    df_timeline = days_df.copy()
+    if schedule is not None:
+        df_schedule = pd.DataFrame({
+            'days': list(schedule.values()),
+            'planned': list(schedule.keys()),
+            })
+        df_timeline = df_timeline.merge(
+            df_schedule,
+            on='days',
+            how='left',
+            ).fillna('')
+
+    for col in deltas.columns:
+        events = pd.DataFrame({
+            'days': deltas[col].astype('Int64'),
+            col: deltas.index,
+            })
+
+        if start_at_day1:
+            events = _shift_day0(events)
+
+        merged = days_df.copy().merge(events, on='days', how='left').fillna('')
+        collapsed = collapse(
+            merged,
+            on='days',
+            template_item='{item}' + linebreak,
+            )
+        df_timeline = df_timeline.merge(
+            collapsed,
+            on='days',
+            how='left',
+            )
+
+    if schedule is not None:
+        df_timeline = _highlight_schedule(df_timeline, schedule)
+
+    return df_timeline
+
+
+def _shift_day0(df: pd.DataFrame) -> pd.DataFrame:
+    if 'days' not in df.columns:
+        return df
+    if 0 not in df['days'].values:
+        return df
+    mask_positive = df['days'] >= 0
+    df.loc[mask_positive, 'days'] += 1
+    return df
+
+
+def _highlight_schedule(
+        df: pd.DataFrame,
+        schedule: dict,
+        ) -> pd.io.formats.style.Styler:
+
+    days_highlight = list(schedule.values())
+
+    df_style = pd.DataFrame(
+        'text-align: left;',
+        columns=df.columns,
+        index=df.index
+        )
+
+    mask_highlight = df['days'].isin(days_highlight)
+    df_style.loc[mask_highlight, :] += f'background-color: {GREEN_LIGHT};'
+
+    df_styled = df.style.apply(
+        lambda x: df_style,
+        axis=None,
+        )
+
+    return df_styled

{dukit-0.2.2 → dukit-0.2.3}/src/dukit/qlang/symbols.py

@@ -6075,8 +6075,20 @@ class ViewParserHelp(Symbol):
                 )
             print('\n\n')
 
+
         if not q.op.operator:
 
+            flags = _get_valid_operators(
+                q.op,
+                'flags',
+                cols_show
+                )
+            _print_or_display(
+                'flags:',
+                flags[cols_show],
+                )
+            print('\n\n')
+
             generic_ops = _get_valid_operators(
                 q.op,
                 'generic',
@@ -6657,7 +6669,7 @@ class FlagNegate(Symbol):
     >>> df.dk.qs(r'id   %%!>20000')
     """
     name = 'FlagNegate'
-    category = 'syntax'
+    category = 'flag'
     regex = (r'!',)
 
     def parse(self, q: Query) -> Query:
@@ -6668,6 +6680,28 @@ class FlagNegate(Symbol):
 
 
 
+class FlagIndex(Symbol):
+    """
+    Condition is applied to the index
+    instead of the values.
+
+    Examples
+    --------
+    >>> df.dk.qs(r'%§0')
+    >>> df.dk.qs(r'name  §3')
+    """
+    name = 'FlagIndex'
+    category = 'flag'
+    regex = (r'§',)
+
+    def parse(self, q: Query) -> Query:
+        q = _preparse_for_getter(q)
+        q.op.flags['index'] = 'apply condition to the index'
+        return q
+
+
+
+
 class FlagColref(Symbol):
     """
     interpret args as col references
@@ -6678,7 +6712,7 @@ class FlagColref(Symbol):
     >>> df.dk.qs(r'age  <@height')
     """
     name = 'FlagColref'
-    category = 'syntax'
+    category = 'flag'
     regex = (r'@',)
 
     def parse(self, q: Query) -> Query:
@@ -6706,7 +6740,7 @@ class Flag(Symbol):
     >>> df.dk.qs(r'age  %%>30 +strict')
     """
     name = 'Flag'
-    category = 'syntax'
+    category = 'flag'
     regex = (r'\+\w+',)
 
     def build(self, str_matched: str) -> 'Symbol':
@@ -6735,6 +6769,7 @@ getters = []
 setters = []
 stylers = []
 viewers = []
+flags = []
 for _obj in list(locals().values()):
     if isinstance(_obj, type) and issubclass(_obj, Symbol):
         if _obj is not Symbol:
@@ -6758,6 +6793,8 @@ for _obj in list(locals().values()):
             elif _obj.category == 'viewer':
                 operators.append(_obj())
                 viewers.append(_obj())
+            elif _obj.category == 'flag':
+                flags.append(_obj())
 
 
 
@@ -6782,6 +6819,7 @@ def as_df(category='all') -> pd.DataFrame:
         'setter',
         'styler',
         'viewer',
+        'flag',
         )
     if category in singular:
         category += 's'
@@ -6804,6 +6842,8 @@ def as_df(category='all') -> pd.DataFrame:
         symbols = stylers
     elif category == 'viewers':
         symbols = viewers
+    elif category == 'flags':
+        symbols = flags
     else:
         raise ValueError(f"Unknown category: {category}")
 
@@ -6813,7 +6853,7 @@ def as_df(category='all') -> pd.DataFrame:
     descriptions = [_get_description(d) for d in docs]
     examples = [_get_example(d) for d in docs]
     categories = [s.category for s in symbols]
-    flags = ['\n'.join(s.flags) for s in symbols]
+    flags_actual = ['\n'.join(s.flags) for s in symbols]
     flags_allowed = ['\n'.join(s.flags_allowed) for s in symbols]
     scopes_allowed = ['\n'.join(s.scopes_allowed) for s in symbols]
     connectors_allowed = ['\n'.join(s.connectors_allowed) for s in symbols]
@@ -6823,7 +6863,7 @@ def as_df(category='all') -> pd.DataFrame:
         'description': descriptions,
         'example': examples,
         'category': categories,
-        'flags': flags,
+        'flags': flags_actual,
         'flags_allowed': flags_allowed,
         'scopes_allowed': scopes_allowed,
         'connectors_allowed': connectors_allowed,
@@ -6844,7 +6884,17 @@ def as_df_styled(category='all') -> pd.io.formats.style.Styler:
 
 
 def _get_lexemes(symbol: Symbol) -> str:
-    string = '\n'.join(symbol.regex).replace('\\', '')
+    show_raw = [
+        'Newline',
+        'Literal',
+        'Whitespace',
+        'Comment',
+        'Flag',
+        ]
+    if symbol.name in show_raw:
+        string = '\n'.join(symbol.regex)
+    else:
+        string = '\n'.join(symbol.regex).replace('\\', '')
     return string