dukit 0.1.0__py3-none-any.whl

dukit/utils.py

@@ -0,0 +1,288 @@
+
+import pandas as pd
+import string as str_module
+import datetime
+import random
+
+from collections.abc import Iterable
+from IPython.display import display
+from IPython.core.getipython import get_ipython
+from .typing import str_
+
+GREEN = '#6dae51'
+RED = '#f73434'
+
+GREY_LIGHT = '#d3d3d3'
+BLUE_LIGHT = '#87ceeb'
+GREEN_LIGHT = '#c0e7b0'
+ORANGE_LIGHT = '#f7d67c'
+RED_LIGHT = '#f7746a'
+
+
+logs = []
+_levels = {
+    'TRACE': 5,
+    'DEBUG': 4,
+    'INFO': 3,
+    'WARNING': 2,
+    'ERROR': 1,
+    }
+_colors = {
+    'TRACE': GREY_LIGHT,
+    'DEBUG': BLUE_LIGHT,
+    'INFO': GREEN_LIGHT,
+    'WARNING': ORANGE_LIGHT,
+    'ERROR': RED_LIGHT,
+    }
+_formats = {
+    'TRACE': f'background-color: {_colors["TRACE"]}',
+    'DEBUG': f'background-color: {_colors["DEBUG"]}',
+    'INFO': f'background-color: {_colors["INFO"]}',
+    'WARNING': f'background-color: {_colors["WARNING"]}',
+    'ERROR': f'background-color: {_colors["ERROR"]}',
+    }
+
+
+def _build_log_context(
+        function: str,
+        verbosity=3,
+        **kwargs,
+        ) -> str:
+    lines = [f'function: {function}\n']
+
+    for key, value in kwargs.items():
+
+        if isinstance(value, pd.DataFrame):
+            pass
+        elif isinstance(value, pd.Series):
+            pass
+
+        if isinstance(value, (tuple, list, dict)):
+            value_str = str_(value)
+        elif verbosity <= 3:
+            value_str = repr(value)
+        else:
+            value_str = str(value)
+
+        lines.append(f'{key}: {value_str}')
+
+    return '\n'.join(lines)
+
+
+def log(
+        text: str = None,
+        context: str = '',
+        verbosity: int = None,
+        clear: bool = False,
+        ) -> pd.io.formats.style.Styler | None:
+    """
+    A very basic "logger" meant to be used in place
+    of print() statements in jupyter notebooks.
+    For more extensive logging purposes use a logging module.
+
+
+    examples:
+
+    from dukit import log
+
+    log('trace: this is a trace entry which will be highlighted grey')
+    log('debug: this is a debug entry which will be highlighted blue')
+    log('info: this is a info entry which will be highlighted green')
+    log('warning: this is a warning entry which will be highlighted orange')
+    log('error: this is a error entry which will be highlighted red')
+
+    log(clear=True)  #clear all log entries
+    log()  #return dataframe of log entries
+    dukit.utils.logs  #location of list containing log entries
+
+    """
+    if verbosity == 0:
+        return None
+
+    time = datetime.datetime.now()
+
+    if clear:
+        logs.clear()
+        if verbosity in (None, 3, 4, 5):
+            print('cleared all logs in dukit.util.logs.')
+        return None
+
+    if text is None:
+        if len(logs) == 0:
+            return pd.DataFrame().style
+        df = pd.DataFrame(logs)
+        col_format = df['level'].replace(_formats)
+        df_style = pd.DataFrame(
+            {col: col_format for col in df.columns},
+            index=df.index,
+            )
+        df_styled = (
+            df
+            .style
+            .apply(lambda x: df_style, axis=None)
+            .set_properties(None, **{'text-align': 'left'})
+            )
+        return df_styled
+
+    color = GREEN_LIGHT
+    level = 'INFO'
+    level_int = 3
+    text_temp = text.upper()
+
+    #detect logging level
+    for level_temp in _levels.keys():
+        if text_temp.startswith(level_temp):
+            level = level_temp
+            level_int = _levels[level]
+            color = _colors[level]
+            text = text[len(level_temp):].strip()
+            if text and text[0] in [':', '-', ' ']:
+                text = text[1:].strip()
+            break
+
+    if verbosity is None:
+        verbosity = level_int
+
+    if len(logs) == 0:
+        total_ms = 0.0
+        delta_ms = 0.0
+    else:
+        total_ms = datetime.datetime.now() - logs[0]['time']
+        delta_ms = datetime.datetime.now() - logs[-1]['time']
+        total_ms = total_ms.total_seconds() * 1000
+        delta_ms = delta_ms.total_seconds() * 1000
+    message = {
+        'level': level,
+        'text': text,
+        'context': context,
+        'time': time,
+        'total_ms': total_ms,
+        'delta_ms': delta_ms,
+        }
+
+    if level_int <= verbosity:
+        logs.append(message)
+
+        #for jupyter
+        if get_ipython().__class__.__name__ == 'ZMQInteractiveShell':  #pragma: no cover
+
+            #make html friendly
+            message['text'] = (
+                message['text']
+                .replace('<br>', '\n')
+                .replace('<', '&lt;')
+                .replace('>', '&gt;')
+                .replace('\n', '<br>')
+                .replace('\t', '&emsp;')
+                )
+            message['context'] = (
+                message['context']
+                .replace('<br>', '\n')
+                .replace('<', '&lt;')
+                .replace('>', '&gt;')
+                .replace('\n', '<br>')
+                .replace('\t', '&emsp;')
+                )
+            message_df = pd.DataFrame(message, index=[len(logs)])
+
+            kwargs_format = {
+                'whitespace': 'normal',
+                'text-align': 'left',
+                }
+            display(
+                message_df
+                .style
+                .hide(axis=1)
+                .apply(lambda x: [f'background-color: {color}' for i in x], axis=1)
+                .set_properties(None, **kwargs_format)
+                )
+
+        #everywhere else
+        else:
+            context_formatted = context.replace('\n', '\n    ')
+            text_formatted = text.replace('\n', '\n    ')
+            string = (
+                f'{level} log message:\n'
+                f'  time: {time}\n'
+                f'  time since first log: {total_ms:.2f} ms\n'
+                f'  time since last log: {delta_ms:.2f} ms\n'
+                f'  context:{context_formatted}\n'
+                f'  text:\n    """{text_formatted}"""\n'
+                )
+            print(string)
+
+
+
+def ensure_unique_string(
+        string: str,
+        taken: Iterable[str],
+        strategy: str = 'increment',
+        ) -> str:
+    """
+    Ensure that a string is unique within a set of taken strings.
+
+    Parameters
+    ----------
+    string : The original string to be made unique.
+    taken : An iterable of strings that are already taken.
+    strategy : The strategy to use for making the string unique. Default is 'increment'.
+        * "increment": use incrementing numbers appended to the string.
+        * "random": use random characters appended to the string.
+        * "timestamp": use a timestamp appended to the string.
+        * "datestamp": append the current date to the string.
+            If date is already taken, incrementing numbers are appended.
+        * "prefix=STR": prepend a custom prefix STR to the string.
+            Will keep prepending until the string is unique.
+        * "suffix=STR": append a custom suffix STR to the string.
+            Will keep appending until the string is unique.
+
+    """
+
+    base_string = string
+
+    if strategy == 'increment':
+        counter = 1
+        while string in taken:
+            string = f"{base_string}{counter}"
+            counter += 1
+    elif strategy == 'random':
+        while string in taken:
+            chars = str_module.ascii_letters + str_module.digits
+            rand_str = ''.join(random.choices(chars, k=6))
+            string = f"{base_string}_{rand_str}"
+    elif strategy == 'timestamp':
+        while string in taken:
+            timestamp = datetime.datetime.now().strftime('%Y_%m_%d_%Hh%Mm%Ss')
+            string = f"{base_string}_{timestamp}"
+    elif strategy == 'datestamp':
+        datestamp = datetime.datetime.now().strftime('%Y_%m_%d')
+        string_temp = f"{base_string}_{datestamp}"
+        counter = 1
+        while string_temp in taken:
+            string_temp = f"{base_string}_{datestamp}_v{counter}"
+            counter += 1
+        string = string_temp
+    elif strategy.startswith('prefix='):
+        prefix = strategy.split('=')[1]
+        while string in taken:
+            string = f"{prefix}{string}"
+    elif strategy.startswith('suffix='):
+        suffix = strategy.split('=')[1]
+        while string in taken:
+            string = f"{string}{suffix}"
+    else:
+        raise ValueError(f'Unknown strategy: {strategy}')
+
+    return string
+
+
+
+def now(fmt='%Y_%m_%d'):
+    """
+    alias for datetime.datetime.now().strftime(format_str)
+
+    common format_str options:
+    '%Y_%m_%d_%Hh%Mm%Ss': standard
+    %Y_%b_%d:  3 letter month
+    """
+    return datetime.datetime.now().strftime(fmt)

dukit-0.1.0.dist-info/METADATA

@@ -0,0 +1,113 @@
+Metadata-Version: 2.3
+Name: dukit
+Version: 0.1.0
+Summary: dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas
+Author: MartinVoelkl94
+Author-email: MartinVoelkl94 <martinv16248@gmail.com>
+Requires-Dist: ipython>=9.15.0
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: pandas>=3.0.3
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+
+# dukit - data utilities kit
+
+Provides utilities focused on data exploration, modification and visualization of small but messy datasets in pandas.
+
+
+
+# qlang
+
+A small query language.
+It is implemented as a pandas [accessor extension](https://pandas.pydata.org/docs/development/extending.html), meaning it can be called from a dataframe without any further preparation by calling df.dk.qs() with a valid text string.
+
+example:
+```python
+from dukit import get_df
+
+df = get_df()
+
+#show all patients with ID > 20000
+df.dk.qs(r'id  >20000')
+
+#select all patients whose name contains "john"
+df.dk.qs(r'name  ?john')
+
+#select all patients whose name contains "j" and "a"
+df.dk.qs(r'name  ?j  &&?a')
+
+#select all patients whose name contains "j" or "a"
+df.dk.qs(r'name  ?j  //?a')
+
+#select ages between 18 and 80 and highlight them green
+df.dk.qs(r'age  >18  &&<80  .color(green)')
+```
+
+Take a look at "interactive_demo.ipynb" in the github repo for a quick syntax introduction and more examples.
+
+The query language allows for arbitrary code execution via eval(), please be aware of the risks.
+
+Currently published on [testpypi](https://test.pypi.org/project/dukit/).
+
+<br>
+<br>
+
+
+
+
+# other utilities
+
+
+## logging with dukit.log()
+
+A small logger to be used in notebooks or the REPL. Makes it easier to keep track of outputs in large notebooks by providing color coded output. Does not log to file, but instead to a dataframe which can then be viewed at the end of the notebook.
+<br>
+<br>
+
+
+## dukit.diff()
+
+creates colored diff output for two dataframes, see interactive_demo.ipynb for examples. Works with pandas dataframes, csv files, excel files and excel files with multiple sheets.
+<br>
+<br>
+
+
+## "bashlike" wrappers
+
+While python has functions to achieve similar results as common bash commands, they are often more verbose, less intuitive if you are already used to the bash names and spread out over different modules and different namespaces in those modules (os, os.path, shutil, sys, ...).  
+These wrappers use the same names as the bash commands and offer some additional functionality.
+
+
+available wrappers:  
+- dukit.ls()  
+- dukit.lsr()  
+- dukit.pwd()  
+- dukit.cd()  
+- dukit.cp()  
+- dukit.mkdir()  
+- dukit.isdir()  
+- dukit.isfile()  
+- dukit.ispath()  
+<br>
+<br>
+
+
+## type conversion 
+
+Mostly wrappers for pandas functions but with some additional functionality and generally more lenient handling of edge cases. 
+
+available functions:  
+- dukit.int()  
+- dukit.float()  
+- dukit.num()  
+- dukit.bool()  
+- dukit.date()  
+- dukit.datetime()  
+- dukit.na()  
+- dukit.nk()  
+- dukit.yn()  
+- dukit.type()  
+- dukit.convert()  
+<br>
+<br>
+

dukit-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+dukit/__init__.py,sha256=hcqXqlbc_1cFVTeMwKp43D92JqCxc9cL-5Cm729BMt4,1579
+dukit/_test_utils.py,sha256=-B2po0YjMoIo9DgCKfXdfHSC2NIXqC2jODTXpZPwt_4,10106
+dukit/diffing.py,sha256=zCMLryNvWp1kpwCuZOAzJr9zm-lfh7Ga6xzLAyBbGEU,41718
+dukit/excel.py,sha256=AciWK6rL2G9oWCHLkwqXyt21LeRlnTZvyCefYjJRdvs,3648
+dukit/interfaces.py,sha256=sWgi5IzSUIFDliH7SJPWNi7fFoGmkV4m_gOrfqjLBUw,3283
+dukit/os.py,sha256=Vc7dOOXxwGe1zVhqeik_-Lndk_xOLziql6uaQJaD33k,7233
+dukit/pandas.py,sha256=nSH-7pAUvBqbnz8cF1BRxvNr_OZ-nWo0wMCQ1hYT2g4,13535
+dukit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dukit/qlang/__init__.py,sha256=mArIX3zbwuJ5_ruRmKIOVadaPVtpjFWlGrOGLJCqr1c,1128
+dukit/qlang/engine.py,sha256=YrJMVD0I_Gpe4Q0n4ovhYAhKDYN2RMrmr-clyGLTiBg,12279
+dukit/qlang/symbols.py,sha256=twCMBOJTt8WAWvjbMcKRSyakMZFv5wvnN1ihRH3dgjs,169665
+dukit/typing.py,sha256=7vCtd6c9JeEtNA403CN-DA9_gRAbnFIWGbGQ4K_Ntyc,18354
+dukit/utils.py,sha256=7e-bCPmVSypN-ZCpXRCsOPMicz8ZB-BA7kxkd2deoR8,8827
+dukit-0.1.0.dist-info/WHEEL,sha256=mydTeHxOpFHo-DnYhAd_3ATePms-g4rrYvM7wJK8P-U,80
+dukit-0.1.0.dist-info/METADATA,sha256=PzZBtt3BJEpo9E-O1lNKQowz77ME12M17FVy94B09-8,3144
+dukit-0.1.0.dist-info/RECORD,,

dukit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.10.9
+Root-Is-Purelib: true
+Tag: py3-none-any