qutePandas 1.0.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

qutePandas/__init__.py

@@ -1,11 +1,11 @@
-"""
-qutePandas - A pandas-like library for q/kdb+
-"""
-
 import os
+import sys
 
-def _setup_environment():
-    """Validates and sets up the environment for PyKX."""
+def _setup_pykx_environment():
+    """
+    Sets up the environment for PyKX BEFORE any PyKX imports.
+    This must run before importing any module that uses PyKX.
+    """
     root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
     env_path = os.path.join(root, ".env")
     
@@ -20,17 +20,47 @@ def _setup_environment():
                     if len(parts) == 2:
                         key, value = parts
                         value = value.strip().strip('"').strip("'")
+                        # Only set if not already in environment
                         if key.strip() and key.strip() not in os.environ:
                             os.environ[key.strip()] = value
-
+    
+    # Check for valid license files in priority order
     qutepandas_home = os.path.expanduser("~/.qutepandas")
     local_kdb = os.path.join(root, "kdb_lic")
+    
+    license_found = False
+    valid_license_path = None
+    
     if os.path.exists(os.path.join(local_kdb, "kc.lic")):
-        os.environ['QLIC'] = local_kdb
+        valid_license_path = local_kdb
+        license_found = True
     elif os.path.exists(os.path.join(qutepandas_home, "kc.lic")):
-        os.environ['QLIC'] = qutepandas_home
+        valid_license_path = qutepandas_home
+        license_found = True
+    
+    if license_found and valid_license_path:
+        os.environ['QLIC'] = valid_license_path
+        os.environ.pop('PYKX_UNLICENSED', None)
+    elif 'QLIC' in os.environ:
+        qlic_path = os.environ['QLIC']
+        if not (os.path.exists(os.path.join(qlic_path, "kc.lic")) or 
+                os.path.exists(os.path.join(qlic_path, "k4.lic"))):
+            del os.environ['QLIC']
+            if 'QHOME' in os.environ:
+                del os.environ['QHOME']
+            license_found = False
+    
+    if not license_found and 'PYKX_UNLICENSED' not in os.environ:
+        os.environ['PYKX_UNLICENSED'] = 'true'
+    
+    if 'PYKX_RELEASE_GIL' not in os.environ:
+        os.environ['PYKX_RELEASE_GIL'] = 'true'
+    
+    if 'PYKX_ENFORCE_EMBEDDED_IMPORT' not in os.environ:
+        os.environ['PYKX_ENFORCE_EMBEDDED_IMPORT'] = '0'
+
+_setup_pykx_environment()
 
-_setup_environment()
 from .core.dataframe import DataFrame
 from .core.connection import connect, get_license_info, install_license
 from .core.display import py, np, pd, pa, pt, print
@@ -59,4 +89,4 @@ from .introspection.dtypes import dtypes
 
 from .indexing import loc, iloc
 
-__version__ = "1.0.0"
+__version__ = "1.1.1"

qutePandas/apply/__init__.py

@@ -0,0 +1,4 @@
+from .apply import apply
+from .apply_col import apply_col
+
+__all__ = ['apply', 'apply_col'] 

qutePandas/apply/apply.py

@@ -0,0 +1,76 @@
+import pykx as kx
+import pandas as pd
+from ..utils import _ensure_q_table, _handle_return
+
+_VECTORIZED_AXIS1 = {
+    'sum': '{sum (0^) each value flip x}',
+}
+
+
+def apply(df, func, axis=0, return_type='q'):
+    """
+    Applies function to DataFrame along specified axis.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame or pykx.Table
+        Input DataFrame.
+    func : callable or str
+        Function to apply.
+    axis : int, default 0
+        Axis along which to apply function (0=columns, 1=rows).
+    return_type : str, default 'q'
+        Desired return type ('p' or 'q').
+
+    Returns
+    -------
+    pandas.DataFrame or pandas.Series or pykx.Table or pykx.K
+        Result of applying function.
+    """
+    try:
+        q_table = _ensure_q_table(df)
+        
+        if len(q_table) == 0:
+            if axis == 1:
+                return _handle_return(kx.toq([]), return_type)
+            if not isinstance(func, str):
+                pdf = df if isinstance(df, pd.DataFrame) else q_table.pd()
+                return pdf.apply(func, axis=axis)
+            elif func == "sum":
+                cols = kx.q("cols", q_table).py()
+                result = kx.toq({c: 0 for c in cols})
+                ret = _handle_return(result, return_type)
+                return pd.Series(ret) if return_type == 'p' else ret
+
+        if isinstance(func, str):
+            if axis == 1 and func in _VECTORIZED_AXIS1:
+                result = kx.q(_VECTORIZED_AXIS1[func], q_table)
+            elif axis == 1:
+                result = kx.q(f"{{({func}) each x}}", q_table)
+            else:
+                result = kx.q(f"{{({func}) each flip x}}", q_table)
+                
+
+        else:
+            if axis == 1:
+                pdf = q_table.pd()
+                res_list = [func(row) for _, row in pdf.iterrows()]
+                result = kx.toq(res_list)
+                
+            else:
+                cols = kx.q("cols", q_table).py()
+                
+                res_dict = {}
+                for col in cols:
+                    col_data = kx.q(f"{{x`{col}}}", q_table).pd()
+                    res_dict[col] = func(col_data)
+                    
+                result = kx.toq(res_dict)
+            
+        ret = _handle_return(result, return_type)
+        if return_type == 'p' and isinstance(ret, dict):
+            return pd.Series(ret)
+        return ret
+
+    except Exception as e:
+        raise RuntimeError(f"Failed to apply function: {e}")

qutePandas/apply/apply_col.py

@@ -0,0 +1,50 @@
+import pykx as kx
+import pandas as pd
+import numpy as np
+from ..utils import _ensure_q_table, _handle_return
+
+
+def apply_col(df, col, func, return_type='q'):
+    """
+    Applies function to a single column of DataFrame.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame or pykx.Table
+        Input DataFrame.
+    col : str
+        Column name to apply function to.
+    func : callable or str
+        Function to apply to the column. If string, applied as q function string.
+    return_type : str, default 'q'
+        Desired return type ('p' or 'q').
+
+    Returns
+    -------
+    pandas.DataFrame or pykx.Table
+        DataFrame with function applied to specified column.
+    """
+    try:
+        q_table = _ensure_q_table(df)
+        if len(q_table) == 0:
+            return _handle_return(q_table, return_type)
+        
+        if isinstance(func, str):
+            result = kx.q(f"{{update {col}:({func}) each {col} from x}}", q_table)
+        else:
+            col_data = kx.q(f"{{x`{col}}}", q_table).py()
+            
+            if hasattr(col_data, 'apply'):
+                new_data = col_data.apply(func)
+            elif isinstance(col_data, list):
+                new_data = [func(x) for x in col_data]
+            else:
+                new_data = np.vectorize(func)(col_data)
+            
+            q_new_data = kx.toq(new_data)
+            result = kx.q(f"{{update {col}:y from x}}", q_table, q_new_data)
+            
+        return _handle_return(result, return_type)
+    except Exception as e:
+        raise RuntimeError(f"Failed to apply function to column {col}: {e}")
+ 

qutePandas/cleaning/__init__.py

@@ -0,0 +1,6 @@
+from .dropna import dropna
+from .dropna_col import dropna_col
+from .fillna import fillna
+from .remove_duplicates import remove_duplicates
+
+__all__ = ['dropna', 'dropna_col', 'fillna', 'remove_duplicates']

qutePandas/cleaning/dropna.py

@@ -0,0 +1,26 @@
+import pykx as kx
+import pandas as pd
+from ..utils import _ensure_q_table, _handle_return
+
+def dropna(df, return_type='q'):
+    """
+    Drops any row containing null values.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame or pykx.Table
+        Input DataFrame.
+    return_type : str, default 'q'
+        Desired return type ('p' for pandas, 'q' for kdb+).
+
+    Returns
+    -------
+    pandas.DataFrame or pykx.Table
+        DataFrame with null rows removed.
+    """
+    try:
+        q_table = _ensure_q_table(df)
+        result = kx.q("{select from x where not any null each value flip x}", q_table)
+        return _handle_return(result, return_type)
+    except Exception as e:
+        raise RuntimeError(f"Failed to dropna: {e}")

qutePandas/cleaning/dropna_col.py

@@ -0,0 +1,28 @@
+import pykx as kx
+import pandas as pd
+from ..utils import _ensure_q_table, _handle_return
+
+def dropna_col(df, col, return_type='q'):
+    """
+    Drops rows where a specific column is null.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame or pykx.Table
+        Input DataFrame.
+    col : str
+        Column name to check for nulls.
+    return_type : str, default 'q'
+        Desired return type ('p' for pandas, 'q' for kdb+).
+
+    Returns
+    -------
+    pandas.DataFrame or pykx.Table
+        DataFrame with filtered rows.
+    """
+    try:
+        q_table = _ensure_q_table(df)
+        result = kx.q("{[t; c] select from t where not null t c}", q_table, kx.SymbolAtom(col))
+        return _handle_return(result, return_type)
+    except Exception as e:
+        raise RuntimeError(f"Failed to dropna from column {col}: {e}")

qutePandas/cleaning/fillna.py

@@ -0,0 +1,49 @@
+import pykx as kx
+import pandas as pd
+from ..utils import _ensure_q_table, _handle_return
+
+def fillna(df, col_or_values, fill_value=None, return_type='q'):
+    """
+    Fills null values in specified columns.
+
+    Can be called in two ways:
+        fillna(df, values_dict, return_type='q')
+        fillna(df, col_name, fill_value, return_type='q')
+
+    Parameters
+    ----------
+    df : pandas.DataFrame or pykx.Table
+        Input DataFrame.
+    col_or_values : str or dict
+        If str, the column name to fill (requires fill_value).
+        If dict, a mapping of column names to fill values.
+    fill_value : scalar, optional
+        The value to fill nulls with when col_or_values is a column name.
+    return_type : str, default 'q'
+        Desired return type ('p' or 'q').
+
+    Returns
+    -------
+    pandas.DataFrame or pykx.Table
+        DataFrame with nulls filled.
+    """
+    try:
+        if isinstance(col_or_values, str):
+            if fill_value is None:
+                raise ValueError("fill_value is required when col_or_values is a column name")
+            values = {col_or_values: fill_value}
+        elif isinstance(col_or_values, dict):
+            values = col_or_values
+        else:
+            raise ValueError("col_or_values must be a column name (str) or a dictionary")
+
+        q_table = _ensure_q_table(df)
+        result = q_table
+
+        for col, val in values.items():
+            fill_val = f'`{val}' if isinstance(val, str) else str(val)
+            result = kx.q(f"{{update {col}:{fill_val}^{col} from x}}", result)
+
+        return _handle_return(result, return_type)
+    except Exception as e:
+        raise RuntimeError(f"Failed to fillna: {e}")

qutePandas/cleaning/remove_duplicates.py

@@ -0,0 +1,28 @@
+import pykx as kx
+import pandas as pd
+from ..utils import _ensure_q_table, _handle_return
+
+
+def remove_duplicates(df, return_type='q'):
+    """
+    Removes duplicate rows from the DataFrame, keeping the first occurrence.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame or pykx.Table
+        Input DataFrame.
+    return_type : str, default 'q'
+        Desired return type ('p' or 'q').
+
+    Returns
+    -------
+    pandas.DataFrame or pykx.Table
+        DataFrame with duplicate rows removed.
+    """
+    try:
+        q_table = _ensure_q_table(df)
+        result = kx.q("{distinct x}", q_table)
+        return _handle_return(result, return_type)
+    except Exception as e:
+        raise RuntimeError(f"Failed to remove duplicates from table: {e}")
+ 

qutePandas/core/__init__.py

@@ -0,0 +1,9 @@
+"""
+Core functionality for qutePandas - DataFrame creation and basic operations.
+"""
+
+from .dataframe import DataFrame
+from .display import py, np, pd, pa, pt, print
+from .connection import connect, get_license_info
+
+__all__ = ['DataFrame', 'py', 'np', 'pd', 'pa', 'pt', 'print', 'connect', 'get_license_info'] 

qutePandas/core/connection.py

@@ -0,0 +1,153 @@
+"""
+Connection and license management for qutePandas.
+Enforces strict license validation with fail-fast behavior.
+"""
+
+import os
+import pykx as kx
+import base64
+import shutil
+
+
+def _get_project_lic_dir():
+    """
+    Get the project-local license directory.
+    """
+    root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+    return os.path.join(root, "kdb_lic")
+
+
+def install_license(content, is_base64=True):
+    """
+    Installs a kdb+ license from base64 content or file path.
+
+    Parameters
+    ----------
+    content : str
+        License content (base64) or file path.
+    is_base64 : bool, default True
+        Whether content is base64 encoded.
+
+    Returns
+    -------
+    bool
+        True if license installation was successful.
+    """
+    target_dir = _get_project_lic_dir()
+    if not os.path.exists(target_dir):
+        target_dir = os.path.expanduser("~/.qutepandas")
+        
+    os.makedirs(target_dir, exist_ok=True)
+    target_path = os.path.join(target_dir, "kc.lic")
+    
+    content = content.strip().strip('"').strip("'")
+    
+    if is_base64:
+        padding = len(content) % 4
+        if padding:
+            content += "=" * (4 - padding)
+        content = content.replace('-', '+').replace('_', '/')
+        
+        try:
+            data = base64.b64decode(content)
+        except Exception as e:
+            raise RuntimeError(f"Invalid base64 license content: {e}")
+            
+        try:
+            text_data = data.decode('utf-8')
+            if text_data.startswith('0001'):
+                os.environ['KX_TOKEN'] = text_data
+                os.environ['QTOK'] = text_data
+                if os.path.exists(target_path):
+                    os.remove(target_path)
+                return True
+        except:
+            pass
+            
+        with open(target_path, "wb") as f:
+            f.write(data)
+    else:
+        if not os.path.exists(content):
+            raise RuntimeError(f"License file not found: {content}")
+        shutil.copy(content, target_path)
+    
+    os.environ['QLIC'] = target_dir
+    return True
+
+
+def connect(license_path=None):
+    """
+    Establishes connection to kdb+.
+
+    Parameters
+    ----------
+    license_path : str, optional
+        Path to the license file or directory.
+    """
+    if license_path:
+        if not os.path.exists(license_path):
+            raise RuntimeError(f"License path does not exist: {license_path}")
+        os.environ['QLIC'] = license_path
+    
+    try:
+        kx.q('1+1')
+        return True
+    except:
+        pass
+    
+    token = (os.environ.get('KDB_TOKEN') or os.environ.get('KX_TOKEN', '')).strip()
+    if token:
+        install_license(token)
+        try:
+            kx.q('1+1')
+            return True
+        except:
+            pass
+    
+    possible_paths = [
+        _get_project_lic_dir(),
+        os.path.expanduser("~/.qutepandas"),
+        '/Applications/kdb+:q',
+        os.path.expanduser("~/kdb+"),
+        os.path.expanduser("~/q"),
+        os.path.expanduser("~/.pykx")
+    ]
+    
+    for path in possible_paths:
+        if not os.path.exists(path):
+            continue
+            
+        if os.path.exists(os.path.join(path, "kc.lic")) or os.path.exists(os.path.join(path, "k4.lic")):
+            os.environ['QLIC'] = path
+            try:
+                kx.q('1+1')
+                return True
+            except:
+                continue
+    
+    raise RuntimeError(
+        "No valid kdb+ license found. "
+        "Set KDB_TOKEN in .env or place kc.lic in local kdb_lic/ or ~/.qutepandas/"
+    )
+
+
+def get_license_info():
+    """
+    Returns current license configuration status.
+    """
+    try:
+        kx.q('1+1')
+        return {
+            'qlic_path': os.environ.get('QLIC', 'Not set'),
+            'qhome_path': os.environ.get('QHOME', 'Not set'),
+            'kx_token_set': 'Yes' if os.environ.get('KX_TOKEN') else 'No',
+            'connection_status': 'Connected'
+        }
+    except Exception as e:
+        return {
+            'qlic_path': os.environ.get('QLIC', 'Not set'),
+            'qhome_path': os.environ.get('QHOME', 'Not set'),
+            'kx_token_set': 'Yes' if os.environ.get('KX_TOKEN') else 'No',
+            'connection_status': 'Failed',
+            'error': str(e)
+        }

qutePandas/core/dataframe.py

@@ -0,0 +1,118 @@
+import pykx as kx
+import pandas as pd
+import atexit
+from ..utils import _handle_return
+
+def DataFrame(data, columns=None):
+    """
+    Creates a qutePandas DataFrame (internal pykx Table).
+
+    Parameters
+    ----------
+    data : array-like, dict, or pandas.DataFrame
+        Data to be stored in the table.
+    columns : list, optional
+        Column names to use if data does not already have them.
+
+    Returns
+    -------
+    pykx.Table
+        The resulting kdb+ table.
+    """
+    try:
+        if isinstance(data, pd.DataFrame):
+            q_res = kx.toq(data)
+        elif isinstance(data, (kx.Table, kx.KeyedTable)):
+            q_res = data
+        elif isinstance(data, dict):
+            q_res = _dict_to_table(data)
+        elif isinstance(data, list) and data and isinstance(data[0], list):
+            q_res = _lists_to_table(data, columns)
+        else:
+            q_res = _data_to_table(data, columns)
+        return _handle_return(q_res)
+    except Exception as e:
+        raise RuntimeError(f"Failed to create kdb+ table: {e}")
+
+
+def _dict_to_table(data_dict):
+    """
+    Converts a dictionary to a kdb+ table.
+
+    Parameters
+    ----------
+    data_dict : dict
+        Dictionary to convert.
+
+    Returns
+    -------
+    pykx.Table
+        The resulting kdb+ table.
+    """
+    try:
+        q_dict = kx.toq(data_dict)
+        return kx.q("{flip x}", q_dict)
+    except Exception as e:
+        raise RuntimeError(f"Failed to create kdb+ table from dict: {e}")
+
+
+def _lists_to_table(data_lists, columns=None):
+    """
+    Converts a list of lists to a kdb+ table.
+
+    Parameters
+    ----------
+    data_lists : list of lists
+        Data to convert.
+    columns : list, optional
+        Column names.
+
+    Returns
+    -------
+    pykx.Table
+        The resulting kdb+ table.
+    """
+    try:
+        if not data_lists:
+            return kx.q("([] )")
+            
+        if columns is None:
+            columns = [f'col_{i}' for i in range(len(data_lists[0]))]
+
+        transposed = list(zip(*data_lists))
+        data_dict = {columns[i]: list(col_data) for i, col_data in enumerate(transposed)}
+        q_dict = kx.toq(data_dict)
+        return kx.q("{flip x}", q_dict)
+    except Exception as e:
+        raise RuntimeError(f"Failed to create kdb+ table from lists: {e}")
+
+
+def _data_to_table(data, columns=None):
+    """
+    Converts generic data to a kdb+ table.
+
+    Parameters
+    ----------
+    data : any
+        Data to convert.
+    columns : list, optional
+        Column names.
+
+    Returns
+    -------
+    pykx.Table
+        The resulting kdb+ table.
+    """
+    try:
+        if hasattr(data, '__iter__') and not isinstance(data, (str, dict)):
+            data_list = list(data)
+            if columns:
+                data_dict = {col: [row[i] if hasattr(row, '__getitem__') else row for row in data_list] 
+                           for i, col in enumerate(columns)}
+                return _dict_to_table(data_dict)
+            else:
+                return kx.toq(pd.DataFrame({'data': data_list}))
+        else:
+            return kx.toq(pd.DataFrame({'data': [data]}))
+    except Exception as e:
+        raise RuntimeError(f"Failed to create kdb+ table from data: {e}")