variable-explorer 0.1.0__py3-none-any.whl

variable_explorer/__init__.py

@@ -0,0 +1,7 @@
+"""Variable Explorer - A Spyder/Stata-grade variable explorer for JupyterLab."""
+
+from ._version import __version__
+
+
+def _jupyter_labextension_paths():
+    return [{"src": "labextension", "dest": "variable-explorer"}]

variable_explorer/_version.py

@@ -0,0 +1,4 @@
+# This file is auto-generated by Hatchling. As such, do not:
+#   - modify
+#   - track in version control e.g. be sure to add to .gitignore
+__version__ = VERSION = '0.1.0'

variable_explorer/kernel/__init__.py

@@ -0,0 +1,5 @@
+"""Kernel-side handlers for variable exploration."""
+
+from .comm_handler import init_comm
+
+__all__ = ["init_comm"]

variable_explorer/kernel/comm_handler.py

@@ -0,0 +1,255 @@
+"""Comm target handler for variable exploration."""
+
+from __future__ import annotations
+
+import json
+import traceback
+from typing import Any
+
+from .introspection import get_variable_list
+from .data_provider import get_data_page
+from .statistics import compute_column_stats
+from .editor import edit_cell
+from .serialization import safe_serialize
+
+
+_active_comm = None
+_initialized = False
+
+
+def init_comm():
+    """Register the comm target and post_execute hook. Called from frontend."""
+    global _initialized
+
+    try:
+        from ipykernel.comm import Comm
+        ip = _get_ipython()
+        if ip is None:
+            return
+
+        # Register comm target
+        ip.kernel.comm_manager.register_target(
+            'variable_explorer', _on_comm_open
+        )
+
+        # Register post_execute hook for auto-refresh
+        if not _initialized:
+            ip.events.register('post_execute', _on_post_execute)
+            _initialized = True
+
+    except Exception as e:
+        print(f"Variable Explorer: init_comm error: {e}")
+
+
+def _get_ipython():
+    """Get the current IPython instance."""
+    try:
+        from IPython import get_ipython
+        return get_ipython()
+    except ImportError:
+        return None
+
+
+def _get_user_ns() -> dict:
+    """Get the user namespace."""
+    ip = _get_ipython()
+    if ip is None:
+        return {}
+    return ip.user_ns
+
+
+def _on_comm_open(comm, open_msg):
+    """Handle comm open from frontend."""
+    global _active_comm
+    _active_comm = comm
+    comm.on_msg(_on_comm_msg)
+    comm.on_close(_on_comm_close)
+
+    # Send initial variable list
+    _send_variable_list(comm)
+
+
+def _on_comm_close(msg):
+    """Handle comm close."""
+    global _active_comm
+    _active_comm = None
+
+
+def _on_comm_msg(msg):
+    """Route incoming messages from frontend."""
+    global _active_comm
+    if _active_comm is None:
+        return
+
+    try:
+        data = msg['content']['data']
+        msg_type = data.get('type', '')
+
+        handlers = {
+            'refresh': _handle_refresh,
+            'get_data': _handle_get_data,
+            'get_stats': _handle_get_stats,
+            'get_properties': _handle_get_properties,
+            'edit_cell': _handle_edit_cell,
+        }
+
+        handler = handlers.get(msg_type)
+        if handler:
+            handler(data)
+        else:
+            _send_error(f"Unknown message type: {msg_type}", msg_type)
+
+    except Exception as e:
+        _send_error(f"Handler error: {e}\n{traceback.format_exc()}", 'handler')
+
+
+def _handle_refresh(data: dict):
+    """Refresh the variable list."""
+    _send_variable_list(_active_comm)
+
+
+def _handle_get_data(data: dict):
+    """Get paginated data for a variable."""
+    user_ns = _get_user_ns()
+    var_name = data.get('variable', '')
+    start_row = data.get('startRow', 0)
+    end_row = data.get('endRow', 1000)
+    sort_model = data.get('sortModel')
+
+    if var_name not in user_ns:
+        _send_error(f"Variable '{var_name}' not found", 'get_data')
+        return
+
+    child_key = data.get('childKey')
+    result = get_data_page(var_name, user_ns[var_name], start_row, end_row, sort_model, child_key)
+    _send(_active_comm, result)
+
+
+def _handle_get_stats(data: dict):
+    """Get column statistics for a variable."""
+    user_ns = _get_user_ns()
+    var_name = data.get('variable', '')
+
+    if var_name not in user_ns:
+        _send_error(f"Variable '{var_name}' not found", 'get_stats')
+        return
+
+    result = compute_column_stats(var_name, user_ns[var_name])
+    _send(_active_comm, result)
+
+
+def _handle_get_properties(data: dict):
+    """Get dataset-level properties."""
+    user_ns = _get_user_ns()
+    var_name = data.get('variable', '')
+
+    if var_name not in user_ns:
+        _send_error(f"Variable '{var_name}' not found", 'get_properties')
+        return
+
+    obj = user_ns[var_name]
+    result = _get_properties(var_name, obj)
+    _send(_active_comm, result)
+
+
+def _handle_edit_cell(data: dict):
+    """Edit a cell value."""
+    user_ns = _get_user_ns()
+    var_name = data.get('variable', '')
+    row_index = data.get('rowIndex', 0)
+    column = data.get('column', '')
+    new_value = data.get('newValue', '')
+
+    if var_name not in user_ns:
+        _send_error(f"Variable '{var_name}' not found", 'edit_cell')
+        return
+
+    result = edit_cell(user_ns[var_name], row_index, column, new_value)
+    result['variable'] = var_name
+    _send(_active_comm, result)
+
+
+def _get_properties(var_name: str, obj: Any) -> dict:
+    """Extract dataset-level properties."""
+    import sys
+
+    result: dict[str, Any] = {
+        'type': 'properties',
+        'variable': var_name,
+        'shape': [],
+        'memoryBytes': 0,
+        'dtypes': {},
+        'indexName': '',
+        'indexDtype': '',
+        'sourceFile': None,
+    }
+
+    try:
+        import pandas as pd
+        if isinstance(obj, pd.DataFrame):
+            result['shape'] = list(obj.shape)
+            result['memoryBytes'] = int(obj.memory_usage(deep=True).sum())
+            result['dtypes'] = {col: str(dtype) for col, dtype in obj.dtypes.items()}
+            result['indexName'] = str(obj.index.name or 'RangeIndex')
+            result['indexDtype'] = str(obj.index.dtype)
+            result['sourceFile'] = obj.attrs.get('source_file')
+            return result
+    except ImportError:
+        pass
+
+    # Generic fallback
+    if hasattr(obj, 'shape'):
+        result['shape'] = list(obj.shape)
+    if hasattr(obj, '__len__'):
+        if not result['shape']:
+            result['shape'] = [len(obj)]
+
+    result['memoryBytes'] = sys.getsizeof(obj)
+    return result
+
+
+def _send_variable_list(comm):
+    """Send the current variable list via comm."""
+    if comm is None:
+        return
+    user_ns = _get_user_ns()
+    variables = get_variable_list(user_ns)
+    _send(comm, {
+        'type': 'variable_list',
+        'variables': variables
+    })
+
+
+def _send_error(message: str, request_type: str = ''):
+    """Send an error message to frontend."""
+    if _active_comm is None:
+        return
+    _send(_active_comm, {
+        'type': 'error',
+        'message': message,
+        'requestType': request_type
+    })
+
+
+def _send(comm, data: dict):
+    """Send data through comm with safe serialization."""
+    try:
+        serialized = safe_serialize(data)
+        comm.send(serialized)
+    except Exception as e:
+        try:
+            comm.send({
+                'type': 'error',
+                'message': f'Serialization error: {e}'
+            })
+        except Exception:
+            pass
+
+
+def _on_post_execute():
+    """Called after every cell execution — send updated variable list."""
+    if _active_comm is not None:
+        try:
+            _send_variable_list(_active_comm)
+        except Exception:
+            pass

variable_explorer/kernel/data_provider.py

@@ -0,0 +1,235 @@
+"""Paginated data extraction from DataFrame-like objects."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .sorter import apply_sort
+
+
+def get_data_page(
+    var_name: str,
+    obj: Any,
+    start_row: int,
+    end_row: int,
+    sort_model: list[dict] | None = None,
+    child_key: str | None = None
+) -> dict:
+    """Extract a page of data from a DataFrame-like object.
+
+    child_key: for containers (list/dict of DataFrames), which child to show.
+               e.g. "0" for list index, or "my_key" for dict key.
+    """
+    import pandas as pd
+
+    # If accessing a child of a container
+    if child_key is not None:
+        obj = _get_child(obj, child_key)
+        if obj is None:
+            return {
+                'type': 'error',
+                'message': f"Child '{child_key}' not found",
+                'requestType': 'get_data'
+            }
+
+    # Convert to DataFrame if needed
+    df = _to_dataframe(obj)
+    if df is None:
+        return {
+            'type': 'error',
+            'message': f"Cannot convert {type(obj).__name__} to tabular format",
+            'requestType': 'get_data'
+        }
+
+    total_rows = len(df)
+
+    if sort_model:
+        df = apply_sort(df, sort_model)
+
+    page = df.iloc[start_row:end_row]
+    columns = _get_column_defs(df)
+    rows = _serialize_rows(page, start_row)
+
+    return {
+        'type': 'data_page',
+        'variable': var_name,
+        'startRow': start_row,
+        'totalRows': total_rows,
+        'columns': columns,
+        'rows': rows,
+    }
+
+
+def _get_child(obj: Any, key: str) -> Any:
+    """Access a child element from a container."""
+    if isinstance(obj, dict):
+        return obj.get(key)
+    if isinstance(obj, (list, tuple)):
+        try:
+            return obj[int(key)]
+        except (ValueError, IndexError):
+            return None
+    return None
+
+
+def _to_dataframe(obj: Any):
+    """Convert various types to pandas DataFrame."""
+    try:
+        import pandas as pd
+
+        if isinstance(obj, pd.DataFrame):
+            return obj
+
+        if isinstance(obj, pd.Series):
+            return obj.to_frame()
+
+        # numpy array
+        try:
+            import numpy as np
+            if isinstance(obj, np.ndarray):
+                if obj.ndim == 1:
+                    return pd.DataFrame({'values': obj})
+                elif obj.ndim == 2:
+                    return pd.DataFrame(obj, columns=[f'col_{i}' for i in range(obj.shape[1])])
+        except ImportError:
+            pass
+
+        # Dict types
+        if isinstance(obj, dict):
+            values = list(obj.values())
+            if not values:
+                return pd.DataFrame()
+
+            # Dict of lists → standard conversion
+            if all(isinstance(v, (list, tuple)) for v in values):
+                return pd.DataFrame(obj)
+
+            # Dict of dicts → each key becomes a row
+            if all(isinstance(v, dict) for v in values):
+                return pd.DataFrame.from_dict(obj, orient='index')
+
+            # Dict of DataFrames → show summary table
+            if all(isinstance(v, pd.DataFrame) for v in values):
+                rows = []
+                for k, v in obj.items():
+                    rows.append({
+                        'key': str(k),
+                        'type': 'DataFrame',
+                        'rows': len(v),
+                        'columns': len(v.columns),
+                        'memory': f"{v.memory_usage(deep=True).sum():,.0f} B",
+                        'column_names': ', '.join(str(c) for c in v.columns[:10]),
+                    })
+                return pd.DataFrame(rows)
+
+            # Dict with scalar values → single-row or key-value table
+            try:
+                return pd.DataFrame([obj])
+            except Exception:
+                return pd.DataFrame({
+                    'key': [str(k) for k in obj.keys()],
+                    'value': [str(v) for v in obj.values()],
+                    'type': [type(v).__name__ for v in obj.values()]
+                })
+
+        # List types
+        if isinstance(obj, (list, tuple)):
+            if not obj:
+                return pd.DataFrame()
+
+            # List of DataFrames → show summary table
+            if all(isinstance(v, pd.DataFrame) for v in obj):
+                rows = []
+                for i, v in enumerate(obj):
+                    rows.append({
+                        'index': i,
+                        'type': 'DataFrame',
+                        'rows': len(v),
+                        'columns': len(v.columns),
+                        'memory': f"{v.memory_usage(deep=True).sum():,.0f} B",
+                        'column_names': ', '.join(str(c) for c in v.columns[:10]),
+                    })
+                return pd.DataFrame(rows)
+
+            # List of dicts → standard conversion (JSON-like data)
+            if all(isinstance(v, dict) for v in obj):
+                return pd.DataFrame(obj)
+
+            # List of lists/tuples → tabular
+            if all(isinstance(v, (list, tuple)) for v in obj):
+                max_cols = max(len(v) for v in obj)
+                cols = [f'col_{i}' for i in range(max_cols)]
+                # Pad shorter rows with None
+                padded = [list(v) + [None] * (max_cols - len(v)) for v in obj]
+                return pd.DataFrame(padded, columns=cols)
+
+            # Simple list of scalars → single column
+            return pd.DataFrame({'value': obj})
+
+        return None
+
+    except Exception:
+        return None
+
+
+def _get_column_defs(df) -> list[dict]:
+    """Build column definitions from a DataFrame."""
+    import pandas as pd
+
+    columns = []
+    for col in df.columns:
+        dtype = df[col].dtype
+        columns.append({
+            'name': str(col),
+            'dtype': str(dtype),
+            'isNumeric': pd.api.types.is_numeric_dtype(dtype) and not pd.api.types.is_bool_dtype(dtype),
+            'isBool': pd.api.types.is_bool_dtype(dtype),
+            'isDatetime': pd.api.types.is_datetime64_any_dtype(dtype),
+        })
+    return columns
+
+
+def _serialize_rows(page, start_row: int) -> list[dict]:
+    """Serialize DataFrame rows to list of dicts."""
+    rows = []
+    for idx, (_, row) in enumerate(page.iterrows()):
+        record: dict[str, Any] = {'__row_index__': start_row + idx}
+        for col, val in row.items():
+            record[str(col)] = _serialize_value(val)
+        rows.append(record)
+    return rows
+
+
+def _serialize_value(val: Any) -> Any:
+    """Convert a single value to JSON-safe type."""
+    if val is None:
+        return None
+
+    import pandas as pd
+    import numpy as np
+
+    if pd.isna(val):
+        return None
+
+    if isinstance(val, (np.integer,)):
+        return int(val)
+    if isinstance(val, (np.floating,)):
+        v = float(val)
+        if np.isnan(v) or np.isinf(v):
+            return None
+        return v
+    if isinstance(val, (np.bool_,)):
+        return bool(val)
+
+    if isinstance(val, pd.Timestamp):
+        return val.isoformat()
+    if hasattr(val, 'isoformat'):
+        return val.isoformat()
+
+    if isinstance(val, bytes):
+        return val.decode('utf-8', errors='replace')
+
+    if isinstance(val, str) and len(val) > 500:
+        return val[:500] + '...'
+
+    return val

variable_explorer/kernel/editor.py

@@ -0,0 +1,88 @@
+"""Cell value editing — write back to the DataFrame in the kernel."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def edit_cell(df: Any, row_index: int, column: str, new_value: str) -> dict:
+    """Modify a cell value in a DataFrame.
+
+    Returns a result dict with success/error info.
+    """
+    try:
+        import pandas as pd
+
+        if not isinstance(df, pd.DataFrame):
+            return {
+                'type': 'edit_result',
+                'success': False,
+                'rowIndex': row_index,
+                'column': column,
+                'error': 'Only DataFrame editing is supported'
+            }
+
+        if column not in df.columns:
+            return {
+                'type': 'edit_result',
+                'success': False,
+                'rowIndex': row_index,
+                'column': column,
+                'error': f"Column '{column}' not found"
+            }
+
+        # Convert value to match column dtype
+        dtype = df[column].dtype
+        converted = _convert_value(new_value, dtype)
+
+        # Apply the edit
+        df.iat[row_index, df.columns.get_loc(column)] = converted
+
+        return {
+            'type': 'edit_result',
+            'success': True,
+            'rowIndex': row_index,
+            'column': column,
+        }
+
+    except Exception as e:
+        return {
+            'type': 'edit_result',
+            'success': False,
+            'rowIndex': row_index,
+            'column': column,
+            'error': str(e)
+        }
+
+
+def _convert_value(value_str: str, dtype: Any) -> Any:
+    """Convert a string value to the appropriate dtype."""
+    import pandas as pd
+    import numpy as np
+
+    # Handle empty/null
+    if value_str in ('', 'null', 'None', 'NaN', 'nan', 'NA'):
+        if pd.api.types.is_numeric_dtype(dtype):
+            return np.nan
+        if pd.api.types.is_bool_dtype(dtype):
+            return pd.NA
+        return None
+
+    # Bool
+    if pd.api.types.is_bool_dtype(dtype):
+        return value_str.lower() in ('true', '1', 'yes')
+
+    # Integer
+    if pd.api.types.is_integer_dtype(dtype):
+        return dtype.type(int(float(value_str)))
+
+    # Float
+    if pd.api.types.is_float_dtype(dtype):
+        return dtype.type(float(value_str))
+
+    # Datetime
+    if pd.api.types.is_datetime64_any_dtype(dtype):
+        return pd.Timestamp(value_str)
+
+    # String/object
+    return value_str