open-data-sci 0.1.0__py3-none-any.whl

opendatasci/_utils/async_utils.py

@@ -0,0 +1,11 @@
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+_executor = ThreadPoolExecutor(max_workers=16)
+
+
+async def run_in_executor(func: Any, *args: Any, executor: ThreadPoolExecutor | None = None) -> Any:
+    """Run a blocking callable in a thread pool executor."""
+    loop = asyncio.get_event_loop()
+    return await loop.run_in_executor(executor or _executor, func, *args)

opendatasci/_utils/data_formats.py

@@ -0,0 +1,135 @@
+"""
+File-format detection and extension registry.
+
+Centralising format metadata here means ``LocalWorkspace`` does not grow when new
+formats are added — only this module changes.  Any other module that needs to
+know which extensions are supported can import from here rather than
+re-declaring its own sets.
+"""
+
+from pathlib import Path
+from typing import Literal
+
+EXCEL_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".xlsm", ".xlsb"})
+CSV_EXTENSIONS: frozenset[str] = frozenset({".csv", ".tsv"})
+JSON_EXTENSIONS: frozenset[str] = frozenset({".json", ".jsonl", ".ndjson"})
+PARQUET_EXTENSIONS: frozenset[str] = frozenset({".parquet", ".pq"})
+ORC_EXTENSIONS: frozenset[str] = frozenset({".orc"})
+FEATHER_EXTENSIONS: frozenset[str] = frozenset({".feather", ".arrow"})
+HDF_EXTENSIONS: frozenset[str] = frozenset({".h5", ".hdf", ".hdf5"})
+PICKLE_EXTENSIONS: frozenset[str] = frozenset({".pkl", ".pickle"})
+XML_EXTENSIONS: frozenset[str] = frozenset({".xml"})
+STATA_EXTENSIONS: frozenset[str] = frozenset({".dta"})
+SAS_EXTENSIONS: frozenset[str] = frozenset({".sas7bdat", ".xpt"})
+SPSS_EXTENSIONS: frozenset[str] = frozenset({".sav", ".zsav"})
+TEXT_EXTENSIONS: frozenset[str] = frozenset(
+    {
+        ".txt",
+        ".md",
+        ".log",
+        ".yml",
+        ".yaml",
+        ".toml",
+        ".ini",
+        ".cfg",
+        ".conf",
+        ".rst",
+        ".text",
+    }
+)
+ARCHIVE_EXTENSIONS: frozenset[str] = frozenset({".zip", ".gz", ".bz2", ".xz", ".zst"})
+
+# All extensions that can be loaded directly as data (archives are containers,
+# not data files themselves, so they are tracked separately).
+LOADABLE_EXTENSIONS: frozenset[str] = (
+    EXCEL_EXTENSIONS
+    | CSV_EXTENSIONS
+    | JSON_EXTENSIONS
+    | PARQUET_EXTENSIONS
+    | ORC_EXTENSIONS
+    | FEATHER_EXTENSIONS
+    | HDF_EXTENSIONS
+    | PICKLE_EXTENSIONS
+    | XML_EXTENSIONS
+    | STATA_EXTENSIONS
+    | SAS_EXTENSIONS
+    | SPSS_EXTENSIONS
+    | TEXT_EXTENSIONS
+)
+
+ALL_SUPPORTED_EXTENSIONS: frozenset[str] = LOADABLE_EXTENSIONS | ARCHIVE_EXTENSIONS
+
+FileFormat = Literal[
+    "excel",
+    "csv",
+    "json",
+    "jsonl",
+    "parquet",
+    "orc",
+    "feather",
+    "hdf",
+    "pickle",
+    "xml",
+    "stata",
+    "sas",
+    "spss",
+    "text",
+    "zip",
+    "unknown",
+]
+
+
+def detect_format(path: Path) -> tuple[FileFormat, str | None]:
+    """Detect the file format and compression of *path*.
+
+    Returns:
+        A ``(format, compression)`` tuple.  ``compression`` is the compression
+        scheme without a leading dot (e.g. ``"gz"``) or ``None`` for
+        uncompressed files.
+
+    Examples::
+
+        detect_format(Path("data.csv"))        # ("csv", None)
+        detect_format(Path("data.csv.gz"))     # ("csv", "gz")
+        detect_format(Path("archive.zip"))     # ("zip", None)
+    """
+    suffix = path.suffix.lower()
+    compression: str | None = None
+    actual_suffix = suffix
+
+    if suffix in {".gz", ".bz2", ".xz", ".zst"}:
+        compression = suffix[1:]  # strip leading dot
+        actual_suffix = Path(path.stem).suffix.lower()
+
+    if suffix == ".zip":
+        return "zip", None
+    if actual_suffix in EXCEL_EXTENSIONS:
+        return "excel", compression
+    if actual_suffix in CSV_EXTENSIONS:
+        return "csv", compression
+    if actual_suffix in {".jsonl", ".ndjson"}:
+        return "jsonl", compression
+    if actual_suffix in JSON_EXTENSIONS:
+        return "json", compression
+    if actual_suffix in PARQUET_EXTENSIONS:
+        return "parquet", compression
+    if actual_suffix in ORC_EXTENSIONS:
+        return "orc", compression
+    if actual_suffix in FEATHER_EXTENSIONS:
+        return "feather", compression
+    if actual_suffix in HDF_EXTENSIONS:
+        return "hdf", compression
+    if actual_suffix in PICKLE_EXTENSIONS:
+        return "pickle", compression
+    if actual_suffix in XML_EXTENSIONS:
+        return "xml", compression
+    if actual_suffix in STATA_EXTENSIONS:
+        return "stata", compression
+    if actual_suffix in SAS_EXTENSIONS:
+        return "sas", compression
+    if actual_suffix in SPSS_EXTENSIONS:
+        return "spss", compression
+    if actual_suffix in TEXT_EXTENSIONS:
+        return "text", compression
+
+    return "unknown", compression

opendatasci/_utils/hash_utils.py

@@ -0,0 +1,52 @@
+import asyncio
+from pathlib import Path
+from typing import cast
+
+import xxhash
+
+from opendatasci._utils.async_utils import run_in_executor
+
+_HASHING_DATA_CHUNK_SIZE = 32 * 1024 * 1024  # 32MB
+
+
+def _is_visible(file: Path, root: Path) -> bool:
+    """Return ``True`` if *file* is not under a hidden (dot-prefixed) path."""
+    return not any(part.startswith(".") for part in file.relative_to(root).parts)
+
+
+async def hash_path(path: Path) -> str:
+    """Return the 128-bit xxh3 hex digest of *path* (file or directory)."""
+    if path.is_file():
+        return cast(str, await run_in_executor(hash_file, path))
+    return await hash_dir(path)
+
+
+def hash_file(path: Path, chunk_size: int | None = None) -> str:
+    """Return the 128-bit xxh3 hex digest of a file's contents."""
+    chunk_size = chunk_size or _HASHING_DATA_CHUNK_SIZE
+    h = xxhash.xxh3_128()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(chunk_size), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+async def hash_dir(path: Path) -> str:
+    """Return a 128-bit hash of a directory tree, excluding hidden/system files."""
+    files: list[Path] = sorted(
+        (f for f in path.rglob("*") if f.is_file() and _is_visible(f, path)),
+        key=lambda p: str(p),
+    )
+
+    if not files:
+        return xxhash.xxh3_128(b"").hexdigest()
+
+    coros = [run_in_executor(hash_file, f) for f in files]
+    hashes: list[str] = await asyncio.gather(*coros)
+
+    h_prev: bytes = b""
+    for h in hashes:
+        file_hash = xxhash.xxh3_128(bytes.fromhex(h)).digest()
+        h_prev = xxhash.xxh3_128(h_prev + file_hash).digest()
+
+    return h_prev.hex()

opendatasci/_utils/langchain_utils.py

@@ -0,0 +1,155 @@
+"""LangChain / LangGraph message and state utilities."""
+
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage, ToolMessage
+from langgraph.types import StateSnapshot
+
+
+def get_message_text_content(msg: BaseMessage) -> str:
+    """Extract plain text from a message, skipping non-text blocks (e.g. thinking)."""
+    content = msg.content
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return str(content)
+    parts: list[str] = []
+    for block in content:
+        if isinstance(block, dict):
+            if block.get("type") == "text":
+                parts.append(block.get("text", ""))
+        elif isinstance(block, str):
+            parts.append(block)
+    return "\n".join(parts)
+
+
+def render_turn(messages: list[BaseMessage]) -> str:
+    """Render a sequence of messages (a full turn or an ongoing slice) as a readable string.
+
+    Each message type is formatted as follows:
+
+    - ``HumanMessage`` → ``User: <content>``
+    - ``AIMessage`` with tool calls → ``[TOOL CALL: <name>]\\n<args>`` (one entry per call)
+    - ``AIMessage`` without tool calls → ``Agent: <text>``
+    - ``ToolMessage`` → ``[TOOL OUTPUT]\\n<content>``
+
+    Other message types are silently skipped.
+    """
+    parts: list[str] = []
+    for msg in messages:
+        if isinstance(msg, HumanMessage):
+            content = msg.content if isinstance(msg.content, str) else str(msg.content)
+            content = content.strip()
+            if content:
+                parts.append(f"User: {content}")
+        elif isinstance(msg, AIMessage) and msg.tool_calls:
+            for tc in msg.tool_calls:
+                args_str = str(tc.get("args", {}))
+                parts.append(f"[TOOL CALL: {tc['name']}]\n{args_str}")
+        elif isinstance(msg, AIMessage):
+            text = get_message_text_content(msg).strip()
+            if text:
+                parts.append(f"Agent: {text}")
+        elif isinstance(msg, ToolMessage):
+            content = msg.content if isinstance(msg.content, str) else str(msg.content)
+            parts.append(f"[TOOL OUTPUT]\n{content}")
+    return "\n\n".join(parts) if parts else "(no messages)"
+
+
+def render_turns(turns: list[list[BaseMessage]]) -> str:
+    """Render a list of turns as a single readable string.
+
+    Each turn is rendered via :func:`render_turn` and separated by a blank line.
+    Returns ``"(no conversation to render)"`` when *turns* is empty.
+    """
+    rendered = [render_turn(t) for t in turns if t]
+    return "\n\n".join(rendered) if rendered else "(no conversation to render)"
+
+
+def prepend_messages(
+    history: list[BaseMessage],
+    messages: list[BaseMessage],
+) -> list[BaseMessage]:
+    """Prepend *messages* to *history*, dropping any existing SystemMessages from *history*."""
+    non_system = [m for m in history if not isinstance(m, SystemMessage)]
+    return messages + non_system
+
+
+def is_interrupt_state_snapshot(state: StateSnapshot) -> bool:
+    """Return True if *state* contains at least one pending LangGraph interrupt."""
+    return any(intr for task in state.tasks for intr in task.interrupts)
+
+
+def is_final_ai_message(msg: BaseMessage) -> bool:
+    """Return True if *msg* is an AIMessage with no pending tool calls."""
+    return isinstance(msg, AIMessage) and not bool(getattr(msg, "tool_calls", None))
+
+
+def get_ongoing_turn_messages(messages: list[BaseMessage]) -> list[BaseMessage]:
+    """Return the messages of the current (still-in-progress) conversation turn.
+
+    Uses the same turn-boundary rules as ``get_last_turn_messages``: the turn
+    begins at the most recent turn-opening ``HumanMessage`` (one whose
+    ``additional_kwargs`` does **not** set ``is_input_on_interrupt`` to ``True``).
+
+    Raises:
+        ValueError: if the turn is already complete (not ongoing).
+    """
+    for i in range(len(messages) - 1, -1, -1):
+        msg = messages[i]
+        if isinstance(msg, HumanMessage) and not msg.additional_kwargs.get(
+            "is_input_on_interrupt", False
+        ):
+            turn = messages[i:]
+            if not is_ongoing_turn(turn):
+                raise ValueError("Current turn is already complete")
+            return turn
+    return []
+
+
+def get_last_turn_messages(messages: list[BaseMessage]) -> list[BaseMessage]:
+    """Return the messages of the most recent conversation turn.
+
+    A turn begins at the most recent *turn-opening* ``HumanMessage`` — one whose
+    ``additional_kwargs`` does **not** flag it as an interrupt reply
+    (``is_input_on_interrupt`` is ``False`` or absent) — and extends to the end of
+    *messages*. HumanMessages flagged as interrupt replies are skipped, so a turn
+    that paused to ask the user a question is still treated as a single turn.
+
+    Returns ``[]`` when no turn-opening HumanMessage exists.
+    """
+    for i in range(len(messages) - 1, -1, -1):
+        msg = messages[i]
+        if isinstance(msg, HumanMessage) and not msg.additional_kwargs.get(
+            "is_input_on_interrupt", False
+        ):
+            return messages[i:]
+    return []
+
+
+def get_final_ai_message(chat_history: list[BaseMessage]) -> AIMessage:
+    """Return the last AIMessage in *chat_history*, or raise ValueError if none exists."""
+    for msg in reversed(chat_history):
+        if isinstance(msg, AIMessage):
+            return msg
+    raise ValueError("No AIMessage found in chat history")
+
+
+def is_ongoing_turn(turn: list[BaseMessage]) -> bool:
+    """Return True if *turn* is an active, in-progress ReAct turn.
+
+    A valid ongoing turn starts with a HumanMessage and ends with either an
+    AIMessage carrying pending tool calls, a ToolMessage (tool results not yet
+    processed by the agent), or an interrupt-reply HumanMessage (the agent
+    paused to ask the user a question and has not yet resumed).
+    """
+    if not turn:
+        return False
+    if not isinstance(turn[0], HumanMessage):
+        return False
+    last = turn[-1]
+    if isinstance(last, ToolMessage):
+        return True
+    if isinstance(last, HumanMessage) and last.additional_kwargs.get(
+        "is_input_on_interrupt", False
+    ):
+        return True
+    return isinstance(last, AIMessage) and bool(last.tool_calls)

opendatasci/_utils/streaming_utils.py

@@ -0,0 +1,23 @@
+_CONNECTION_EXC_NAMES = frozenset(
+    ["ConnectError", "APIConnectionError", "ConnectionError", "NetworkError"]
+)
+_CONNECTION_KEYWORDS = frozenset(
+    ["connection error", "connection refused", "name or service not known"]
+)
+
+
+def format_stream_error(exc: Exception) -> str:
+    """Return a user-friendly error message for a streaming exception."""
+    exc_type = type(exc).__name__
+    msg = str(exc)
+    msg_lower = msg.lower()
+    if exc_type in _CONNECTION_EXC_NAMES or any(kw in msg_lower for kw in _CONNECTION_KEYWORDS):
+        return (
+            "Connection error — could not reach the API. "
+            "Check your internet connection and verify your API key is valid."
+        )
+    if exc_type in ("AuthenticationError", "AuthError") or any(
+        kw in msg_lower for kw in ("authentication", "api_key", "invalid x-api-key", "unauthorized")
+    ):
+        return f"Authentication error — your API key may be invalid or expired. ({msg})"
+    return msg

opendatasci/agents/__init__.py

@@ -0,0 +1,12 @@
+"""Agent layer — LLM provider factory, memory, tools, skills, and the Agent class."""
+
+from opendatasci.agents.agents import Agent
+from opendatasci.agents.agents_factory import create_agent
+from opendatasci.agents.chat_memory import ChatHistoryBuilder, PreparedHistory
+
+__all__ = [
+    "Agent",
+    "create_agent",
+    "ChatHistoryBuilder",
+    "PreparedHistory",
+]