DeepFabric 4.4.0__py3-none-any.whl

deepfabric/update_checker.py

@@ -0,0 +1,166 @@
+import importlib.metadata
+import json
+import logging
+import os
+import urllib.error
+import urllib.request
+
+from typing import TypedDict
+
+from packaging.version import Version, parse
+
+from .metrics import trace
+from .tui import get_tui
+
+logger = logging.getLogger(__name__)
+
+
+class PyPIPackageInfo(TypedDict, total=False):
+    """PyPI package info section."""
+
+    version: str
+
+
+class PyPIResponse(TypedDict, total=False):
+    """PyPI JSON API response structure."""
+
+    info: PyPIPackageInfo
+
+
+# PyPI API endpoint for deepfabric package
+PYPI_API_URL = "https://pypi.org/pypi/deepfabric/json"
+
+# Timeout for PyPI API request (2 seconds)
+REQUEST_TIMEOUT = 2.0
+
+
+def _get_current_version() -> str | None:
+    """
+    Get the current installed version of deepfabric.
+
+    Returns:
+        str | None: Version string or None if unable to determine
+    """
+    try:
+        return importlib.metadata.version("deepfabric")
+    except (ImportError, importlib.metadata.PackageNotFoundError):
+        logger.debug("Unable to determine current version")
+        return None
+
+
+def _is_update_check_disabled() -> bool:
+    """
+    Check if update checking is disabled via environment variable.
+
+    Returns:
+        bool: True if DEEPFABRIC_NO_UPDATE_CHECK is set to any truthy value
+    """
+    env_value = os.environ.get("DEEPFABRIC_NO_UPDATE_CHECK", "").lower()
+    return env_value in ("1", "true", "yes", "on")
+
+
+def _fetch_latest_version_from_pypi() -> str | None:
+    """
+    Fetch the latest version from PyPI API.
+
+    Returns:
+        str | None: Latest version string or None if fetch fails
+    """
+    try:
+        with urllib.request.urlopen(  # noqa: S310 # nosec
+            PYPI_API_URL, timeout=REQUEST_TIMEOUT
+        ) as response:
+            data: PyPIResponse = json.loads(response.read().decode("utf-8"))
+            latest_version = data.get("info", {}).get("version")
+            if latest_version:
+                logger.debug("Fetched latest version from PyPI: %s", latest_version)
+                return latest_version
+            logger.debug("No version found in PyPI response")
+            return None
+    except TimeoutError:
+        logger.debug("PyPI request timed out after %s seconds", REQUEST_TIMEOUT)
+        return None
+    except urllib.error.URLError as e:
+        logger.debug("Failed to fetch from PyPI: %s", e)
+        return None
+    except (KeyError, ValueError, json.JSONDecodeError) as e:
+        logger.debug("Failed to parse PyPI response: %s", e)
+        return None
+
+
+def _compare_versions(current: str, latest: str) -> bool:
+    """
+    Compare version strings to determine if an update is available.
+
+    Args:
+        current: Current version string
+        latest: Latest version string
+
+    Returns:
+        bool: True if latest > current, False otherwise
+    """
+    try:
+        current_version: Version = parse(current)
+        latest_version: Version = parse(latest)
+    except Exception as e:
+        logger.debug("Failed to compare versions: %s", e)
+        return False
+    else:
+        return latest_version > current_version
+
+
+def check_for_updates() -> None:
+    """
+    Check for available updates and notify user if a newer version exists.
+
+    This function:
+    1. Checks if update checking is disabled via environment variable
+    2. Gets the current installed version
+    3. Fetches the latest version from PyPI
+    4. Compares versions and displays a warning if update is available
+    5. Tracks metrics about the update check
+
+    The function is designed to fail silently and never block CLI execution.
+    All errors are logged at DEBUG level and do not interrupt the user.
+    """
+    # Check if update checking is disabled
+    if _is_update_check_disabled():
+        logger.debug("Update check disabled via DEEPFABRIC_NO_UPDATE_CHECK")
+        return
+
+    # Get current version
+    current_version = _get_current_version()
+    if not current_version or current_version == "development":
+        logger.debug("Skipping update check for development version")
+        return
+
+    # Fetch latest version from PyPI
+    latest_version = _fetch_latest_version_from_pypi()
+    if not latest_version:
+        logger.debug("Could not fetch latest version from PyPI")
+        return
+
+    # Track metrics about the check
+    try:
+        trace(
+            "update_check_performed",
+            {
+                "current_version": current_version,
+                "latest_version": latest_version,
+                "update_available": _compare_versions(current_version, latest_version),
+            },
+        )
+    except Exception as e:
+        logger.debug("Failed to track update check metrics: %s", e)
+
+    # Compare versions and notify user if update is available
+    if _compare_versions(current_version, latest_version):
+        try:
+            tui = get_tui()
+            tui.warning(
+                f"Update available: deepfabric {latest_version} "
+                f"(you have {current_version})\n"
+                f"   Run: pip install --upgrade deepfabric"
+            )
+        except Exception as e:
+            logger.debug("Failed to display update notification: %s", e)

deepfabric/utils.py

@@ -0,0 +1,150 @@
+import ast
+import asyncio
+import json
+import re
+
+VALIDATION_ERROR_INDICATORS = [
+    "validation error",
+    "value error",
+    "is null",
+    "is empty string",
+    "must provide actual value",
+    "invalid schema",
+    "pydantic",
+    "string should have at least",
+    "field required",
+]
+
+
+def is_validation_error(error: Exception) -> bool:
+    """Check if an error is a validation/schema error that can be retried."""
+    error_str = str(error).lower()
+    return any(indicator in error_str for indicator in VALIDATION_ERROR_INDICATORS)
+
+
+def ensure_not_running_loop(method_name: str) -> None:
+    """Raise when invoked inside an active asyncio event loop."""
+
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        return
+
+    if loop.is_running():
+        msg = (
+            f"{method_name} cannot be called while an event loop is running. "
+            "Use the async variant instead."
+        )
+        raise RuntimeError(msg)
+
+
+def extract_list(input_string: str):
+    """
+    Extracts a Python list from a given input string.
+
+    This function attempts to parse the input string as JSON. If that fails,
+    it searches for the first Python list within the string by identifying
+    the opening and closing brackets. If a list is found, it is evaluated
+    safely to ensure it is a valid Python list.
+
+    Args:
+        input_string (str): The input string potentially containing a Python list.
+
+    Returns:
+        list: The extracted Python list if found and valid, otherwise an empty list.
+
+    Raises:
+        None: This function handles its own exceptions and does not raise any.
+    """
+    try:
+        return json.loads(input_string)
+    except json.JSONDecodeError:
+        print("Failed to parse the input string as JSON.")
+
+    start = input_string.find("[")
+    if start == -1:
+        print("No Python list found in the input string.")
+        return []
+
+    count = 0
+    for i, char in enumerate(input_string[start:]):
+        if char == "[":
+            count += 1
+        elif char == "]":
+            count -= 1
+        if count == 0:
+            end = i + start + 1
+            break
+    else:
+        print("No matching closing bracket found.")
+        return []
+
+    found_list_str = input_string[start:end]
+    found_list = safe_literal_eval(found_list_str)
+    if found_list is None:
+        print("Failed to parse the list due to syntax issues.")
+        return []
+
+    return found_list
+
+
+def remove_linebreaks_and_spaces(input_string):
+    """
+    Remove line breaks and extra spaces from the input string.
+
+    This function replaces all whitespace characters (including line breaks)
+    with a single space and then ensures that there are no consecutive spaces
+    in the resulting string.
+
+    Args:
+        input_string (str): The string from which to remove line breaks and extra spaces.
+
+    Returns:
+        str: The processed string with line breaks and extra spaces removed.
+    """
+    no_linebreaks = re.sub(r"\s+", " ", input_string)
+    return " ".join(no_linebreaks.split())
+
+
+def safe_literal_eval(list_string: str):
+    """
+    Safely evaluate a string containing a Python literal expression.
+
+    This function attempts to evaluate a string containing a Python literal
+    expression using `ast.literal_eval`. If a `SyntaxError` or `ValueError`
+    occurs, it tries to sanitize the string by replacing problematic apostrophes
+    with the actual right single quote character and attempts the evaluation again.
+
+    Args:
+        list_string (str): The string to be evaluated.
+
+    Returns:
+        The result of the evaluated string if successful, otherwise `None`.
+    """
+    try:
+        return ast.literal_eval(list_string)
+    except (SyntaxError, ValueError):
+        # Replace problematic apostrophes with the actual right single quote character
+        sanitized_string = re.sub(r"(\w)'(\w)", r"\1’\2", list_string)
+        try:
+            return ast.literal_eval(sanitized_string)
+        except (SyntaxError, ValueError):
+            print("Failed to parse the list due to syntax issues.")
+            return None
+
+
+def read_topic_tree_from_jsonl(file_path: str) -> list[dict]:
+    """
+    Read the topic tree from a JSONL file.
+
+    Args:
+        file_path (str): The path to the JSONL file.
+
+    Returns:
+        list[dict]: The topic tree.
+    """
+    topic_tree = []
+    with open(file_path) as file:
+        for line in file:
+            topic_tree.append(json.loads(line.strip()))
+    return topic_tree

deepfabric/validation.py

@@ -0,0 +1,143 @@
+import time
+
+from .exceptions import ConfigurationError
+from .tui import get_tui
+
+
+def calculate_expected_paths(mode: str, depth: int, degree: int) -> int:
+    """
+    Calculate expected number of paths for tree/graph generation.
+
+    Args:
+        mode: Generation mode ('tree' or 'graph')
+        depth: Depth of the tree/graph
+        degree: Branching factor
+
+    Returns:
+        Expected number of paths
+    """
+    if mode == "tree":
+        # Tree paths = degree^depth (exact - each leaf is a unique path)
+        return degree**depth
+    # mode == "graph"
+    # Graph paths vary widely due to cross-connections
+    # Can range from degree^depth * 0.5 to degree^depth * 2+
+    # Use base estimate as rough middle ground, but warn it's approximate
+    return degree**depth
+
+
+def validate_path_requirements(
+    mode: str,
+    depth: int,
+    degree: int,
+    num_steps: int,
+    batch_size: int,
+    loading_existing: bool = False,
+) -> None:
+    """
+    Validate that the topic generation parameters will produce enough paths.
+
+    Args:
+        mode: Generation mode ('tree' or 'graph')
+        depth: Depth of the tree/graph
+        degree: Branching factor
+        num_steps: Number of generation steps
+        batch_size: Batch size for generation
+        loading_existing: Whether loading existing topic model from file
+
+    Raises:
+        ConfigurationError: If validation fails
+    """
+    if loading_existing:
+        # Can't validate existing files without loading them
+        return
+
+    expected_paths = calculate_expected_paths(mode, depth, degree)
+    required_samples = num_steps * batch_size
+
+    if required_samples > expected_paths:
+        # Alternative: provide exact combinations that use all paths
+        optimal_combinations = []
+        for test_steps in range(1, expected_paths + 1):
+            test_batch = expected_paths // test_steps
+            if test_steps * test_batch <= expected_paths and test_batch > 0:
+                optimal_combinations.append((test_steps, test_batch))
+
+        # Sort by preference (fewer steps first, then larger batches)
+        optimal_combinations.sort(key=lambda x: (x[0], -x[1]))
+
+        tui = get_tui()
+        tui.error(" Path validation failed - stopping before topic generation")
+
+        # Build recommendations - focus on optimal combinations rather than misleading individual params
+        recommendations = []
+
+        if optimal_combinations:
+            recommendations.append(
+                f"  • Use one of these combinations to utilize the {expected_paths} paths:"
+            )
+            for steps, batch in optimal_combinations[:3]:  # Show top 3
+                total_samples = steps * batch
+                recommendations.append(
+                    f"    --num-steps {steps} --batch-size {batch}  (generates {total_samples} samples)"
+                )
+
+        recommendations.extend(
+            [
+                f"  • Or increase --depth (currently {depth}) or --degree (currently {degree})",
+            ]
+        )
+
+        estimation_note = ""
+        if mode == "graph":
+            estimation_note = " (estimated - graphs vary due to cross-connections)"
+
+        error_msg = (
+            f"Insufficient expected paths for dataset generation:\n"
+            f"  • Expected {mode} paths: ~{expected_paths}{estimation_note} (depth={depth}, degree={degree})\n"
+            f"  • Requested samples: {required_samples} ({num_steps} steps × {batch_size} batch size)\n"
+            f"  • Shortfall: ~{required_samples - expected_paths} samples\n\n"
+            f"Recommendations:\n" + "\n".join(recommendations)
+        )
+
+        if mode == "graph":
+            error_msg += f"\n\nNote: Graph path counts are estimates. The actual graph may produce {expected_paths // 2}-{expected_paths * 2} paths due to cross-connections."
+
+        raise ConfigurationError(error_msg)
+
+
+def show_validation_success(
+    mode: str,
+    depth: int,
+    degree: int,
+    num_steps: int,
+    batch_size: int,
+    loading_existing: bool = False,
+) -> None:
+    """
+    Show validation success message.
+
+    Args:
+        mode: Generation mode ('tree' or 'graph')
+        depth: Depth of the tree/graph
+        degree: Branching factor
+        num_steps: Number of generation steps
+        batch_size: Batch size for generation
+        loading_existing: Whether loading existing topic model from file
+    """
+    if loading_existing:
+        return
+
+    expected_paths = calculate_expected_paths(mode, depth, degree)
+    total_samples = num_steps * batch_size
+
+    tui = get_tui()
+    tui.success("Path Validation Passed")
+    tui.info(f"  Expected {mode} paths: ~{expected_paths} (depth={depth}, degree={degree})")
+    tui.info(f"  Requested samples: {total_samples} ({num_steps} steps x {batch_size} batch size)")
+    tui.info(f"  Path utilization: ~{min(100, (total_samples / expected_paths) * 100):.1f}%")
+
+    if mode == "graph":
+        tui.info("  Note: Graph paths may vary due to cross-connections")
+    print()  # Extra space before topic generation
+    time.sleep(0.5)  # Brief pause to allow user to see the information