DeepFabric 4.11.0__py3-none-any.whl → 4.12.0__py3-none-any.whl

deepfabric/graph.py

@@ -70,6 +70,11 @@ class GraphConfig(BaseModel):
         le=20,
         description="Maximum concurrent LLM calls during graph expansion (helps avoid rate limits)",
     )
+    max_tokens: int = Field(
+        default=DEFAULT_MAX_TOKENS,
+        ge=1,
+        description="Maximum tokens for topic generation LLM calls",
+    )
     base_url: str | None = Field(
         default=None,
         description="Base URL for API endpoint (e.g., custom OpenAI-compatible servers)",
@@ -156,6 +161,7 @@ class Graph(TopicModel):
         self.degree = self.config.degree
         self.depth = self.config.depth
         self.max_concurrent = self.config.max_concurrent
+        self.max_tokens = self.config.max_tokens
         self.prompt_style = self.config.prompt_style
 
         # Initialize LLM client
@@ -211,6 +217,139 @@ class Graph(TopicModel):
             if parent_node not in child_node.parents:
                 child_node.parents.append(parent_node)
 
+    def find_node_by_uuid(self, uuid: str) -> Node | None:
+        """Find a node by its UUID.
+
+        Args:
+            uuid: The UUID string to search for.
+
+        Returns:
+            The Node if found, None otherwise.
+        """
+        for node in self.nodes.values():
+            if node.metadata.get("uuid") == uuid:
+                return node
+        return None
+
+    def remove_node(self, node_id: int) -> None:
+        """Remove a single node from the graph, cleaning up bidirectional references.
+
+        Does not remove children — use remove_subtree() for cascading removal.
+
+        Args:
+            node_id: The ID of the node to remove.
+
+        Raises:
+            ValueError: If node_id is the root node or does not exist.
+        """
+        if node_id == self.root.id:
+            raise ValueError("Cannot remove the root node")  # noqa: TRY003
+        node = self.nodes.get(node_id)
+        if node is None:
+            raise ValueError(f"Node {node_id} not found in graph")  # noqa: TRY003
+
+        for parent in node.parents:
+            if node in parent.children:
+                parent.children.remove(node)
+
+        for child in node.children:
+            if node in child.parents:
+                child.parents.remove(node)
+
+        del self.nodes[node_id]
+
+    def remove_subtree(self, node_id: int) -> list[int]:
+        """Remove a node and all its descendants from the graph.
+
+        Args:
+            node_id: The ID of the node to remove (along with all descendants).
+
+        Returns:
+            List of removed node IDs.
+
+        Raises:
+            ValueError: If node_id is the root node or does not exist.
+        """
+        if node_id == self.root.id:
+            raise ValueError("Cannot remove the root node")  # noqa: TRY003
+        node = self.nodes.get(node_id)
+        if node is None:
+            raise ValueError(f"Node {node_id} not found in graph")  # noqa: TRY003
+
+        # BFS to collect all descendant node IDs
+        to_remove: list[int] = []
+        queue = [node]
+        visited: set[int] = set()
+        while queue:
+            current = queue.pop(0)
+            if current.id in visited:
+                continue
+            visited.add(current.id)
+            to_remove.append(current.id)
+            for child in current.children:
+                if child.id not in visited:
+                    queue.append(child)
+
+        # Remove in reverse order (leaves first)
+        for nid in reversed(to_remove):
+            self.remove_node(nid)
+
+        return to_remove
+
+    def prune_at_level(self, max_depth: int) -> list[int]:
+        """Remove all nodes below the given depth level.
+
+        Nodes at exactly max_depth become leaf nodes. Root is depth 0.
+
+        Args:
+            max_depth: Maximum depth to keep (inclusive).
+                       0 = keep only root, 1 = root and its children, etc.
+
+        Returns:
+            List of removed node IDs.
+
+        Raises:
+            ValueError: If max_depth is negative.
+        """
+        if max_depth < 0:
+            raise ValueError("max_depth must be non-negative")  # noqa: TRY003
+
+        # BFS from root to compute node depths
+        node_depths: dict[int, int] = {}
+        queue: list[tuple[Node, int]] = [(self.root, 0)]
+        visited: set[int] = set()
+        while queue:
+            current, depth = queue.pop(0)
+            if current.id in visited:
+                continue
+            visited.add(current.id)
+            node_depths[current.id] = depth
+            for child in current.children:
+                if child.id not in visited:
+                    queue.append((child, depth + 1))
+
+        to_remove_set = {nid for nid, d in node_depths.items() if d > max_depth}
+
+        # Sever children links from boundary nodes
+        for nid, d in node_depths.items():
+            if d == max_depth:
+                self.nodes[nid].children = [
+                    c for c in self.nodes[nid].children if c.id not in to_remove_set
+                ]
+
+        # Remove deeper nodes
+        for nid in to_remove_set:
+            node = self.nodes[nid]
+            for parent in node.parents:
+                if node in parent.children:
+                    parent.children.remove(node)
+            for child in node.children:
+                if node in child.parents:
+                    child.parents.remove(node)
+            del self.nodes[nid]
+
+        return list(to_remove_set)
+
     def to_pydantic(self) -> GraphModel:
         """Converts the runtime graph to its Pydantic model representation."""
         return GraphModel(
@@ -237,6 +376,13 @@ class Graph(TopicModel):
         with open(save_path, "w") as f:
             f.write(self.to_json())
 
+        # Save failed generations if any
+        if self.failed_generations:
+            failed_path = save_path.replace(".json", "_failed.jsonl")
+            with open(failed_path, "w") as f:
+                for failed in self.failed_generations:
+                    f.write(json.dumps({"failed_generation": failed}) + "\n")
+
     @classmethod
     def from_json(cls, json_path: str, params: dict) -> "Graph":
         """Load a topic graph from a JSON file."""
@@ -268,6 +414,36 @@ class Graph(TopicModel):
         graph._next_node_id = max(graph.nodes.keys()) + 1
         return graph
 
+    @classmethod
+    def load(cls, json_path: str) -> "Graph":
+        """Load a graph from JSON without initializing LLM client.
+
+        Intended for inspection and manipulation operations that don't
+        require LLM generation capabilities. Restores provider, model,
+        and temperature from the file metadata so saves preserve them.
+        """
+        params = {
+            "topic_prompt": "loaded",
+            "model_name": "placeholder/model",
+            "degree": 3,
+            "depth": 2,
+            "temperature": 0.7,
+        }
+        graph = cls.from_json(json_path, params)
+
+        # Restore original metadata so save() preserves provenance
+        with open(json_path) as f:
+            raw = json.load(f)
+        file_meta = raw.get("metadata") or {}
+        if file_meta.get("provider"):
+            graph.provider = file_meta["provider"]
+        if file_meta.get("model"):
+            graph.model_name = file_meta["model"]
+        if file_meta.get("temperature") is not None:
+            graph.temperature = file_meta["temperature"]
+
+        return graph
+
     def visualize(self, save_path: str) -> None:
         """Visualize the graph and save it to a file."""
         try:
@@ -454,7 +630,7 @@ class Graph(TopicModel):
                     prompt=prompt,
                     schema=GraphSubtopics,
                     max_retries=1,  # Don't retry inside - we handle it here
-                    max_tokens=DEFAULT_MAX_TOKENS,
+                    max_tokens=self.max_tokens,
                     temperature=self.temperature,
                 )
 

deepfabric/graph_pruner.py

@@ -0,0 +1,122 @@
+"""Graph pruning operations for deepfabric CLI."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from .graph import Graph
+
+
+@dataclass
+class PruneResult:
+    """Result of a pruning operation."""
+
+    operation: Literal["level", "uuid"]
+    removed_count: int
+    removed_node_ids: list[int]
+    remaining_nodes: int
+    remaining_paths: int
+    output_path: str
+
+
+def load_graph_for_pruning(file_path: str) -> Graph:
+    """Load a graph from JSON for pruning operations.
+
+    Args:
+        file_path: Path to the graph JSON file.
+
+    Returns:
+        Loaded Graph instance.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+        ValueError: If the file is not a JSON graph file.
+    """
+    path = Path(file_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Graph file not found: {file_path}")
+    if path.suffix != ".json":
+        raise ValueError(
+            f"Expected a JSON graph file, got: {path.suffix}. "
+            "Pruning is only supported for graph format files."
+        )
+    return Graph.load(file_path)
+
+
+def prune_graph_at_level(
+    file_path: str,
+    max_depth: int,
+    output_path: str | None = None,
+) -> PruneResult:
+    """Prune a graph file by removing all nodes below a depth level.
+
+    Args:
+        file_path: Path to the input graph JSON file.
+        max_depth: Maximum depth to keep (0=root only, 1=root+children, etc.).
+        output_path: Output file path. If None, derives from input filename.
+
+    Returns:
+        PruneResult with operation details.
+    """
+    graph = load_graph_for_pruning(file_path)
+    removed_ids = graph.prune_at_level(max_depth)
+
+    final_output = output_path or _derive_output_path(file_path, f"pruned_level{max_depth}")
+    graph.save(final_output)
+
+    return PruneResult(
+        operation="level",
+        removed_count=len(removed_ids),
+        removed_node_ids=removed_ids,
+        remaining_nodes=len(graph.nodes),
+        remaining_paths=len(graph.get_all_paths()),
+        output_path=final_output,
+    )
+
+
+def prune_graph_by_uuid(
+    file_path: str,
+    uuid: str,
+    output_path: str | None = None,
+) -> PruneResult:
+    """Remove a node (by UUID) and its entire subtree from a graph file.
+
+    Args:
+        file_path: Path to the input graph JSON file.
+        uuid: UUID of the node to remove.
+        output_path: Output file path. If None, derives from input filename.
+
+    Returns:
+        PruneResult with operation details.
+
+    Raises:
+        ValueError: If UUID not found or targets the root node.
+    """
+    graph = load_graph_for_pruning(file_path)
+    node = graph.find_node_by_uuid(uuid)
+
+    if node is None:
+        raise ValueError(f"No node found with UUID: {uuid}")
+
+    removed_ids = graph.remove_subtree(node.id)
+
+    final_output = output_path or _derive_output_path(file_path, "pruned")
+    graph.save(final_output)
+
+    return PruneResult(
+        operation="uuid",
+        removed_count=len(removed_ids),
+        removed_node_ids=removed_ids,
+        remaining_nodes=len(graph.nodes),
+        remaining_paths=len(graph.get_all_paths()),
+        output_path=final_output,
+    )
+
+
+def _derive_output_path(input_path: str, suffix: str) -> str:
+    """Derive a non-destructive output path from the input path.
+
+    Example: topic_graph.json -> topic_graph_pruned_level2.json
+    """
+    p = Path(input_path)
+    return str(p.with_stem(f"{p.stem}_{suffix}"))

deepfabric/topic_inspector.py

@@ -0,0 +1,237 @@
+"""Topic file inspection utilities for deepfabric CLI."""
+
+import hashlib
+import json
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+from .graph import Graph
+from .utils import read_topic_tree_from_jsonl
+
+
+@dataclass
+class TopicInspectionResult:
+    """Result of inspecting a topic file."""
+
+    format: Literal["tree", "graph"]
+    total_paths: int
+    max_depth: int
+    paths_at_level: list[list[str]] | None
+    expanded_paths: list[list[str]] | None  # Paths from level onwards (with --expand)
+    all_paths: list[list[str]] | None
+    metadata: dict[str, Any]
+    source_file: str
+    # Maps path tuple to UUID/topic_id (for --uuid flag)
+    path_to_uuid: dict[tuple[str, ...], str] = field(default_factory=dict)
+    # Maps topic name to UUID (for graph format, all nodes)
+    topic_to_uuid: dict[str, str] = field(default_factory=dict)
+
+
+def detect_format(file_path: str) -> Literal["tree", "graph"]:
+    """Auto-detect topic file format based on content.
+
+    Args:
+        file_path: Path to the topic file
+
+    Returns:
+        "tree" for JSONL format, "graph" for JSON format
+
+    Raises:
+        ValueError: If format cannot be detected
+        FileNotFoundError: If file doesn't exist
+    """
+    path = Path(file_path)
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    with open(file_path, encoding="utf-8") as f:
+        content = f.read().strip()
+        if not content:
+            raise ValueError("Empty file")
+
+        # Try to parse as a complete JSON object (Graph format)
+        try:
+            data = json.loads(content)
+            if isinstance(data, dict) and "nodes" in data and "root_id" in data:
+                return "graph"
+        except json.JSONDecodeError:
+            pass
+
+        # Try to parse first line as JSONL (Tree format)
+        first_line = content.split("\n")[0].strip()
+        try:
+            first_obj = json.loads(first_line)
+            if isinstance(first_obj, dict) and "path" in first_obj:
+                return "tree"
+        except json.JSONDecodeError:
+            pass
+
+    raise ValueError(f"Unable to detect format for: {file_path}")
+
+
+def _load_tree_paths(file_path: str) -> tuple[list[list[str]], dict[tuple[str, ...], str]]:
+    """Load tree paths directly from JSONL without initializing LLM.
+
+    Args:
+        file_path: Path to the JSONL file
+
+    Returns:
+        Tuple of (paths, path_to_uuid mapping)
+    """
+    dict_list = read_topic_tree_from_jsonl(file_path)
+    paths = []
+    path_to_uuid: dict[tuple[str, ...], str] = {}
+
+    for d in dict_list:
+        if "path" not in d:
+            continue
+        path = d["path"]
+        paths.append(path)
+        # Generate hash-based ID from path (same as tree.py)
+        path_str = " > ".join(path)
+        topic_id = hashlib.sha256(path_str.encode()).hexdigest()[:16]
+        path_to_uuid[tuple(path)] = topic_id
+
+    return paths, path_to_uuid
+
+
+def _load_graph_data(
+    file_path: str,
+) -> tuple[list[list[str]], dict[str, Any], dict[tuple[str, ...], str], dict[str, str]]:
+    """Load graph data and extract paths and metadata.
+
+    Args:
+        file_path: Path to the JSON file
+
+    Returns:
+        Tuple of (paths, metadata, path_to_uuid mapping, topic_to_uuid mapping)
+    """
+    graph = Graph.load(file_path)
+
+    # Get paths with UUIDs (for leaf nodes)
+    paths_with_ids = graph.get_all_paths_with_ids()
+    all_paths = [tp.path for tp in paths_with_ids]
+    path_to_uuid: dict[tuple[str, ...], str] = {
+        tuple(tp.path): tp.topic_id for tp in paths_with_ids
+    }
+
+    # Build topic name to UUID mapping for ALL nodes (not just leaves)
+    topic_to_uuid: dict[str, str] = {}
+    for node in graph.nodes.values():
+        node_uuid = node.metadata.get("uuid", "")
+        if node_uuid:
+            topic_to_uuid[node.topic] = node_uuid
+
+    metadata: dict[str, Any] = {
+        "total_nodes": len(graph.nodes),
+        "has_cycles": graph.has_cycle(),
+        "root_topic": graph.root.topic if graph.root else None,
+    }
+
+    # Read graph-level metadata directly from the JSON file
+    # since Graph.from_json doesn't restore provider/model
+    with open(file_path, encoding="utf-8") as f:
+        raw_data = json.load(f)
+
+    if "metadata" in raw_data and raw_data["metadata"]:
+        file_metadata = raw_data["metadata"]
+        if file_metadata.get("created_at"):
+            metadata["created_at"] = file_metadata["created_at"]
+        if file_metadata.get("provider"):
+            metadata["provider"] = file_metadata["provider"]
+        if file_metadata.get("model"):
+            metadata["model"] = file_metadata["model"]
+
+    return all_paths, metadata, path_to_uuid, topic_to_uuid
+
+
+def inspect_topic_file(
+    file_path: str,
+    level: int | None = None,
+    expand_depth: int | None = None,
+    show_all: bool = False,
+) -> TopicInspectionResult:
+    """Inspect a topic file and return structured results.
+
+    Args:
+        file_path: Path to the topic file
+        level: Specific level to show (0=root), or None
+        expand_depth: Number of sublevels to show (-1 for all), or None for no expansion
+        show_all: Whether to include all paths in result
+
+    Returns:
+        TopicInspectionResult with inspection data
+    """
+    format_type = detect_format(file_path)
+
+    # Load paths and metadata based on format
+    topic_to_uuid: dict[str, str] = {}
+    if format_type == "graph":
+        all_paths, metadata, path_to_uuid, topic_to_uuid = _load_graph_data(file_path)
+    else:
+        all_paths, path_to_uuid = _load_tree_paths(file_path)
+        # Extract root topic from paths
+        metadata = {}
+        if all_paths:
+            metadata["root_topic"] = all_paths[0][0]
+
+    max_depth = max(len(p) for p in all_paths) if all_paths else 0
+
+    # Get unique topics at specific level if requested
+    # Level 0 = root, Level 1 = children of root, etc.
+    paths_at_level = None
+    expanded_paths = None
+
+    if level is not None:
+        # Extract unique topic names at the given depth position
+        seen_topics: set[str] = set()
+        unique_topics: list[str] = []
+        for path in all_paths:
+            if len(path) > level:
+                topic_at_level = path[level]
+                if topic_at_level not in seen_topics:
+                    seen_topics.add(topic_at_level)
+                    unique_topics.append(topic_at_level)
+                    # If this topic is a leaf (path ends at level+1), map single-topic to UUID
+                    if len(path) == level + 1:
+                        original_uuid = path_to_uuid.get(tuple(path), "")
+                        if original_uuid:
+                            path_to_uuid[(topic_at_level,)] = original_uuid
+        # Store as single-element paths for consistency
+        paths_at_level = [[t] for t in unique_topics]
+
+        # If expand_depth is set, get paths from level onwards
+        if expand_depth is not None:
+            seen_paths: set[tuple[str, ...]] = set()
+            expanded_paths = []
+            for path in all_paths:
+                if len(path) > level:
+                    original_uuid = path_to_uuid.get(tuple(path), "")
+                    # Trim path to start from the specified level
+                    trimmed_path = path[level:]
+                    # Limit depth if expand_depth is not -1
+                    if expand_depth != -1 and len(trimmed_path) > expand_depth + 1:
+                        trimmed_path = trimmed_path[: expand_depth + 1]
+                    # Deduplicate paths (after trimming, many may be identical)
+                    path_key = tuple(trimmed_path)
+                    if path_key not in seen_paths:
+                        seen_paths.add(path_key)
+                        expanded_paths.append(trimmed_path)
+                        # Map trimmed path to original UUID (for --uuid display)
+                        if original_uuid and path_key not in path_to_uuid:
+                            path_to_uuid[path_key] = original_uuid
+
+    return TopicInspectionResult(
+        format=format_type,
+        total_paths=len(all_paths),
+        max_depth=max_depth,
+        paths_at_level=paths_at_level,
+        expanded_paths=expanded_paths,
+        all_paths=all_paths if show_all else None,
+        metadata=metadata,
+        source_file=file_path,
+        path_to_uuid=path_to_uuid,
+        topic_to_uuid=topic_to_uuid,
+    )

deepfabric/topic_manager.py

@@ -90,6 +90,19 @@ async def _process_graph_events(graph: Graph, debug: bool = False) -> dict | Non
                 tui.finish_building(failed_generations)
                 final_event = event
 
+                if failed_generations > 0 and hasattr(graph, "failed_generations"):
+                    truncated = sum(
+                        1
+                        for f in graph.failed_generations
+                        if "EOF while parsing" in f.get("last_error", "")
+                    )
+                    if truncated:
+                        get_tui().warning(
+                            f"Hint: {truncated} of {failed_generations} failures appear to be "
+                            f"truncated responses. Consider increasing max_tokens "
+                            f"(currently {graph.max_tokens})."
+                        )
+
                 if debug and failed_generations > 0 and hasattr(graph, "failed_generations"):
                     get_tui().error("\nDebug: Graph generation failures:")
                     for idx, failure in enumerate(graph.failed_generations, 1):
@@ -147,6 +160,19 @@ async def _process_tree_events(tree: Tree, debug: bool = False) -> dict | None:
                 tui.finish_building(total_paths, failed_generations)
                 final_event = event
 
+                if failed_generations > 0 and hasattr(tree, "failed_generations"):
+                    truncated = sum(
+                        1
+                        for f in tree.failed_generations
+                        if "EOF while parsing" in f.get("error", "")
+                    )
+                    if truncated:
+                        get_tui().warning(
+                            f"Hint: {truncated} of {failed_generations} failures appear to be "
+                            f"truncated responses. Consider increasing max_tokens "
+                            f"(currently {tree.max_tokens})."
+                        )
+
                 if debug and failed_generations > 0 and hasattr(tree, "failed_generations"):
                     get_tui().error("\nDebug: Tree generation failures:")
                     for idx, failure in enumerate(tree.failed_generations, 1):
@@ -313,6 +339,9 @@ def save_topic_model(
         try:
             tree_save_path = topics_save_as or config.topics.save_as or "topic_tree.jsonl"
             topic_model.save(tree_save_path)
+            if getattr(topic_model, "failed_generations", None):
+                failed_path = tree_save_path.replace(".jsonl", "_failed.jsonl")
+                tui.warning(f"Failed generations saved to: {failed_path}")
             tui.success(f"Topic tree saved to {tree_save_path}")
             tui.info(f"Total paths: {len(topic_model.tree_paths)}")
         except Exception as e:
@@ -322,6 +351,9 @@ def save_topic_model(
         try:
             graph_save_path = topics_save_as or config.topics.save_as or "topic_graph.json"
             topic_model.save(graph_save_path)
+            if getattr(topic_model, "failed_generations", None):
+                failed_path = graph_save_path.replace(".json", "_failed.jsonl")
+                tui.warning(f"Failed generations saved to: {failed_path}")
             tui.success(f"Topic graph saved to {graph_save_path}")
         except Exception as e:
             raise ConfigurationError(f"Error saving topic graph: {str(e)}") from e