DeepFabric 4.10.1__py3-none-any.whl → 4.12.0__py3-none-any.whl

deepfabric/topic_inspector.py

@@ -0,0 +1,237 @@
+"""Topic file inspection utilities for deepfabric CLI."""
+
+import hashlib
+import json
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+from .graph import Graph
+from .utils import read_topic_tree_from_jsonl
+
+
+@dataclass
+class TopicInspectionResult:
+    """Result of inspecting a topic file."""
+
+    format: Literal["tree", "graph"]
+    total_paths: int
+    max_depth: int
+    paths_at_level: list[list[str]] | None
+    expanded_paths: list[list[str]] | None  # Paths from level onwards (with --expand)
+    all_paths: list[list[str]] | None
+    metadata: dict[str, Any]
+    source_file: str
+    # Maps path tuple to UUID/topic_id (for --uuid flag)
+    path_to_uuid: dict[tuple[str, ...], str] = field(default_factory=dict)
+    # Maps topic name to UUID (for graph format, all nodes)
+    topic_to_uuid: dict[str, str] = field(default_factory=dict)
+
+
+def detect_format(file_path: str) -> Literal["tree", "graph"]:
+    """Auto-detect topic file format based on content.
+
+    Args:
+        file_path: Path to the topic file
+
+    Returns:
+        "tree" for JSONL format, "graph" for JSON format
+
+    Raises:
+        ValueError: If format cannot be detected
+        FileNotFoundError: If file doesn't exist
+    """
+    path = Path(file_path)
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    with open(file_path, encoding="utf-8") as f:
+        content = f.read().strip()
+        if not content:
+            raise ValueError("Empty file")
+
+        # Try to parse as a complete JSON object (Graph format)
+        try:
+            data = json.loads(content)
+            if isinstance(data, dict) and "nodes" in data and "root_id" in data:
+                return "graph"
+        except json.JSONDecodeError:
+            pass
+
+        # Try to parse first line as JSONL (Tree format)
+        first_line = content.split("\n")[0].strip()
+        try:
+            first_obj = json.loads(first_line)
+            if isinstance(first_obj, dict) and "path" in first_obj:
+                return "tree"
+        except json.JSONDecodeError:
+            pass
+
+    raise ValueError(f"Unable to detect format for: {file_path}")
+
+
+def _load_tree_paths(file_path: str) -> tuple[list[list[str]], dict[tuple[str, ...], str]]:
+    """Load tree paths directly from JSONL without initializing LLM.
+
+    Args:
+        file_path: Path to the JSONL file
+
+    Returns:
+        Tuple of (paths, path_to_uuid mapping)
+    """
+    dict_list = read_topic_tree_from_jsonl(file_path)
+    paths = []
+    path_to_uuid: dict[tuple[str, ...], str] = {}
+
+    for d in dict_list:
+        if "path" not in d:
+            continue
+        path = d["path"]
+        paths.append(path)
+        # Generate hash-based ID from path (same as tree.py)
+        path_str = " > ".join(path)
+        topic_id = hashlib.sha256(path_str.encode()).hexdigest()[:16]
+        path_to_uuid[tuple(path)] = topic_id
+
+    return paths, path_to_uuid
+
+
+def _load_graph_data(
+    file_path: str,
+) -> tuple[list[list[str]], dict[str, Any], dict[tuple[str, ...], str], dict[str, str]]:
+    """Load graph data and extract paths and metadata.
+
+    Args:
+        file_path: Path to the JSON file
+
+    Returns:
+        Tuple of (paths, metadata, path_to_uuid mapping, topic_to_uuid mapping)
+    """
+    graph = Graph.load(file_path)
+
+    # Get paths with UUIDs (for leaf nodes)
+    paths_with_ids = graph.get_all_paths_with_ids()
+    all_paths = [tp.path for tp in paths_with_ids]
+    path_to_uuid: dict[tuple[str, ...], str] = {
+        tuple(tp.path): tp.topic_id for tp in paths_with_ids
+    }
+
+    # Build topic name to UUID mapping for ALL nodes (not just leaves)
+    topic_to_uuid: dict[str, str] = {}
+    for node in graph.nodes.values():
+        node_uuid = node.metadata.get("uuid", "")
+        if node_uuid:
+            topic_to_uuid[node.topic] = node_uuid
+
+    metadata: dict[str, Any] = {
+        "total_nodes": len(graph.nodes),
+        "has_cycles": graph.has_cycle(),
+        "root_topic": graph.root.topic if graph.root else None,
+    }
+
+    # Read graph-level metadata directly from the JSON file
+    # since Graph.from_json doesn't restore provider/model
+    with open(file_path, encoding="utf-8") as f:
+        raw_data = json.load(f)
+
+    if "metadata" in raw_data and raw_data["metadata"]:
+        file_metadata = raw_data["metadata"]
+        if file_metadata.get("created_at"):
+            metadata["created_at"] = file_metadata["created_at"]
+        if file_metadata.get("provider"):
+            metadata["provider"] = file_metadata["provider"]
+        if file_metadata.get("model"):
+            metadata["model"] = file_metadata["model"]
+
+    return all_paths, metadata, path_to_uuid, topic_to_uuid
+
+
+def inspect_topic_file(
+    file_path: str,
+    level: int | None = None,
+    expand_depth: int | None = None,
+    show_all: bool = False,
+) -> TopicInspectionResult:
+    """Inspect a topic file and return structured results.
+
+    Args:
+        file_path: Path to the topic file
+        level: Specific level to show (0=root), or None
+        expand_depth: Number of sublevels to show (-1 for all), or None for no expansion
+        show_all: Whether to include all paths in result
+
+    Returns:
+        TopicInspectionResult with inspection data
+    """
+    format_type = detect_format(file_path)
+
+    # Load paths and metadata based on format
+    topic_to_uuid: dict[str, str] = {}
+    if format_type == "graph":
+        all_paths, metadata, path_to_uuid, topic_to_uuid = _load_graph_data(file_path)
+    else:
+        all_paths, path_to_uuid = _load_tree_paths(file_path)
+        # Extract root topic from paths
+        metadata = {}
+        if all_paths:
+            metadata["root_topic"] = all_paths[0][0]
+
+    max_depth = max(len(p) for p in all_paths) if all_paths else 0
+
+    # Get unique topics at specific level if requested
+    # Level 0 = root, Level 1 = children of root, etc.
+    paths_at_level = None
+    expanded_paths = None
+
+    if level is not None:
+        # Extract unique topic names at the given depth position
+        seen_topics: set[str] = set()
+        unique_topics: list[str] = []
+        for path in all_paths:
+            if len(path) > level:
+                topic_at_level = path[level]
+                if topic_at_level not in seen_topics:
+                    seen_topics.add(topic_at_level)
+                    unique_topics.append(topic_at_level)
+                    # If this topic is a leaf (path ends at level+1), map single-topic to UUID
+                    if len(path) == level + 1:
+                        original_uuid = path_to_uuid.get(tuple(path), "")
+                        if original_uuid:
+                            path_to_uuid[(topic_at_level,)] = original_uuid
+        # Store as single-element paths for consistency
+        paths_at_level = [[t] for t in unique_topics]
+
+        # If expand_depth is set, get paths from level onwards
+        if expand_depth is not None:
+            seen_paths: set[tuple[str, ...]] = set()
+            expanded_paths = []
+            for path in all_paths:
+                if len(path) > level:
+                    original_uuid = path_to_uuid.get(tuple(path), "")
+                    # Trim path to start from the specified level
+                    trimmed_path = path[level:]
+                    # Limit depth if expand_depth is not -1
+                    if expand_depth != -1 and len(trimmed_path) > expand_depth + 1:
+                        trimmed_path = trimmed_path[: expand_depth + 1]
+                    # Deduplicate paths (after trimming, many may be identical)
+                    path_key = tuple(trimmed_path)
+                    if path_key not in seen_paths:
+                        seen_paths.add(path_key)
+                        expanded_paths.append(trimmed_path)
+                        # Map trimmed path to original UUID (for --uuid display)
+                        if original_uuid and path_key not in path_to_uuid:
+                            path_to_uuid[path_key] = original_uuid
+
+    return TopicInspectionResult(
+        format=format_type,
+        total_paths=len(all_paths),
+        max_depth=max_depth,
+        paths_at_level=paths_at_level,
+        expanded_paths=expanded_paths,
+        all_paths=all_paths if show_all else None,
+        metadata=metadata,
+        source_file=file_path,
+        path_to_uuid=path_to_uuid,
+        topic_to_uuid=topic_to_uuid,
+    )

deepfabric/topic_manager.py

@@ -45,6 +45,8 @@ async def _process_graph_events(graph: Graph, debug: bool = False) -> dict | Non
     progress_reporter = ProgressReporter()
     progress_reporter.attach(tui)
     graph.progress_reporter = progress_reporter
+    if hasattr(graph, "llm_client"):
+        graph.llm_client.retry_handler.progress_reporter = progress_reporter
 
     tui_started = False
 
@@ -88,6 +90,19 @@ async def _process_graph_events(graph: Graph, debug: bool = False) -> dict | Non
                 tui.finish_building(failed_generations)
                 final_event = event
 
+                if failed_generations > 0 and hasattr(graph, "failed_generations"):
+                    truncated = sum(
+                        1
+                        for f in graph.failed_generations
+                        if "EOF while parsing" in f.get("last_error", "")
+                    )
+                    if truncated:
+                        get_tui().warning(
+                            f"Hint: {truncated} of {failed_generations} failures appear to be "
+                            f"truncated responses. Consider increasing max_tokens "
+                            f"(currently {graph.max_tokens})."
+                        )
+
                 if debug and failed_generations > 0 and hasattr(graph, "failed_generations"):
                     get_tui().error("\nDebug: Graph generation failures:")
                     for idx, failure in enumerate(graph.failed_generations, 1):
@@ -116,6 +131,8 @@ async def _process_tree_events(tree: Tree, debug: bool = False) -> dict | None:
     progress_reporter = ProgressReporter()
     progress_reporter.attach(tui)
     tree.progress_reporter = progress_reporter
+    if hasattr(tree, "llm_client"):
+        tree.llm_client.retry_handler.progress_reporter = progress_reporter
 
     final_event = None
     try:
@@ -129,6 +146,8 @@ async def _process_tree_events(tree: Tree, debug: bool = False) -> dict | None:
                     tui.add_failure()
                     if debug and "error" in event:
                         get_tui().error(f"Debug: Tree generation failure - {event['error']}")
+                else:
+                    tui.advance_simple_progress()
             elif event["event"] == "build_complete":
                 total_paths = (
                     int(event["total_paths"]) if isinstance(event["total_paths"], str | int) else 0
@@ -141,6 +160,19 @@ async def _process_tree_events(tree: Tree, debug: bool = False) -> dict | None:
                 tui.finish_building(total_paths, failed_generations)
                 final_event = event
 
+                if failed_generations > 0 and hasattr(tree, "failed_generations"):
+                    truncated = sum(
+                        1
+                        for f in tree.failed_generations
+                        if "EOF while parsing" in f.get("error", "")
+                    )
+                    if truncated:
+                        get_tui().warning(
+                            f"Hint: {truncated} of {failed_generations} failures appear to be "
+                            f"truncated responses. Consider increasing max_tokens "
+                            f"(currently {tree.max_tokens})."
+                        )
+
                 if debug and failed_generations > 0 and hasattr(tree, "failed_generations"):
                     get_tui().error("\nDebug: Tree generation failures:")
                     for idx, failure in enumerate(tree.failed_generations, 1):
@@ -233,8 +265,22 @@ def load_or_build_topic_model(
     tui = get_tui()
 
     if topics_load:
-        # Determine mode from config or file extension
-        is_graph = config.topics.mode == "graph" or topics_load.endswith(".json")
+        # Config mode takes precedence; file extension is only used to warn on mismatch
+        is_graph = config.topics.mode == "graph"
+
+        # Warn if file extension doesn't match the configured mode
+        if not is_graph and topics_load.endswith(".json"):
+            tui.warning(
+                f"File '{topics_load}' has .json extension (typically a graph) "
+                f"but mode is '{config.topics.mode}'. "
+                "If this is a graph set mode: graph in config."
+            )
+        elif is_graph and topics_load.endswith(".jsonl"):
+            tui.warning(
+                f"File '{topics_load}' has .jsonl extension (typically a tree) "
+                "but mode is 'graph'. "
+                "If this is a tree set mode: tree in config."
+            )
 
         if is_graph:
             tui.info(f"Reading topic graph from JSON file: {topics_load}")
@@ -293,6 +339,9 @@ def save_topic_model(
         try:
             tree_save_path = topics_save_as or config.topics.save_as or "topic_tree.jsonl"
             topic_model.save(tree_save_path)
+            if getattr(topic_model, "failed_generations", None):
+                failed_path = tree_save_path.replace(".jsonl", "_failed.jsonl")
+                tui.warning(f"Failed generations saved to: {failed_path}")
             tui.success(f"Topic tree saved to {tree_save_path}")
             tui.info(f"Total paths: {len(topic_model.tree_paths)}")
         except Exception as e:
@@ -302,6 +351,9 @@ def save_topic_model(
         try:
             graph_save_path = topics_save_as or config.topics.save_as or "topic_graph.json"
             topic_model.save(graph_save_path)
+            if getattr(topic_model, "failed_generations", None):
+                failed_path = graph_save_path.replace(".json", "_failed.jsonl")
+                tui.warning(f"Failed generations saved to: {failed_path}")
             tui.success(f"Topic graph saved to {graph_save_path}")
         except Exception as e:
             raise ConfigurationError(f"Error saving topic graph: {str(e)}") from e

deepfabric/topic_model.py

@@ -9,6 +9,18 @@ class TopicPath(NamedTuple):
     topic_id: str
 
 
+class Topic(NamedTuple):
+    """A unique topic with its UUID and content.
+
+    Used for generation where we iterate over unique topics (by UUID)
+    rather than paths. This deduplicated view is essential for graphs
+    where multiple paths can lead to the same topic node.
+    """
+
+    uuid: str
+    topic: str  # The topic text/content
+
+
 class TopicModel(ABC):
     """Abstract base class for topic models like Tree and Graph."""
 
@@ -37,6 +49,20 @@ class TopicModel(ABC):
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def get_unique_topics(self) -> list[Topic]:
+        """Returns deduplicated topics by UUID.
+
+        For generation, we iterate over unique topics rather than paths.
+        This is important for graphs where multiple paths can lead to the
+        same topic node - we only want to generate one sample per unique topic.
+
+        Returns:
+            List of Topic namedtuples containing (uuid, topic).
+            Each UUID appears exactly once.
+        """
+        raise NotImplementedError
+
     def get_path_by_id(self, topic_id: str) -> list[str] | None:
         """Look up a path by its topic_id.
 

deepfabric/tree.py

@@ -1,4 +1,5 @@
 import asyncio
+import hashlib
 import json
 import time
 import warnings
@@ -21,7 +22,7 @@ from .metrics import trace
 from .prompts import TreePromptBuilder
 from .schemas import TopicList
 from .stream_simulator import simulate_stream
-from .topic_model import TopicModel, TopicPath
+from .topic_model import Topic, TopicModel, TopicPath
 
 warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings:.*")
 
@@ -82,6 +83,17 @@ class TreeConfig(BaseModel):
         default=None,
         description="Base URL for API endpoint (e.g., custom OpenAI-compatible servers)",
     )
+    max_concurrent: int = Field(
+        default=4,
+        ge=1,
+        le=20,
+        description="Maximum concurrent LLM calls during tree expansion (helps avoid rate limits)",
+    )
+    max_tokens: int = Field(
+        default=DEFAULT_MAX_TOKENS,
+        ge=1,
+        description="Maximum tokens for topic generation LLM calls",
+    )
 
 
 class TreeValidator:
@@ -147,6 +159,8 @@ class Tree(TopicModel):
         self.temperature = self.config.temperature
         self.provider = self.config.provider
         self.model_name = self.config.model_name
+        self.max_concurrent = self.config.max_concurrent
+        self.max_tokens = self.config.max_tokens
 
         # Initialize LLM client
         llm_kwargs = {}
@@ -242,24 +256,41 @@ class Tree(TopicModel):
         """Returns all the paths in the topic model."""
         return self.tree_paths
 
+    @staticmethod
+    def _path_to_id(path: list[str]) -> str:
+        """Compute a deterministic topic ID from a tree path."""
+        return hashlib.sha256(json.dumps(path).encode()).hexdigest()[:16]
+
+    def _add_path(self, path: list[str]) -> None:
+        """Add a path to the tree.
+
+        Args:
+            path: The topic path to add.
+        """
+        self.tree_paths.append(path)
+
     def get_all_paths_with_ids(self) -> list[TopicPath]:
         """Returns all paths with their unique identifiers.
 
-        For Tree, we generate stable IDs by hashing the path content.
-        This ensures the same path always gets the same ID across runs.
-
         Returns:
             List of TopicPath namedtuples containing (path, topic_id).
+            The topic_id is computed deterministically from the path content.
         """
-        import hashlib  # noqa: PLC0415
+        return [TopicPath(path=path, topic_id=self._path_to_id(path)) for path in self.tree_paths]
 
-        result: list[TopicPath] = []
-        for path in self.tree_paths:
-            # Generate stable ID from path content
-            path_str = "::".join(path)
-            topic_id = hashlib.sha256(path_str.encode()).hexdigest()[:16]
-            result.append(TopicPath(path=path, topic_id=topic_id))
-        return result
+    def get_unique_topics(self) -> list[Topic]:
+        """Returns all leaf topics with computed IDs.
+
+        For Trees, each path is unique by definition, so this returns
+        all leaf topics with deterministic path-based IDs.
+
+        Returns:
+            List of Topic namedtuples containing (uuid, topic).
+        """
+        return [
+            Topic(uuid=self._path_to_id(path), topic=path[-1] if path else "")
+            for path in self.tree_paths
+        ]
 
     async def get_subtopics(
         self, system_prompt: str, node_path: list[str], num_subtopics: int
@@ -282,7 +313,7 @@ class Tree(TopicModel):
                 prompt=prompt,
                 schema=TopicList,
                 max_retries=MAX_RETRY_ATTEMPTS,
-                max_tokens=DEFAULT_MAX_TOKENS,
+                max_tokens=self.max_tokens,
                 temperature=self.temperature,
             )
 
@@ -293,19 +324,11 @@ class Tree(TopicModel):
                 source="tree_generation",
             )
 
-            # Extract and validate subtopics
+            # Extract subtopics — accept whatever the LLM returned
             subtopics = topic_response.subtopics
-            if len(subtopics) >= num_subtopics:
-                return subtopics[:num_subtopics]
-
-            # If insufficient subtopics, pad with defaults
-            while len(subtopics) < num_subtopics:
-                subtopics.append(f"subtopic_{len(subtopics) + 1}_for_{node_path[-1]}")
-
             return subtopics[:num_subtopics]
 
         except Exception as e:
-            # Log the failure and return default subtopics
             self.failed_generations.append(
                 {
                     "node_path": node_path,
@@ -313,9 +336,7 @@ class Tree(TopicModel):
                     "timestamp": time.time(),
                 }
             )
-
-            # Generate default subtopics
-            return [f"subtopic_{i + 1}_for_{node_path[-1]}" for i in range(num_subtopics)]
+            return []
 
     def _detect_domain(self, system_prompt: str, node_path: list[str]) -> str:
         """Detect the appropriate domain for prompt examples based on context."""
@@ -361,7 +382,7 @@ class Tree(TopicModel):
         yield {"event": "subtree_start", "node_path": node_path, "depth": current_depth}
 
         if current_depth > total_depth:
-            self.tree_paths.append(node_path)
+            self._add_path(node_path)
             yield {"event": "leaf_reached", "path": node_path}
             return
 
@@ -383,27 +404,43 @@ class Tree(TopicModel):
         yield event
 
         if not subtopics:
-            self.tree_paths.append(node_path)
+            self._add_path(node_path)
             yield {"event": "leaf_reached", "path": node_path}
             return
 
-        async def _collect_child_events(child_subtopic: str) -> list[dict[str, Any]]:
-            child_path = node_path + [child_subtopic]
-            events: list[dict[str, Any]] = []
-            async for child_event in self._build_subtree_generator(
-                child_path, system_prompt, total_depth, n_child, current_depth + 1
-            ):
-                events.append(child_event)
-            return events
+        event_queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+
+        async def _expand_child(child_subtopic: str) -> None:
+            async with semaphore:
+                child_path = node_path + [child_subtopic]
+                async for child_event in self._build_subtree_generator(
+                    child_path, system_prompt, total_depth, n_child, current_depth + 1
+                ):
+                    await event_queue.put(child_event)
 
-        tasks = [asyncio.create_task(_collect_child_events(subtopic)) for subtopic in subtopics]
+        tasks = [asyncio.create_task(_expand_child(s)) for s in subtopics]
 
-        for child_events in await asyncio.gather(*tasks):
-            for child_event in child_events:
-                yield child_event
+        async def _signal_done() -> None:
+            await asyncio.gather(*tasks)
+            await event_queue.put(None)
+
+        done_task = asyncio.create_task(_signal_done())
+
+        while True:
+            event = await event_queue.get()
+            if event is None:
+                break
+            yield event
+
+        await done_task
 
     def save(self, save_path: str) -> None:
-        """Save the topic tree to a file."""
+        """Save the topic tree to a file.
+
+        Format: {"path": [...]}
+        IDs are computed on-the-fly from path content, not persisted.
+        """
         from pathlib import Path  # noqa: PLC0415
 
         Path(save_path).parent.mkdir(parents=True, exist_ok=True)
@@ -446,8 +483,11 @@ class Tree(TopicModel):
     def from_dict_list(self, dict_list: list[dict[str, Any]]) -> None:
         """Construct the topic tree from a list of dictionaries.
 
+        Accepts both the current format (``{"path": [...]}``) and the
+        legacy format that included a ``leaf_uuid`` field (silently ignored).
+
         Args:
-            dict_list (list[dict]): The list of dictionaries representing the topic tree.
+            dict_list: The list of dictionaries representing the topic tree.
         """
         # Clear existing data
         self.tree_paths = []