DeepFabric 4.11.0__tar.gz → 4.12.0__tar.gz

{deepfabric-4.11.0 → deepfabric-4.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DeepFabric
-Version: 4.11.0
+Version: 4.12.0
 Summary: Curate High Quality Datasets, Train, Evaluate and Ship
 Author-email: DeepFabric Team <oss@alwaysfurther.ai>
 License-File: LICENSE

{deepfabric-4.11.0 → deepfabric-4.12.0}/deepfabric/cli.py

@@ -8,7 +8,13 @@ import signal
 import sys
 
 from pathlib import Path
-from typing import Literal, NoReturn, cast
+from typing import TYPE_CHECKING, Literal, NoReturn, cast
+
+if TYPE_CHECKING:
+    from rich.tree import Tree as RichTree
+
+    from .topic_inspector import TopicInspectionResult
+    from .tui import DeepFabricTUI
 
 import click
 import yaml
@@ -139,7 +145,7 @@ class GenerateOptions(BaseModel):
     batch_size: int | None = None
     base_url: str | None = None
     include_system_message: bool | None = None
-    mode: Literal["tree", "graph"] = Field(default="tree")
+    mode: Literal["tree", "graph"] | None = Field(default=None)
     debug: bool = False
     topic_only: bool = False
     tui: Literal["rich", "simple"] = Field(default="rich")
@@ -332,7 +338,7 @@ def _load_and_prepare_generation_context(
     # Skip path validation for topic-only mode since we're not generating dataset samples
     if not skip_path_validation:
         validate_path_requirements(
-            mode=options.mode,
+            mode=config.topics.mode,
             depth=final_depth,
             degree=final_degree,
             num_samples=final_num_samples,
@@ -341,7 +347,7 @@ def _load_and_prepare_generation_context(
         )
 
         show_validation_success(
-            mode=options.mode,
+            mode=config.topics.mode,
             depth=final_depth,
             degree=final_degree,
             num_samples=final_num_samples,
@@ -655,8 +661,8 @@ def _run_generation(
 @click.option(
     "--mode",
     type=click.Choice(["tree", "graph"]),
-    default="tree",
-    help="Topic generation mode (default: tree)",
+    default=None,
+    help="Topic generation mode (default: graph)",
 )
 @click.option(
     "--debug",
@@ -2050,5 +2056,534 @@ def checkpoint_status(config_file: str) -> None:
         )
 
 
+# Topic inspection command group
+@click.group()
+def topic() -> None:
+    """Topic management commands."""
+    pass
+
+
+@topic.command("inspect")
+@click.argument("file", type=click.Path(exists=True))
+@click.option(
+    "--level",
+    "-l",
+    type=int,
+    default=None,
+    help="Show topics at a specific depth level (0=root, 1=first children, etc.)",
+)
+@click.option(
+    "--expand",
+    "-e",
+    type=int,
+    default=None,
+    is_flag=False,
+    flag_value=-1,  # -1 means expand all levels
+    help="Show subtopics in tree format. Use alone for all levels, or specify depth (e.g., --expand 2)",
+)
+@click.option(
+    "--all",
+    "-a",
+    "show_all",
+    is_flag=True,
+    help="Show the entire tree structure with indentation",
+)
+@click.option(
+    "--format",
+    "-f",
+    "output_format",
+    type=click.Choice(["tree", "table", "json"]),
+    default="tree",
+    help="Output format (default: tree)",
+)
+@click.option(
+    "--uuid",
+    "-u",
+    "show_uuid",
+    is_flag=True,
+    help="Show UUID/topic_id for each leaf node",
+)
+def topic_inspect(
+    file: str,
+    level: int | None,
+    expand: int | None,
+    show_all: bool,
+    output_format: str,
+    show_uuid: bool,
+) -> None:
+    """Inspect a topic tree or graph file.
+
+    Displays metadata and structure of topic files generated by DeepFabric.
+    Supports both Tree (JSONL) and Graph (JSON) formats with auto-detection.
+
+    Examples:
+
+    \b
+        # Show file metadata and summary
+        deepfabric topic inspect topic_tree.jsonl
+
+    \b
+        # Show topics at depth level 2 (just topic names)
+        deepfabric topic inspect topic_tree.jsonl --level 2
+
+    \b
+        # Show level 2 topics and all subtopics (tree format)
+        deepfabric topic inspect topic_tree.jsonl --level 2 --expand
+
+    \b
+        # Show level 2 topics and 1 sublevel only (tree format)
+        deepfabric topic inspect topic_tree.jsonl --level 2 --expand 1
+
+    \b
+        # Show entire tree with indentation
+        deepfabric topic inspect topic_graph.json --all
+
+    \b
+        # Output as JSON for scripting
+        deepfabric topic inspect topic_tree.jsonl --format json
+
+    \b
+        # Show UUIDs for each leaf node
+        deepfabric topic inspect topic_tree.jsonl --all --uuid
+    """
+    from .topic_inspector import inspect_topic_file  # noqa: PLC0415
+
+    tui = get_tui()
+
+    try:
+        # Perform inspection
+        result = inspect_topic_file(file, level=level, expand_depth=expand, show_all=show_all)
+
+        # Handle JSON output format
+        if output_format == "json":
+            output = {
+                "format": result.format,
+                "source_file": result.source_file,
+                "total_paths": result.total_paths,
+                "max_depth": result.max_depth,
+                "metadata": result.metadata,
+            }
+            if result.paths_at_level is not None:
+                output["paths_at_level"] = result.paths_at_level
+            if result.expanded_paths is not None:
+                output["expanded_paths"] = result.expanded_paths
+            if result.all_paths is not None:
+                output["all_paths"] = result.all_paths
+            tui.console.print_json(json.dumps(output))
+            return
+
+        # Rich output (tree or table format)
+        _display_inspection_result(tui, result, output_format, level, expand, show_all, show_uuid)
+
+    except FileNotFoundError as e:
+        tui.error(str(e))
+        sys.exit(1)
+    except ValueError as e:
+        tui.error(f"Invalid file: {e}")
+        sys.exit(1)
+    except Exception as e:
+        tui.error(f"Error inspecting file: {e}")
+        sys.exit(1)
+
+
+def _display_inspection_result(
+    tui: "DeepFabricTUI",
+    result: "TopicInspectionResult",
+    output_format: str,
+    level: int | None,
+    expand: int | None,
+    show_all: bool,
+    show_uuid: bool = False,
+) -> None:
+    """Display inspection result using rich formatting."""
+    from rich.panel import Panel  # noqa: PLC0415
+    from rich.table import Table  # noqa: PLC0415
+
+    # Header with file info
+    format_label = "Graph (JSON)" if result.format == "graph" else "Tree (JSONL)"
+
+    tui.console.print()
+    tui.console.print("[bold cyan]Topic Inspector[/bold cyan]")
+    tui.console.print(f"[dim]{result.source_file}[/dim]")
+    tui.console.print()
+
+    # Statistics panel
+    stats_table = Table(show_header=False, box=None, padding=(0, 1))
+    stats_table.add_column(style="cyan", no_wrap=True)
+    stats_table.add_column(style="white")
+
+    stats_table.add_row("Format:", format_label)
+    stats_table.add_row("Total Paths:", str(result.total_paths))
+    stats_table.add_row("Max Depth:", str(result.max_depth))
+
+    if result.metadata.get("root_topic"):
+        root = result.metadata["root_topic"]
+        if len(root) > 60:  # noqa: PLR2004
+            root = root[:57] + "..."
+        stats_table.add_row("Root Topic:", root)
+
+    if result.metadata.get("total_nodes"):
+        stats_table.add_row("Total Nodes:", str(result.metadata["total_nodes"]))
+
+    if result.metadata.get("has_cycles") is not None:
+        has_cycles = "Yes" if result.metadata["has_cycles"] else "No"
+        stats_table.add_row("Has Cycles:", has_cycles)
+
+    if result.metadata.get("provider"):
+        stats_table.add_row("Provider:", result.metadata["provider"])
+
+    if result.metadata.get("model"):
+        stats_table.add_row("Model:", result.metadata["model"])
+
+    if result.metadata.get("created_at"):
+        stats_table.add_row("Created:", result.metadata["created_at"])
+
+    tui.console.print(Panel(stats_table, title="Statistics", border_style="dim"))
+
+    # Show level-specific topics (without expand) - simple list of topic names
+    if level is not None and expand is None and result.paths_at_level is not None:
+        tui.console.print()
+        tui.console.print(f"[cyan bold]Topics at Level {level}:[/cyan bold]")
+
+        if not result.paths_at_level:
+            tui.console.print(f"  [dim]No topics at level {level}[/dim]")
+        else:
+            # Display as simple list of topic names (with UUIDs)
+            for topic_path in result.paths_at_level:
+                topic_name = topic_path[0] if topic_path else ""
+                if show_uuid:
+                    # For graph format, use topic_to_uuid (node UUIDs)
+                    # For tree format, use path_to_uuid (leaf UUIDs only)
+                    uuid = ""
+                    if result.topic_to_uuid:
+                        uuid = result.topic_to_uuid.get(topic_name, "")
+                    if not uuid and result.path_to_uuid:
+                        uuid = result.path_to_uuid.get(tuple(topic_path), "")
+                    if uuid:
+                        tui.console.print(
+                            f"  • {topic_name} [dim](UUID: {uuid})[/dim]", highlight=False
+                        )
+                    else:
+                        tui.console.print(f"  • {topic_name}")
+                else:
+                    tui.console.print(f"  • {topic_name}")
+
+    # Show expanded subtree from level (with --expand)
+    if level is not None and expand is not None and result.expanded_paths is not None:
+        tui.console.print()
+        depth_info = "all sublevels" if expand == -1 else f"{expand} sublevel(s)"
+        tui.console.print(f"[cyan bold]Subtree from Level {level} ({depth_info}):[/cyan bold]")
+
+        if not result.expanded_paths:
+            tui.console.print(f"  [dim]No topics at or below level {level}[/dim]")
+        elif output_format == "table":
+            _display_paths_as_table(tui, result.expanded_paths)
+        else:
+            _display_paths_as_tree(
+                tui,
+                result.expanded_paths,
+                result.path_to_uuid if show_uuid else None,
+                result.topic_to_uuid if show_uuid else None,
+            )
+
+    # Show all paths with tree structure
+    if show_all and result.all_paths:
+        tui.console.print()
+        tui.console.print("[cyan bold]Full Tree Structure:[/cyan bold]")
+
+        if output_format == "table":
+            _display_paths_as_table(tui, result.all_paths)
+        else:
+            _display_paths_as_tree(
+                tui,
+                result.all_paths,
+                result.path_to_uuid if show_uuid else None,
+                result.topic_to_uuid if show_uuid else None,
+            )
+
+
+def _display_paths_as_table(tui: "DeepFabricTUI", paths: list[list[str]]) -> None:
+    """Display paths in a table format."""
+    from rich.table import Table  # noqa: PLC0415
+
+    table = Table(show_header=True, header_style="bold cyan")
+    table.add_column("#", style="dim")
+    table.add_column("Path", style="white")
+    table.add_column("Depth", style="green")
+
+    for i, path in enumerate(paths[:100], 1):
+        path_str = " > ".join(path)
+        if len(path_str) > 80:  # noqa: PLR2004
+            path_str = path_str[:77] + "..."
+        table.add_row(str(i), path_str, str(len(path)))
+
+    if len(paths) > 100:  # noqa: PLR2004
+        table.add_row("...", f"[dim]{len(paths) - 100} more paths[/dim]", "")
+
+    tui.console.print(table)
+
+
+def _display_paths_as_tree(
+    tui: "DeepFabricTUI",
+    paths: list[list[str]],
+    path_to_uuid: dict[tuple[str, ...], str] | None = None,
+    topic_to_uuid: dict[str, str] | None = None,
+) -> None:
+    """Display paths in an indented tree format."""
+    from rich.tree import Tree as RichTree  # noqa: PLC0415
+
+    if not paths:
+        return
+
+    # Build a tree structure from paths
+    # Group paths by their root topic
+    root_groups: dict[str, list[list[str]]] = {}
+    for path in paths:
+        if path:
+            root = path[0]
+            if root not in root_groups:
+                root_groups[root] = []
+            root_groups[root].append(path)
+
+    if len(root_groups) == 1:
+        # Single root - show directly
+        root_topic = paths[0][0]
+        # Show UUID for root if available (graph format)
+        root_label = f"[bold]{root_topic}[/bold]"
+        if topic_to_uuid and root_topic in topic_to_uuid:
+            root_label += f" [dim](UUID: {topic_to_uuid[root_topic]})[/dim]"
+        tree = RichTree(root_label)
+        _add_children_to_tree(
+            tree, paths, 1, path_to_uuid=path_to_uuid, topic_to_uuid=topic_to_uuid
+        )
+        tui.console.print(tree)
+    else:
+        # Multiple roots - show each as a separate tree
+        for root_topic, root_paths in list(root_groups.items())[:20]:
+            root_label = f"[bold]{root_topic}[/bold]"
+            if topic_to_uuid and root_topic in topic_to_uuid:
+                root_label += f" [dim](UUID: {topic_to_uuid[root_topic]})[/dim]"
+            tree = RichTree(root_label)
+            _add_children_to_tree(
+                tree, root_paths, 1, path_to_uuid=path_to_uuid, topic_to_uuid=topic_to_uuid
+            )
+            tui.console.print(tree)
+        if len(root_groups) > 20:  # noqa: PLR2004
+            tui.console.print(f"[dim]... and {len(root_groups) - 20} more topics[/dim]")
+
+
+def _add_children_to_tree(
+    parent: "RichTree",
+    paths: list[list[str]],
+    depth: int,
+    max_depth: int = 5,
+    path_to_uuid: dict[tuple[str, ...], str] | None = None,
+    topic_to_uuid: dict[str, str] | None = None,
+) -> None:
+    """Recursively add children to a rich tree (limited depth for display)."""
+    if depth > max_depth:
+        remaining = len([p for p in paths if len(p) > depth])
+        if remaining > 0:
+            parent.add(f"[dim]... {remaining} more levels[/dim]")
+        return
+
+    # Group paths by their element at current depth
+    children: dict[str, list[list[str]]] = {}
+    for path in paths:
+        if len(path) > depth:
+            child_topic = path[depth]
+            if child_topic not in children:
+                children[child_topic] = []
+            children[child_topic].append(path)
+
+    # Add children to tree
+    for child_topic, child_paths in list(children.items())[:20]:
+        # Check for UUID: first try topic_to_uuid (graph nodes), then path_to_uuid (leaves)
+        uuid = ""
+        if topic_to_uuid and child_topic in topic_to_uuid:
+            uuid = topic_to_uuid[child_topic]
+        elif path_to_uuid:
+            # Check if this child is a leaf (path ends at depth + 1)
+            is_leaf = any(len(p) == depth + 1 for p in child_paths)
+            if is_leaf:
+                leaf_path = next((p for p in child_paths if len(p) == depth + 1), None)
+                if leaf_path:
+                    uuid = path_to_uuid.get(tuple(leaf_path), "")
+
+        if uuid:
+            child_node = parent.add(f"{child_topic} [dim](UUID: {uuid})[/dim]")
+        else:
+            child_node = parent.add(child_topic)
+        _add_children_to_tree(
+            child_node, child_paths, depth + 1, max_depth, path_to_uuid, topic_to_uuid
+        )
+
+    if len(children) > 20:  # noqa: PLR2004
+        parent.add(f"[dim]... and {len(children) - 20} more siblings[/dim]")
+
+
+@topic.command("prune")
+@click.argument("file", type=click.Path(exists=True))
+@click.option(
+    "--level",
+    "-l",
+    type=int,
+    default=None,
+    help="Prune all nodes below this depth level (0=root, 1=children, etc.)",
+)
+@click.option(
+    "--uuid",
+    "-u",
+    type=str,
+    default=None,
+    help="Remove the node with this UUID and its entire subtree",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    default=None,
+    help="Output file path (default: auto-generated from input filename)",
+)
+@click.option(
+    "--force",
+    "-f",
+    is_flag=True,
+    help="Overwrite the input file instead of creating a new one",
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Show what would be removed without making changes",
+)
+def topic_prune(
+    file: str,
+    level: int | None,
+    uuid: str | None,
+    output: str | None,
+    force: bool,
+    dry_run: bool,
+) -> None:
+    """Prune a topic graph by removing nodes.
+
+    Supports two modes:
+
+    \b
+        # Remove all nodes below depth level 2
+        deepfabric topic prune topic_graph.json --level 2
+
+    \b
+        # Remove a specific node and its subtree by UUID
+        deepfabric topic prune topic_graph.json --uuid abc-123-def
+
+    \b
+        # Preview what would be removed (no file written)
+        deepfabric topic prune topic_graph.json --level 1 --dry-run
+
+    \b
+        # Overwrite the original file
+        deepfabric topic prune topic_graph.json --uuid abc-123 --force
+    """
+    from .graph_pruner import (  # noqa: PLC0415
+        load_graph_for_pruning,
+        prune_graph_at_level,
+        prune_graph_by_uuid,
+    )
+
+    tui = get_tui()
+
+    # Validate: exactly one mode must be specified
+    if level is None and uuid is None:
+        tui.error("Specify either --level or --uuid")
+        sys.exit(1)
+    if level is not None and uuid is not None:
+        tui.error("Cannot use --level and --uuid together")
+        sys.exit(1)
+
+    try:
+        if dry_run:
+            graph = load_graph_for_pruning(file)
+            total_nodes = len(graph.nodes)
+
+            tui.console.print()
+            tui.console.print("[bold]DRY RUN[/bold] — no changes will be made")
+            tui.console.print()
+
+            if level is not None:
+                # BFS to compute node depths
+                node_depths: dict[int, int] = {}
+                queue: list[tuple] = [(graph.root, 0)]
+                visited: set[int] = set()
+                while queue:
+                    current, d = queue.pop(0)
+                    if current.id in visited:
+                        continue
+                    visited.add(current.id)
+                    node_depths[current.id] = d
+                    for child in current.children:
+                        if child.id not in visited:
+                            queue.append((child, d + 1))
+
+                to_remove = {nid for nid, d in node_depths.items() if d > level}
+                tui.console.print(f"  Graph:         {total_nodes} unique nodes")
+                tui.console.print(f"  Would remove:  {len(to_remove)} nodes below level {level}")
+                tui.console.print(f"  Would keep:    {total_nodes - len(to_remove)} nodes")
+            else:
+                target = graph.find_node_by_uuid(uuid)
+                if target is None:
+                    tui.error(f"No node found with UUID: {uuid}")
+                    sys.exit(1)
+
+                # BFS to count subtree
+                subtree_count = 0
+                bfs_queue = [target]
+                visited_ids: set[int] = set()
+                while bfs_queue:
+                    current = bfs_queue.pop(0)
+                    if current.id in visited_ids:
+                        continue
+                    visited_ids.add(current.id)
+                    subtree_count += 1
+                    for child in current.children:
+                        if child.id not in visited_ids:
+                            bfs_queue.append(child)
+
+                tui.console.print(f"  Graph:         {total_nodes} unique nodes")
+                tui.console.print(
+                    f"  Target:        {target.topic}",
+                    highlight=False,
+                )
+                tui.console.print(f"  Would remove:  {subtree_count} nodes (including subtree)")
+                tui.console.print(f"  Would keep:    {total_nodes - subtree_count} nodes")
+            return
+
+        # Determine output path
+        output_path = file if force else output
+
+        if level is not None:
+            result = prune_graph_at_level(file, level, output_path)
+        else:
+            result = prune_graph_by_uuid(file, uuid, output_path)
+
+        tui.console.print()
+        tui.success("Graph pruned successfully")
+        tui.console.print(f"  Removed:   {result.removed_count} nodes")
+        tui.console.print(
+            f"  Remaining: {result.remaining_nodes} nodes, {result.remaining_paths} paths"
+        )
+        tui.console.print(f"  Saved to:  {result.output_path}")
+
+    except FileNotFoundError as e:
+        tui.error(str(e))
+        sys.exit(1)
+    except ValueError as e:
+        tui.error(str(e))
+        sys.exit(1)
+
+
+# Register the topic command group
+cli.add_command(topic)
+
+
 if __name__ == "__main__":
     cli()

{deepfabric-4.11.0 → deepfabric-4.12.0}/deepfabric/config.py

@@ -8,6 +8,7 @@ from pydantic import BaseModel, Field, field_validator, model_validator
 
 from .constants import (
     DEFAULT_MAX_RETRIES,
+    DEFAULT_MAX_TOKENS,
     DEFAULT_MODEL,
     DEFAULT_PROVIDER,
     DEFAULT_SAMPLE_RETRIES,
@@ -86,7 +87,7 @@ class TopicsConfig(BaseModel):
         ..., min_length=1, description="The initial prompt to start topic generation"
     )
     mode: Literal["tree", "graph"] = Field(
-        default="tree", description="Topic generation mode: tree or graph"
+        default="graph", description="Topic generation mode: tree or graph"
     )
     system_prompt: str = Field(
         default="", description="System prompt for topic exploration and generation"
@@ -109,6 +110,11 @@ class TopicsConfig(BaseModel):
         le=20,
         description="Maximum concurrent LLM calls during graph expansion (helps avoid rate limits)",
     )
+    max_tokens: int = Field(
+        default=DEFAULT_MAX_TOKENS,
+        ge=1,
+        description="Maximum tokens for topic generation LLM calls",
+    )
     save_as: str | None = Field(default=None, description="Where to save the generated topics")
     prompt_style: Literal["default", "isolated", "anchored"] = Field(
         default="default",
@@ -589,6 +595,7 @@ See documentation for full examples.
             "depth": self.topics.depth,
             "degree": self.topics.degree,
             "max_concurrent": self.topics.max_concurrent,
+            "max_tokens": self.topics.max_tokens,
             "prompt_style": self.topics.prompt_style,
         }
 

{deepfabric-4.11.0 → deepfabric-4.12.0}/deepfabric/config_manager.py

@@ -37,7 +37,7 @@ def load_config(  # noqa: PLR0913
     topics_save_as: str | None = None,
     output_save_as: str | None = None,
     include_system_message: bool | None = None,
-    mode: str = "tree",
+    mode: str | None = None,
     # Modular conversation configuration
     conversation_type: str | None = None,
     reasoning_style: str | None = None,
@@ -83,6 +83,8 @@ def load_config(  # noqa: PLR0913
         except Exception as e:
             raise ConfigurationError(f"Error loading config file: {str(e)}") from e
         else:
+            if mode is not None:
+                config.topics.mode = mode
             return config
 
     # No config file provided - create minimal configuration from CLI args
@@ -92,6 +94,9 @@ def load_config(  # noqa: PLR0913
     tui = get_tui()
     tui.info("No config file provided - using CLI parameters")
 
+    # Default to graph mode when no config file and no explicit mode
+    mode = mode or "graph"
+
     # Create minimal config dict with new structure
     default_prompt = generation_system_prompt or "You are a helpful AI assistant."