DeepFabric 4.10.1__tar.gz → 4.11.0__tar.gz

{deepfabric-4.10.1 → deepfabric-4.11.0}/.github/workflows/docs.yml

@@ -23,7 +23,7 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v5
 
       - name: Install uv
-        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7
+        uses: astral-sh/setup-uv@803947b9bd8e9f986429fa0c5a41c367cd732b41 # v7
 
       - name: Set up Python
         run: uv python install

{deepfabric-4.10.1 → deepfabric-4.11.0}/.github/workflows/integration.yml

@@ -58,7 +58,7 @@ jobs:
           cache: pip
 
       - name: Install uv
-        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7
+        uses: astral-sh/setup-uv@803947b9bd8e9f986429fa0c5a41c367cd732b41 # v7
         with:
             enable-cache: true
 

{deepfabric-4.10.1 → deepfabric-4.11.0}/.github/workflows/publish.yml

@@ -24,7 +24,7 @@ jobs:
           python-version: '3.x'
 
       - name: Install uv
-        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7
+        uses: astral-sh/setup-uv@803947b9bd8e9f986429fa0c5a41c367cd732b41 # v7
 
       - name: Build distributions
         run: uv build

{deepfabric-4.10.1 → deepfabric-4.11.0}/.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install uv
-      uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7
+      uses: astral-sh/setup-uv@803947b9bd8e9f986429fa0c5a41c367cd732b41 # v7
       with:
         enable-cache: true
 

{deepfabric-4.10.1 → deepfabric-4.11.0}/CLAUDE.md

@@ -59,6 +59,9 @@ The engine includes robust JSON parsing with regex extraction and retry logic fo
 - Failed samples tracked separately from successful ones
 - Comprehensive error reporting in final summary
 
+### Topic Path Context in Generation
+Dataset generation must always pass the **full hierarchical path** (root -> ... -> leaf) to `build_prompt()`, not just the leaf topic text. This applies to all generation modes (step-based and cycle-based). The full path provides essential context for the LLM to generate domain-specific samples. When modifying generation logic, verify that `subtopics_list` receives the complete path from `TopicModel.get_path_by_id()` or `TopicPath.path`, never just the leaf text in isolation.
+
 ### System Message Control
 The `sys_msg` parameter controls whether system messages are included in the final dataset format - this affects training data structure.
 

{deepfabric-4.10.1 → deepfabric-4.11.0}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2025 Luke Hinds
+   Copyright 2025 Always Further, Inc
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{deepfabric-4.10.1 → deepfabric-4.11.0}/Makefile

@@ -30,7 +30,7 @@ test-integration:
 	$(PYTEST_INTEGRATION) --maxfail=1
 
 test-integration-verbose:
-	uv run pytest tests/integration -v -rA --durations=10
+	$(PYTEST_INTEGRATION) -s -rA --durations=10 --tb=long
 
 test-integration-openai:
 	$(PYTEST_INTEGRATION) -m openai

{deepfabric-4.10.1 → deepfabric-4.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DeepFabric
-Version: 4.10.1
+Version: 4.11.0
 Summary: Curate High Quality Datasets, Train, Evaluate and Ship
 Author-email: DeepFabric Team <oss@alwaysfurther.ai>
 License-File: LICENSE
@@ -29,9 +29,11 @@ Requires-Dist: transformers==5.0.0
 Provides-Extra: dev
 Requires-Dist: bandit>=1.7.10; extra == 'dev'
 Requires-Dist: mermaid-py>=0.2.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
 Requires-Dist: pytest-httpx>=0.30.0; extra == 'dev'
 Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
+Requires-Dist: pytest-rerunfailures>=14.0; extra == 'dev'
 Requires-Dist: pytest>=7.0.0; extra == 'dev'
 Requires-Dist: requests-mock>=1.11.0; extra == 'dev'
 Requires-Dist: ruff>=0.1.0; extra == 'dev'

{deepfabric-4.10.1 → deepfabric-4.11.0}/deepfabric/cli.py

@@ -2,6 +2,8 @@ import contextlib
 import json
 import math
 import os
+import platform
+import select
 import signal
 import sys
 
@@ -419,6 +421,35 @@ def _trigger_cloud_upload(
     )
 
 
+def _prompt_with_timeout(
+    choices: list[str],
+    default: str,
+    timeout: int = 20,
+) -> str:
+    """Prompt for a choice with a visible countdown, auto-selecting default on timeout."""
+    if platform.system() == "Windows":
+        return click.prompt(
+            f"  Choose [{'/'.join(choices)}]",
+            type=click.Choice(choices),
+            default=default,
+        )
+    valid = set(choices)
+    for remaining in range(timeout, 0, -1):
+        sys.stdout.write(f"\r  Choose [{'/'.join(choices)}] (auto-{default} in {remaining:2d}s): ")
+        sys.stdout.flush()
+        ready, _, _ = select.select([sys.stdin], [], [], 1.0)
+        if ready:
+            line = sys.stdin.readline().strip()
+            sys.stdout.write("\n")
+            sys.stdout.flush()
+            if line in valid:
+                return line
+            return default
+    sys.stdout.write("\n")
+    sys.stdout.flush()
+    return default
+
+
 def _run_generation(
     *,
     preparation: GenerationPreparation,
@@ -470,14 +501,10 @@ def _run_generation(
         tui.console.print("  [cyan]3)[/cyan] Abort")
         tui.console.print()
 
-        choice = click.prompt(
-            "Choose an option",
-            type=click.Choice(["1", "2", "3"]),
-            default="1",
-        )
+        choice = _prompt_with_timeout(["1", "2", "3"], default="1", timeout=20)
 
         if choice == "1":
-            # User wants to resume
+            # User wants to resume (or auto-selected after timeout)
             options.resume = True
         elif choice == "2":
             # Clear and start fresh
@@ -493,7 +520,7 @@ def _run_generation(
         if engine.load_checkpoint(retry_failed=options.retry_failed):
             samples_done = engine._flushed_samples_count
             failures_done = engine._flushed_failures_count
-            ids_processed = len(engine._processed_ids)
+            ids_processed = len(engine._completed)
             retry_msg = " (retrying failed samples)" if options.retry_failed else ""
 
             # Update TUI status panel with checkpoint progress
@@ -503,17 +530,18 @@ def _run_generation(
             if failures_done > 0:
                 tui.info(
                     f"Resuming from checkpoint: {samples_done} samples, "
-                    f"{failures_done} failed, {ids_processed} IDs processed{retry_msg}"
+                    f"{failures_done} failed, {ids_processed} UUIDs processed{retry_msg}"
                 )
             else:
                 tui.info(
                     f"Resuming from checkpoint: {samples_done} samples, "
-                    f"{ids_processed} IDs processed{retry_msg}"
+                    f"{ids_processed} UUIDs processed{retry_msg}"
                 )
         else:
             tui.info("No checkpoint found, starting fresh generation")
 
-    # Set up graceful Ctrl+C handling for checkpoint-based stop
+    # Set up graceful Ctrl+C handling
+    has_checkpoint = generation_params.get("checkpoint_interval") is not None
     interrupt_count = 0
 
     def handle_sigint(_signum, _frame):
@@ -522,7 +550,12 @@ def _run_generation(
 
         if interrupt_count == 1:
             engine.stop_requested = True
-            tui.warning("Stopping after current checkpoint... (Ctrl+C again to force quit)")
+            if has_checkpoint:
+                tui.warning("Stopping after current checkpoint... (Ctrl+C again to force quit)")
+            else:
+                tui.warning(
+                    "Stopping... partial results will be saved. (Ctrl+C again to force quit)"
+                )
             dataset_tui = get_dataset_tui()
             dataset_tui.log_event("⚠ Graceful stop requested")
             dataset_tui.status_stop_requested()
@@ -547,12 +580,22 @@ def _run_generation(
     finally:
         signal.signal(signal.SIGINT, original_handler)
 
-    # If gracefully stopped, don't save partial dataset or clean up checkpoints
+    output_config = preparation.config.get_output_config()
+    output_save_path = options.output_save_as or output_config["save_as"]
+
+    # If gracefully stopped, handle based on checkpoint availability
     if engine.stop_requested:
+        if has_checkpoint:
+            # Checkpoint on disk — user can resume later
+            return
+        # No checkpoint — save whatever was generated so far
+        if dataset and len(dataset) > 0:
+            tui.info(f"Saving {len(dataset)} samples generated before stop")
+            save_dataset(dataset, output_save_path, preparation.config, engine=engine)
+        else:
+            tui.warning("No samples were generated before stop")
         return
 
-    output_config = preparation.config.get_output_config()
-    output_save_path = options.output_save_as or output_config["save_as"]
     save_dataset(dataset, output_save_path, preparation.config, engine=engine)
 
     # Clean up checkpoint files after successful completion
@@ -651,8 +694,7 @@ def _run_generation(
     "--cloud-upload",
     type=click.Choice(["all", "dataset", "graph", "none"], case_sensitive=False),
     default=None,
-    help="Upload to DeepFabric Cloud (experimental): all, dataset, graph, or none. "
-    "Enables headless mode for CI. Requires DEEPFABRIC_API_KEY or prior auth.",
+    help="Upload to DeepFabric Cloud (experimental): all, dataset, graph, or none. ",
 )
 @click.option(
     "--checkpoint-interval",
@@ -783,7 +825,9 @@ def generate(  # noqa: PLR0913
 
         # Compute checkpoint directory once for consistent use throughout generation
         # Use config file for hash, fallback to output path for config-less runs
-        path_source = options.config_file or options.output_save_as or preparation.config.output.save_as
+        path_source = (
+            options.config_file or options.output_save_as or preparation.config.output.save_as
+        )
         checkpoint_dir = options.checkpoint_path or get_checkpoint_dir(path_source)
 
         # Auto-infer topics-load when resuming from checkpoint
@@ -1295,23 +1339,34 @@ def validate(config_file: str, check_api: bool) -> None:  # noqa: PLR0912
             f"estimated_paths={estimated_paths} ({degree}^{depth})"
         )
 
-        # Output summary with step size and checkpoint info
+        # Output summary with cycle-based generation info
         num_samples = config.output.num_samples
         batch_size = config.output.batch_size
-        # Calculate num_steps - handle 'auto' and percentage strings
-        if isinstance(num_samples, int):
-            num_steps = math.ceil(num_samples / batch_size)
-            output_info = f"Output: num_samples={num_samples}, batch_size={batch_size}, num_steps={num_steps}"
-        else:
-            # For 'auto' or percentage, we can't compute steps without topic count
-            output_info = f"Output: num_samples={num_samples}, batch_size={batch_size}"
 
-        # Add checkpoint info if enabled
+        # Show output configuration
+        output_info = f"Output: num_samples={num_samples}, concurrency={batch_size}"
         if config.output.checkpoint:
             checkpoint = config.output.checkpoint
             output_info += f", checkpoint_interval={checkpoint.interval}"
         tui.info(output_info)
 
+        # Calculate and display cycle-based generation info
+        if isinstance(num_samples, int):
+            cycles_needed = math.ceil(num_samples / estimated_paths)
+            final_cycle_size = num_samples - (cycles_needed - 1) * estimated_paths
+            is_partial = final_cycle_size < estimated_paths
+
+            tui.info(
+                f"  → Cycles needed: {cycles_needed} "
+                f"({num_samples} samples ÷ {estimated_paths} unique topics)"
+            )
+            if is_partial:
+                tui.info(f"  → Final cycle: {final_cycle_size} topics (partial)")
+        elif num_samples == "auto":
+            tui.info(f"  → Will generate 1 sample per unique topic ({estimated_paths} samples)")
+        else:
+            tui.info("  → Samples calculated at runtime based on topic count")
+
         if config.huggingface:
             hf_config = config.get_huggingface_config()
             tui.info(f"Hugging Face: repo={hf_config.get('repository', 'not set')}")
@@ -1893,7 +1948,8 @@ def checkpoint_status(config_file: str) -> None:
     # Check if checkpoint exists
     if not metadata_path.exists():
         tui.info(f"No checkpoint found at: {metadata_path}")
-        tui.info("\nTo enable checkpointing, run:")
+        tui.console.print()
+        tui.info("To enable checkpointing, run:")
         tui.info(f"  deepfabric generate {config_file} --checkpoint-interval 10")
         return
 

{deepfabric-4.10.1 → deepfabric-4.11.0}/deepfabric/cloud_upload.py

@@ -778,7 +778,7 @@ def handle_cloud_upload(  # noqa: PLR0911
 
         # Build prompt based on what's available
         if has_dataset and has_graph:
-            prompt_text = "  Upload to DeepFabric Cloud?"
+            prompt_text = "  Upload graph and dataset to DeepFabric Cloud?"
             hint = "[dim](Y=both, n=skip, c=choose)[/dim]"
         elif has_dataset:
             prompt_text = "  Upload dataset to DeepFabric Cloud?"

{deepfabric-4.10.1 → deepfabric-4.11.0}/deepfabric/config.py

@@ -628,11 +628,13 @@ See documentation for full examples.
             "output_save_as": self.output.save_as,
             # Checkpoint config (nested inside output)
             # Note: checkpoint_path can be None, meaning "auto-resolve" at runtime
-            "checkpoint_interval": self.output.checkpoint.interval if self.output.checkpoint else None,
+            "checkpoint_interval": self.output.checkpoint.interval
+            if self.output.checkpoint
+            else None,
             "checkpoint_path": self.output.checkpoint.path if self.output.checkpoint else None,
-            "checkpoint_retry_failed": (
-                self.output.checkpoint.retry_failed if self.output.checkpoint else False
-            ),
+            "checkpoint_retry_failed": self.output.checkpoint.retry_failed
+            if self.output.checkpoint
+            else False,
         }
 
         # Tool config

{deepfabric-4.10.1 → deepfabric-4.11.0}/deepfabric/constants.py

@@ -93,7 +93,7 @@ FAILED_TREE_SUFFIX = "_failed.jsonl"
 CHECKPOINT_METADATA_SUFFIX = ".checkpoint.json"
 CHECKPOINT_SAMPLES_SUFFIX = ".checkpoint.jsonl"
 CHECKPOINT_FAILURES_SUFFIX = ".checkpoint.failures.jsonl"
-CHECKPOINT_VERSION = 3  # Increment when checkpoint format changes
+CHECKPOINT_VERSION = 4  # v4: (uuid, cycle) tuple tracking for cycle-based generation
 
 # Stream simulation defaults
 STREAM_SIM_CHUNK_SIZE = 8  # characters per chunk