DeepFabric 4.11.0__py3-none-any.whl → 4.12.0__py3-none-any.whl

deepfabric/tree.py

@@ -83,6 +83,17 @@ class TreeConfig(BaseModel):
         default=None,
         description="Base URL for API endpoint (e.g., custom OpenAI-compatible servers)",
     )
+    max_concurrent: int = Field(
+        default=4,
+        ge=1,
+        le=20,
+        description="Maximum concurrent LLM calls during tree expansion (helps avoid rate limits)",
+    )
+    max_tokens: int = Field(
+        default=DEFAULT_MAX_TOKENS,
+        ge=1,
+        description="Maximum tokens for topic generation LLM calls",
+    )
 
 
 class TreeValidator:
@@ -148,6 +159,8 @@ class Tree(TopicModel):
         self.temperature = self.config.temperature
         self.provider = self.config.provider
         self.model_name = self.config.model_name
+        self.max_concurrent = self.config.max_concurrent
+        self.max_tokens = self.config.max_tokens
 
         # Initialize LLM client
         llm_kwargs = {}
@@ -300,7 +313,7 @@ class Tree(TopicModel):
                 prompt=prompt,
                 schema=TopicList,
                 max_retries=MAX_RETRY_ATTEMPTS,
-                max_tokens=DEFAULT_MAX_TOKENS,
+                max_tokens=self.max_tokens,
                 temperature=self.temperature,
             )
 
@@ -311,19 +324,11 @@ class Tree(TopicModel):
                 source="tree_generation",
             )
 
-            # Extract and validate subtopics
+            # Extract subtopics — accept whatever the LLM returned
             subtopics = topic_response.subtopics
-            if len(subtopics) >= num_subtopics:
-                return subtopics[:num_subtopics]
-
-            # If insufficient subtopics, pad with defaults
-            while len(subtopics) < num_subtopics:
-                subtopics.append(f"subtopic_{len(subtopics) + 1}_for_{node_path[-1]}")
-
             return subtopics[:num_subtopics]
 
         except Exception as e:
-            # Log the failure and return default subtopics
             self.failed_generations.append(
                 {
                     "node_path": node_path,
@@ -331,9 +336,7 @@ class Tree(TopicModel):
                     "timestamp": time.time(),
                 }
             )
-
-            # Generate default subtopics
-            return [f"subtopic_{i + 1}_for_{node_path[-1]}" for i in range(num_subtopics)]
+            return []
 
     def _detect_domain(self, system_prompt: str, node_path: list[str]) -> str:
         """Detect the appropriate domain for prompt examples based on context."""
@@ -405,20 +408,32 @@ class Tree(TopicModel):
             yield {"event": "leaf_reached", "path": node_path}
             return
 
-        async def _collect_child_events(child_subtopic: str) -> list[dict[str, Any]]:
-            child_path = node_path + [child_subtopic]
-            events: list[dict[str, Any]] = []
-            async for child_event in self._build_subtree_generator(
-                child_path, system_prompt, total_depth, n_child, current_depth + 1
-            ):
-                events.append(child_event)
-            return events
+        event_queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+
+        async def _expand_child(child_subtopic: str) -> None:
+            async with semaphore:
+                child_path = node_path + [child_subtopic]
+                async for child_event in self._build_subtree_generator(
+                    child_path, system_prompt, total_depth, n_child, current_depth + 1
+                ):
+                    await event_queue.put(child_event)
+
+        tasks = [asyncio.create_task(_expand_child(s)) for s in subtopics]
+
+        async def _signal_done() -> None:
+            await asyncio.gather(*tasks)
+            await event_queue.put(None)
+
+        done_task = asyncio.create_task(_signal_done())
 
-        tasks = [asyncio.create_task(_collect_child_events(subtopic)) for subtopic in subtopics]
+        while True:
+            event = await event_queue.get()
+            if event is None:
+                break
+            yield event
 
-        for child_events in await asyncio.gather(*tasks):
-            for child_event in child_events:
-                yield child_event
+        await done_task
 
     def save(self, save_path: str) -> None:
         """Save the topic tree to a file.

{deepfabric-4.11.0.dist-info → deepfabric-4.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DeepFabric
-Version: 4.11.0
+Version: 4.12.0
 Summary: Curate High Quality Datasets, Train, Evaluate and Ship
 Author-email: DeepFabric Team <oss@alwaysfurther.ai>
 License-File: LICENSE

{deepfabric-4.11.0.dist-info → deepfabric-4.12.0.dist-info}/RECORD

@@ -3,10 +3,10 @@ deepfabric/__main__.py,sha256=Ojx6VFnAWx4wY52VehsWhW85IaEmPb8FP_sGvOk628g,113
 deepfabric/auth.py,sha256=7UEvk84XPx7wiXj0QfhIH_MJ5BlI7wQHodwgd4LPO80,12977
 deepfabric/builders.py,sha256=2-tJh0_yzGqcBn8qRy3ocd5qIhBciwLsGPiqP1PK75w,10821
 deepfabric/builders_agent.py,sha256=youDxc8tWJF0DbA3Al-6zcswx68gcI7z9Gz7hPsDTJo,28326
-deepfabric/cli.py,sha256=rVKzUcMUWTDQnpS_Si7FzrqdQbKAN-X2Sd1ks1__Wtw,70777
+deepfabric/cli.py,sha256=cCJfGg6mhLgyldDekvq7OgmxBc4Ou74UT0MWLftJ0Sw,89114
 deepfabric/cloud_upload.py,sha256=Xu7xOycBnX0Jb7AhNBWEdEcAlxw5pXfKQQR4xcwD0BU,27819
-deepfabric/config.py,sha256=ru2eUEyHjrGdf822rn1eo179W8OxIC2YXbqUZnMRQ7Q,34006
-deepfabric/config_manager.py,sha256=c0N4Rb0P970khoiKgpN1ULDYV0bZEYIbec45cSfg92c,9334
+deepfabric/config.py,sha256=ISJpay7CJvDoY48ydvsZJnc25-omwHr-mLrKYR9Tr3g,34235
+deepfabric/config_manager.py,sha256=WH_usbD6JQ20K5xh9PzoT8oThAsnc2ywyfMy_06VuvM,9511
 deepfabric/constants.py,sha256=2DIJeIP2bqwvHjCXaZ0FKIx_8l7rf5yWSULN4Amd6_A,2846
 deepfabric/dataset.py,sha256=bZfx35A-dt0kMflgskU9Ge-NLVesq8xNKHsrxTnNn6Q,9740
 deepfabric/dataset_manager.py,sha256=3BC1fyECMxMvDkdx9POyUv1Lg98fFPAb3gAD8Z73Sn0,36151
@@ -14,7 +14,8 @@ deepfabric/error_codes.py,sha256=HGGWsahUTI8UG996C74X-XgNuaPX8RHo4gOidlaJql4,176
 deepfabric/exceptions.py,sha256=pEg4YFQaDEWtBoJaSkxsJJoBBp2-6EE3M7m5H7R6i_8,1586
 deepfabric/factory.py,sha256=OCqo3w-eiYNWvK_I_egDZuWj192kf18yD3SPj8rrPxU,753
 deepfabric/generator.py,sha256=9H_bDoGJYSxL9OSbEi7eUWk7RvhL1ofLaWvV8Z2AxWE,97072
-deepfabric/graph.py,sha256=mq__qIgMMm3YvRWTTz_ypaoX5g8iSVbdnpEEtDBe0xk,26675
+deepfabric/graph.py,sha256=eOc5GsH0b7nojPztnxyHKvRPl3Q1MpC4Xi7AIx7Rcq0,32717
+deepfabric/graph_pruner.py,sha256=bA8JhA70zI7dUX6Y6tiw6BtOMI3X3tdnwY1g6OTrMCw,3443
 deepfabric/hf_hub.py,sha256=hw2CWqZ3CzyAzMo552VPZKVWtuv-j0TQ2_gV5K0AUto,7670
 deepfabric/kaggle_hub.py,sha256=CXVO1Lv3IRhdO0bp9_IQr6nUs-v5jOWi5k4EwPkbJmw,7927
 deepfabric/loader.py,sha256=YNTGZZE-POjR0BIlx6WCT4bIzf0T4lW_fQl7ev9UFqE,18584
@@ -23,9 +24,10 @@ deepfabric/progress.py,sha256=_CWiNNBWLlNkBn-16NJnqb0DYQSzwveFaLLwOlqi-x0,10697
 deepfabric/prompts.py,sha256=pAFkw_0WAcYdzjqkHgywyLiSEdDlRdZwmyykUC307Gk,13872
 deepfabric/schemas.py,sha256=ebSwaXCu3lwshFGEhOhuAAUQA3deaYWXUcoRaeslqLc,37880
 deepfabric/stream_simulator.py,sha256=GzvAxWxHVsuTwgXlqwXNfrTUDn6sND2kJOoQuYg88FA,3028
-deepfabric/topic_manager.py,sha256=Q025z4Ri61nGrBlxdBEaSLcay_uJIo-Zr_rOoonj5gY,12109
+deepfabric/topic_inspector.py,sha256=AmoVgPMbjgN_Xpd_i8Z8j7M6zrW5yAGCeGBrRF6YEeo,8435
+deepfabric/topic_manager.py,sha256=5OLXUo7H98ukyHss8NL0S-mxySntlQky5fBbbxj2BA0,13837
 deepfabric/topic_model.py,sha256=GzxnlWhgYnAOgundlQrDxOHtaahC7bPX32nVhl-bWvM,2510
-deepfabric/tree.py,sha256=mMfHTvlBbe0x3cCLiiP6zdi-reBr-Tk54Hd0rrWt2J4,16578
+deepfabric/tree.py,sha256=m3QlGjO1UFe_c4naBEAU96n6guOJaPTnDBn2IZRibB0,16870
 deepfabric/tui.py,sha256=-sXa-FC1bl-AwpXKAQTt4wxpdJEwhokJGk-V4kwa1c8,55612
 deepfabric/update_checker.py,sha256=AUa9iUdkGNzu7tWkQRxIlF19YRmKLetwxu-Ys2ONS8Y,5145
 deepfabric/utils.py,sha256=CZSijqXHuyqStlBLvfJLnnaNTB7c8sXXaMq0RgcwkoU,12544
@@ -69,8 +71,8 @@ deepfabric/training/api_key_prompt.py,sha256=pSIMX3eDGyV9x_r7MHE4TyIsIB2SqYb8gKC
 deepfabric/training/callback.py,sha256=5zdifbHA2PWILHl2cVFyO65aW7cGAQhcvDqm3s8_I0Q,13221
 deepfabric/training/dataset_utils.py,sha256=klx8DoawEwuMigBDP-RpMAfe7FvYxRbhj599MErxBr4,7313
 deepfabric/training/metrics_sender.py,sha256=ZCyvMv5hRu8XJnQYVGXJ9wh7HEMJ0l3Ktyi8_etOpZs,10833
-deepfabric-4.11.0.dist-info/METADATA,sha256=woNg0duKMI2zC-qTA2bFWyQVmViRMShb6r2l2s9uz0Y,20607
-deepfabric-4.11.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-deepfabric-4.11.0.dist-info/entry_points.txt,sha256=zatevils13hfs8x29_vmUyivQ6rTtq7hE2RBusZw1Fo,50
-deepfabric-4.11.0.dist-info/licenses/LICENSE,sha256=OvJLTvm6YMMZmVsIiQYJI9zK9KoeeVbNZXPqQApENUM,11349
-deepfabric-4.11.0.dist-info/RECORD,,
+deepfabric-4.12.0.dist-info/METADATA,sha256=sqyIyrdJi-4bgYy5S0cyum3u0F0SVqV-64X4tJjJres,20607
+deepfabric-4.12.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+deepfabric-4.12.0.dist-info/entry_points.txt,sha256=zatevils13hfs8x29_vmUyivQ6rTtq7hE2RBusZw1Fo,50
+deepfabric-4.12.0.dist-info/licenses/LICENSE,sha256=OvJLTvm6YMMZmVsIiQYJI9zK9KoeeVbNZXPqQApENUM,11349
+deepfabric-4.12.0.dist-info/RECORD,,