sdg-hub 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

sdg_hub/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.0'
-__version_tuple__ = version_tuple = (0, 3, 0)
+__version__ = version = '0.3.1'
+__version_tuple__ = version_tuple = (0, 3, 1)
 
 __commit_id__ = commit_id = None

sdg_hub/core/blocks/llm/client_manager.py

@@ -214,8 +214,33 @@ class LLMClientManager:
             messages_list = messages
 
             if max_concurrency is not None:
+                if max_concurrency < 1:
+                    raise ValueError(
+                        "max_concurrency must be greater than 0, got {max_concurrency}"
+                    )
+                # Adjust concurrency based on n parameter to avoid overwhelming API
+                # when n > 1 (multiple completions per request)
+                n_value = overrides.get("n") or self.config.n or 1
+                if n_value > 1:
+                    # Warn if max_concurrency is less than n
+                    if max_concurrency < n_value:
+                        logger.warning(
+                            f"max_concurrency ({max_concurrency}) is less than n ({n_value}). "
+                            f"This may result in very low concurrency. Consider increasing max_concurrency "
+                            f"or reducing n for better performance."
+                        )
+
+                    # Reduce concurrency when generating multiple completions per request
+                    adjusted_concurrency = max(1, max_concurrency // n_value)
+                    logger.debug(
+                        f"Adjusted max_concurrency from {max_concurrency} to {adjusted_concurrency} "
+                        f"for n={n_value} completions per request"
+                    )
+                else:
+                    adjusted_concurrency = max_concurrency
+
                 # Use semaphore for concurrency control
-                semaphore = asyncio.Semaphore(max_concurrency)
+                semaphore = asyncio.Semaphore(adjusted_concurrency)
 
                 async def _create_with_semaphore(msgs):
                     async with semaphore:

sdg_hub/core/utils/datautils.py

@@ -1,5 +1,6 @@
 # Third Party
 from datasets import Dataset, concatenate_datasets
+import numpy as np
 
 # Local
 from .error_handling import FlowValidationError
@@ -39,28 +40,45 @@ def validate_no_duplicates(dataset: Dataset) -> None:
 
     df = dataset.to_pandas()
 
-    # Try pandas duplicated() first - only convert types if we hit unhashable error
-    try:
-        duplicate_count = int(df.duplicated(keep="first").sum())
-    except TypeError as e:
-        if "unhashable type" in str(e):
-            # Convert unhashable types to tuples so pandas can hash them
-            for col in df.columns:
-                if df[col].dtype == "object":  # Only check object columns
-                    df[col] = df[col].apply(
-                        lambda x: (
-                            tuple(sorted(x.items()))
-                            if isinstance(x, dict)
-                            else tuple(x)
-                            if hasattr(x, "__iter__")
-                            and not isinstance(x, (str, bytes))
-                            else x
-                        )
-                    )
-            duplicate_count = int(df.duplicated(keep="first").sum())
-        else:
-            raise  # Re-raise if it's a different TypeError
-
+    def is_hashable(x):
+        try:
+            hash(x)
+            return True
+        except TypeError:
+            return False
+
+    def make_hashable(x):
+        if is_hashable(x):
+            # int, float, str, bytes, None etc. are already hashable
+            return x
+        if isinstance(x, np.ndarray):
+            if x.ndim == 0:
+                return make_hashable(x.item())
+            return tuple(make_hashable(i) for i in x)
+        if isinstance(x, dict):
+            # sort robustly even with heterogeneous key types
+            return tuple(
+                sorted(
+                    ((k, make_hashable(v)) for k, v in x.items()),
+                    key=lambda kv: repr(kv[0]),
+                )
+            )
+        if isinstance(x, (set, frozenset)):
+            # order‑insensitive
+            return frozenset(make_hashable(i) for i in x)
+        if hasattr(x, "__iter__"):
+            # lists, tuples, custom iterables
+            return tuple(make_hashable(i) for i in x)
+        # last‑resort fallback to a stable representation
+        return repr(x)
+
+    # Apply to the whole dataframe to ensure every cell is hashable
+    if hasattr(df, "map"):
+        df = df.map(make_hashable)
+    else:
+        df = df.applymap(make_hashable)
+
+    duplicate_count = int(df.duplicated(keep="first").sum())
     if duplicate_count > 0:
         raise FlowValidationError(
             f"Input dataset contains {duplicate_count} duplicate rows. "

{sdg_hub-0.3.0.dist-info → sdg_hub-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.3.0
+Version: 0.3.1
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0

{sdg_hub-0.3.0.dist-info → sdg_hub-0.3.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sdg_hub/__init__.py,sha256=Tw-6R5a8_W1kJcTAsW3R9ltBDP1dy5-fe7Tvt3cSyCQ,550
-sdg_hub/_version.py,sha256=5zTqm8rgXsWYBpB2M3Zw_K1D-aV8wP7NsBLrmMKkrAQ,704
+sdg_hub/_version.py,sha256=gGLpQUQx-ty9SEy9PYw9OgJWWzJLBnCpfJOfzL7SjlI,704
 sdg_hub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/core/__init__.py,sha256=NwqB4fwhC29W50VW7QXZssLxx122YvgO9LHDLdgAnrI,496
 sdg_hub/core/blocks/__init__.py,sha256=9sCkCvDQzJGSedaePVlEIpbNwrkBz_K500VW_6FLhuE,1601
@@ -22,7 +22,7 @@ sdg_hub/core/blocks/evaluation/verify_question_block.py,sha256=LKoIHdxUuTVO24n_M
 sdg_hub/core/blocks/filtering/__init__.py,sha256=isxSVSvDqkMjG8dQSl3Q2M4g5c1t9fTjBSA21icf-yA,275
 sdg_hub/core/blocks/filtering/column_value_filter.py,sha256=H8Gif0q9Wc_d1TnVow8Zpsg7blJOFGN1EZmV6OPpkcg,5971
 sdg_hub/core/blocks/llm/__init__.py,sha256=N6-Prgd4X85oWbMQzhYMrq7OX-NTJm57cghowK-val0,844
-sdg_hub/core/blocks/llm/client_manager.py,sha256=IpMUwECL9_oNFC3yxg9A6BRqMcdg0Wdpzx28BhX45Xo,14742
+sdg_hub/core/blocks/llm/client_manager.py,sha256=6RNqYvFIh4SF6jopI6tTY5MA01y8Qo-tAhsE0GeHZZ0,16109
 sdg_hub/core/blocks/llm/config.py,sha256=gc4xp5D20MSlKMFEos0QAaKUwgbZpBtMGXmn6LsIk78,11289
 sdg_hub/core/blocks/llm/error_handler.py,sha256=7T-019ZFB9qgZoX1ybIiXyaLjPzrF96qcKmUu6vmO6g,12178
 sdg_hub/core/blocks/llm/llm_chat_block.py,sha256=9ytjxjADM0FydkLapZPSQPfzjjrFIdFONs3EJEoKnaw,23007
@@ -45,7 +45,7 @@ sdg_hub/core/flow/migration.py,sha256=6and-RBqV0t2gRipr1GiOOVnyBJdtyyjw1kO08Z--d
 sdg_hub/core/flow/registry.py,sha256=DzCqEEgwhvwnCBAGLogoMVdwXh4pCHrxOWqoxam7O8I,12162
 sdg_hub/core/flow/validation.py,sha256=pUJvgaUjLpKNwvW6djcqVOF-HShOjegEmGOnUnoX4BA,9722
 sdg_hub/core/utils/__init__.py,sha256=C2FzLn3dHprwGJDEgI4fyFS3aoCJR-9PhHsunxropJ8,351
-sdg_hub/core/utils/datautils.py,sha256=q94NzBEtNwRFhzpk3FHofgJJU0gVRgAV3AAWZ1MroFk,3860
+sdg_hub/core/utils/datautils.py,sha256=__HkUe1DxcJVHKrFX68z_hDXwxJygBlJDfjJLnj7rHc,4230
 sdg_hub/core/utils/error_handling.py,sha256=yku8cGj_nKCyXDsnb-mHCpgukkkAMucJ4iAUrIzqysc,5510
 sdg_hub/core/utils/flow_id_words.yaml,sha256=5QHpQdP7zwahRuooyAlJIwBY7WcDR7vtbJXxVJqujbg,2317
 sdg_hub/core/utils/flow_identifier.py,sha256=aAHfK_G9AwEtMglLRMdMpi_AI1dciub5UqBGm4yb2HE,2841
@@ -83,8 +83,8 @@ sdg_hub/flows/text_analysis/structured_insights/extract_entities.yaml,sha256=Q_S
 sdg_hub/flows/text_analysis/structured_insights/extract_keywords.yaml,sha256=_nPPMdHnxag_lYbhYUjGJGo-CvRwWvwdGX7cQhdZ1S0,847
 sdg_hub/flows/text_analysis/structured_insights/flow.yaml,sha256=2HuGTyNwYe6a8Ev-QdKZXwe29NL4wOkq4ecEV9a7NDg,4221
 sdg_hub/flows/text_analysis/structured_insights/summarize.yaml,sha256=WXwQak1pF8e1OwnOoI1EHu8QB6iUNW89rfkTdi1Oq54,687
-sdg_hub-0.3.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sdg_hub-0.3.0.dist-info/METADATA,sha256=eVLM1fK2-9uD_eWhSRW5VTbdUs-XIn_Va3Z-rY31Utk,9735
-sdg_hub-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sdg_hub-0.3.0.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
-sdg_hub-0.3.0.dist-info/RECORD,,
+sdg_hub-0.3.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sdg_hub-0.3.1.dist-info/METADATA,sha256=-dPDzTaPfnMb_n6p7Jcvkqv3Y-Ihi76psItQL7DQBX8,9735
+sdg_hub-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sdg_hub-0.3.1.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
+sdg_hub-0.3.1.dist-info/RECORD,,