rust-crate-pipeline 1.2.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

rust_crate_pipeline/ai_processing.py

@@ -166,34 +166,30 @@ class LLMEnricher:
         prompt: str,
         validation_func: Callable[[str], bool],
         temp: float = 0.2,
-        max_tokens: int = 256,        retries: int = 2  # Reduced default retries
+        max_tokens: int = 256,        retries: int = 4  # Increased from 2 to 4 for better success rates
     ) -> Optional[str]:
         """Run LLM with validation and automatic retry on failure"""
         for attempt in range(retries):
-            try:
-                # Adjust temperature slightly upward on retries to get different results
-                adjusted_temp = temp * (1 + (attempt * 0.1))
+            try:                # More generous temperature adjustment for better variety
+                adjusted_temp = temp * (1 + (attempt * 0.2))  # 20% increases instead of 10%
                 result = self.run_llama(prompt, temp=adjusted_temp, max_tokens=max_tokens)
                 
                 # Validate the result
                 if result and validation_func(result):
                     return result
                 
-                # If we get here, validation failed - use debug level for early attempts
-                if attempt == retries - 1:
-                    logging.warning(f"Final validation attempt failed. Using best available result.")
+                # If we get here, validation failed - use debug level for early attempts                if attempt == retries - 1:
+                    logging.debug(f"All {retries} validation attempts failed, using last available result.")
                 else:
-                    logging.debug(f"Validation failed on attempt {attempt+1}/{retries}. Retrying with modified parameters.")
-                
-                # For the last attempt, simplify the prompt
-                if attempt == retries - 2:
+                    logging.debug(f"Validation failed on attempt {attempt+1}/{retries}. Retrying with adjusted temp={adjusted_temp:.2f}")
+                  # Only simplify prompt on later attempts (attempt 2+)
+                if attempt >= 2:
                     prompt = self.simplify_prompt(prompt)
                     
             except Exception as e:
                 logging.error(f"Generation error on attempt {attempt+1}: {str(e)}")
-            
-            # Reduced backoff to minimize waiting time
-            time.sleep(1.0 + (attempt * 0.5))
+              # More generous backoff - give the model more time
+            time.sleep(2.0 + (attempt * 1.0))  # 2s, 3s, 4s, 5s delays
         
         # If we exhausted all retries, return the last result even if not perfect
         return result if 'result' in locals() else None
@@ -247,9 +243,10 @@ class LLMEnricher:
                     temp=0.3, 
                     max_tokens=300
                 )
-            
-            # Extract key dependencies for context
-            key_deps = [dep.get("crate_id") for dep in crate.dependencies[:5] if dep.get("kind") == "normal"]
+              # Extract key dependencies for context
+            key_deps = [dep.get("crate_id") for dep in crate.dependencies[:5] 
+                       if dep.get("kind") == "normal" and dep.get("crate_id")]
+            key_deps_str = ", ".join(str(dep) for dep in key_deps) if key_deps else "None"
             
             # Generate other enrichments
             enriched.feature_summary = self.summarize_features(crate)
@@ -296,13 +293,13 @@ class LLMEnricher:
 
     def classify_use_case(self, crate: CrateMetadata, readme_summary: str) -> str:
         """Classify the use case of a crate with rich context"""
-        try:
-            # Calculate available tokens for prompt (classification usually needs ~20 response tokens)
+        try:            # Calculate available tokens for prompt (classification usually needs ~20 response tokens)
             available_prompt_tokens = self.config.model_token_limit - 200  # Reserve for response
             
             joined = ", ".join(crate.keywords[:10]) if crate.keywords else "None"
-            key_deps = [dep.get("crate_id") for dep in crate.dependencies[:5] if dep.get("kind") == "normal"]
-            key_deps_str = ", ".join(key_deps) if key_deps else "None"
+            key_deps = [dep.get("crate_id") for dep in crate.dependencies[:5] 
+                       if dep.get("kind") == "normal" and dep.get("crate_id")]
+            key_deps_str = ", ".join(str(dep) for dep in key_deps) if key_deps else "None"
             
             # Adaptively truncate different sections based on importance
             token_budget = available_prompt_tokens - 400  # Reserve tokens for prompt template
@@ -341,13 +338,12 @@ class LLMEnricher:
                 f"Category (pick only one): [AI, Database, Web Framework, Networking, Serialization, Utilities, DevTools, ML, Cryptography, Unknown]\n"
                 f"<|end|>"
             )
-            
-            # Validate classification with retry
+              # Validate classification with retry - more generous parameters
             result = self.validate_and_retry(
                 prompt, 
                 validation_func=self.validate_classification,
-                temp=0.1, 
-                max_tokens=20
+                temp=0.2,  # Increased from 0.1 for more variety
+                max_tokens=50  # Increased from 20 to allow more complete responses
             )
             
             return result or "Unknown"
@@ -377,13 +373,12 @@ class LLMEnricher:
                 f"Create exactly 5 pairs.\n"
                 f"<|end|>"
             )
-            
-            # Use validation for retry
+              # Use validation for retry - more generous parameters
             result = self.validate_and_retry(
                 prompt, 
                 validation_func=self.validate_factual_pairs, 
-                temp=0.6, 
-                max_tokens=500
+                temp=0.7,  # Increased from 0.6 for more creativity
+                max_tokens=800  # Increased from 500 for more complete responses
             )
             
             return result or "Factual pairs generation failed."

rust_crate_pipeline/version.py

@@ -1,6 +1,6 @@
 """Version information for rust-crate-pipeline."""
 
-__version__ = "1.2.0"
+__version__ = "1.2.1"
 __version_info__ = tuple(int(x) for x in __version__.split("."))
 
 # Version history

{rust_crate_pipeline-1.2.0.dist-info → rust_crate_pipeline-1.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust-crate-pipeline
-Version: 1.2.0
+Version: 1.2.1
 Summary: A comprehensive system for gathering, enriching, and analyzing metadata for Rust crates using AI-powered insights
 Home-page: https://github.com/DaveTmire85/SigilDERG-Data_Production
 Author: SuperUser666-Sigil

{rust_crate_pipeline-1.2.0.dist-info → rust_crate_pipeline-1.2.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 rust_crate_pipeline/__init__.py,sha256=m9fb1WGbyOimxK2e18FSgvLWGYBwbLoHM_mscr-nAPs,1429
 rust_crate_pipeline/__main__.py,sha256=fYgtPofuk4vkwiZ7ELP4GVMNj_QiKmZMSlvhzsNGuDs,155
-rust_crate_pipeline/ai_processing.py,sha256=Ma5Oo4_pRfhoyvti_ZF6xV9zi4kEukMRzBva76F7cEM,18351
+rust_crate_pipeline/ai_processing.py,sha256=zs0eHxnoSajRqpUoZ5fXLxrK68jZ-J1Vavtsujskc9Q,18821
 rust_crate_pipeline/analysis.py,sha256=ijP4zp3cFnN09nZkeCluyAvbyAtAW_M2YSxALpQX8LY,18615
 rust_crate_pipeline/config.py,sha256=r4Y_5SD-lfrM1112edk9T0S0MiVxaNSSHk4q2yDrM88,1528
 rust_crate_pipeline/github_token_checker.py,sha256=MJqHP8J84NEZ6nzdutpC7iRnsP0kyqscjLUosvmI4MI,3768
@@ -8,12 +8,12 @@ rust_crate_pipeline/main.py,sha256=J8ORQA6s3wyWw2R3oB_IEm2J5tx1CFdspw5kb5Ep8zQ,6
 rust_crate_pipeline/network.py,sha256=t_G8eh_WHNugm_laMftcWVbHsmP0bOlTPnVW9DqF6SU,13375
 rust_crate_pipeline/pipeline.py,sha256=Uwfw4uLL3aN1gJl5xSwvvyaY9ceeP7LVr02IzNx0tPM,12033
 rust_crate_pipeline/production_config.py,sha256=2GT8bxytcrMRrcfjzpay5RTtATE3rbmDvNUBvVhrYSQ,2472
-rust_crate_pipeline/version.py,sha256=Ne-Iy0D2YOCWyWVo3gFNVhuUg4tBtSnlqGIDUEeWtws,1022
+rust_crate_pipeline/version.py,sha256=8LymcBLNSPWijVTHpd9sS_Aqp8vRoMyZLCupAOnQkfs,1022
 rust_crate_pipeline/utils/file_utils.py,sha256=lnHeLrt1JYaQhRDKtA1TWR2HIyRO8zwOyWb-KmAmWgk,2126
 rust_crate_pipeline/utils/logging_utils.py,sha256=O4Jnr_k9dBchrVqXf-vqtDKgizDtL_ljh8g7G2VCX_c,2241
-rust_crate_pipeline-1.2.0.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
-rust_crate_pipeline-1.2.0.dist-info/METADATA,sha256=0iLlshmEVa7L-CNZp2RtrG2eTyGULwT_wx-GfbckhD4,16741
-rust_crate_pipeline-1.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rust_crate_pipeline-1.2.0.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
-rust_crate_pipeline-1.2.0.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
-rust_crate_pipeline-1.2.0.dist-info/RECORD,,
+rust_crate_pipeline-1.2.1.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
+rust_crate_pipeline-1.2.1.dist-info/METADATA,sha256=zRFlOck4m9GDEHrmBM_cvzKVVKD-zBK4hzfClVbhuq0,16741
+rust_crate_pipeline-1.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rust_crate_pipeline-1.2.1.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
+rust_crate_pipeline-1.2.1.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
+rust_crate_pipeline-1.2.1.dist-info/RECORD,,