rust-crate-pipeline 1.4.1__py3-none-any.whl → 1.4.2__py3-none-any.whl

rust_crate_pipeline/main.py

@@ -468,7 +468,7 @@ def main() -> None:
                 logging.info("Falling back to standard pipeline")
 
                 logging.debug("Creating standard pipeline as Sigil fallback")
-                standard_pipeline = CrateDataPipeline(config)
+                standard_pipeline = CrateDataPipeline(config, **pipeline_kwargs)
                 logging.debug("Standard pipeline created, about to run asynchronously")
 
                 # Run standard pipeline (asynchronous)
@@ -487,7 +487,7 @@ def main() -> None:
         else:
             logging.info("Standard pipeline mode")
             logging.debug("Creating standard pipeline")
-            standard_pipeline = CrateDataPipeline(config)
+            standard_pipeline = CrateDataPipeline(config, **pipeline_kwargs)
             logging.info(f"Starting pipeline with {len(vars(args))} arguments")
             logging.debug("Standard pipeline created, about to run asynchronously")
 

rust_crate_pipeline/pipeline.py

@@ -34,10 +34,21 @@ except ImportError:
     logging.warning("Enhanced scraping not available - using basic methods")
 
 
+class CustomJSONEncoder(json.JSONEncoder):
+    """Custom JSON encoder to handle non-serializable objects"""
+    def default(self, obj):
+        if hasattr(obj, 'to_dict'):
+            return obj.to_dict()
+        elif hasattr(obj, '__dict__'):
+            return obj.__dict__
+        else:
+            return str(obj)
+
+
 class CrateDataPipeline:
     """Orchestrates the entire data collection, enrichment, and analysis pipeline."""
 
-    def __init__(self, config: PipelineConfig) -> None:
+    def __init__(self, config: PipelineConfig, crate_list: "List[str] | None" = None, **kwargs) -> None:
         self.config = config
         self.api_client = CrateAPIClient(config)
         self.github_client = GitHubBatchClient(config)
@@ -60,7 +71,13 @@ class CrateDataPipeline:
         # Initialize cargo analyzer
         self.cargo_analyzer = CrateAnalyzer(".")
         
-        self.crates = self._get_crate_list()
+        # Use provided crate_list or load from file
+        if crate_list:
+            self.crates = crate_list
+            logging.info(f"Using provided crate list: {len(crate_list)} crates")
+        else:
+            self.crates = self._get_crate_list()
+        
         self.output_dir = self._create_output_dir()
         self.enhanced_scraper: Any = (
             self._initialize_enhanced_scraper()
@@ -280,7 +297,7 @@ class CrateDataPipeline:
 
         with open(filename, "w") as f:
             for item in data:
-                f.write(json.dumps(item.to_dict()) + "\n")
+                f.write(json.dumps(item.to_dict(), cls=CustomJSONEncoder) + "\n")
 
         logging.info(f"Saved checkpoint to {filename}")
         return filename
@@ -297,7 +314,7 @@ class CrateDataPipeline:
         )
         with open(final_output_path, "w") as f:
             for item in data:
-                f.write(json.dumps(item.to_dict()) + "\n")
+                f.write(json.dumps(item.to_dict(), cls=CustomJSONEncoder) + "\n")
 
         # Save dependency analysis
         dep_file_path = os.path.join(

rust_crate_pipeline/version.py

@@ -1,12 +1,15 @@
 from typing import Dict, List, Tuple, Optional, Any
 """Version information for rust-crate-pipeline."""
 
-__version__ = "1.4.1"
+__version__ = "1.4.2"
 __version_info__ = tuple(int(x) for x in __version__.split("-")[0].split("."))
 __author__ = "SigilDERG Team"
 __email__ = "sigilderg@example.com"
 
 # Version history
+# 1.4.2 - Maintenance Release
+#         - Updated project to version 1.4.2
+#         - General maintenance and dependency updates
 # 1.2.5-dev.20250621 - Dev branch: experimental, not a formal
 # release. Originated from v1.2.5.
 # 1.2.5 - Last official release.

{rust_crate_pipeline-1.4.1.dist-info → rust_crate_pipeline-1.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust-crate-pipeline
-Version: 1.4.1
+Version: 1.4.2
 Summary: A comprehensive pipeline for analyzing Rust crates with AI enrichment and enhanced scraping
 Home-page: https://github.com/SigilDERG/rust-crate-pipeline
 Author: SigilDERG Team

{rust_crate_pipeline-1.4.1.dist-info → rust_crate_pipeline-1.4.2.dist-info}/RECORD

@@ -7,14 +7,14 @@ rust_crate_pipeline/config.py,sha256=Fw3fRKCZawKaLQi7YqsmNNku4whZi89mWzr8BVRNS5E
 rust_crate_pipeline/crate_analysis.py,sha256=GsoXemJ9VFyAbb4Sm5gY5ToTqNtOA4pI38AtngAQONk,2090
 rust_crate_pipeline/crate_list.txt,sha256=W3NxDtxvihyKp9SN85FYXX6p8Hh49IFih1M4-c-CynM,4334
 rust_crate_pipeline/github_token_checker.py,sha256=0IpTh78DSaw4znaed031cSVSZDsi92eDManPzRIIN3Y,3670
-rust_crate_pipeline/main.py,sha256=iGYEAYvXkoFFvaA6DIVGiUL3wLhiCzatB6Fvf-Yrj2A,18858
+rust_crate_pipeline/main.py,sha256=Dh-ZNFJvnuU8qawTbfT68cPUbErWOhjMWZxWsc4AH3E,18896
 rust_crate_pipeline/network.py,sha256=mWjiRvOX31piBZ2QiJ-F75DBD4l6cqzTXcQdJvHxe90,12718
-rust_crate_pipeline/pipeline.py,sha256=CqPHLLRvMOpy-3ONL6hnPahV6Vh6S4M8oDsHd_lDrPc,16203
+rust_crate_pipeline/pipeline.py,sha256=daiC3GimbM5FJK3BuVt4TSf5C6i-_sAkmUqYo0FMdv8,16855
 rust_crate_pipeline/production_config.py,sha256=uWylP9AIZZx7-9aT4sFmAKEEW9miJDxaiek8VE6WP-0,2372
 rust_crate_pipeline/progress_monitor.py,sha256=5K9KP-Xggi1JEINfRmq2W-wGUHtNIBTcocpDtB1t8iM,13743
 rust_crate_pipeline/unified_llm_processor.py,sha256=eo7KotNuqwc7_hgpFm18QLokFoufFslnvi8TnDsSYEg,25064
 rust_crate_pipeline/unified_pipeline.py,sha256=2yglmXVlQfSkVq0HVTPonDee6VxWaQWZw0X2l4lLBGw,23704
-rust_crate_pipeline/version.py,sha256=whkmTDquEVytez4svUFUBfbfK0EOvDTPA8K5TuZffbE,4481
+rust_crate_pipeline/version.py,sha256=22bKSUCFGD1KWTGmUBAw6tp4jwaVet8z2BK7UhPtdqE,4611
 rust_crate_pipeline/core/__init__.py,sha256=Sq4HWdANGqoYln7JdCog7m3BsGeR3tHdseeflvNetoQ,509
 rust_crate_pipeline/core/canon_registry.py,sha256=_3cu0akJvLc7ZnomMaLeMa8adOBYn1dtjpB0yE3vGL8,4700
 rust_crate_pipeline/core/irl_engine.py,sha256=QRZUdkN24W9XutLkj8JDplEz6FmnquUrwKsl0s2zRr4,10491
@@ -23,9 +23,9 @@ rust_crate_pipeline/scraping/__init__.py,sha256=ySkTRg7nIxgcbHJQ3L1XzcrOo281NZu0
 rust_crate_pipeline/scraping/unified_scraper.py,sha256=ZE2gkc0vQ3BOLdSX_IV-kMe8QAm2Av4M7VqpkxEKyT4,9965
 rust_crate_pipeline/utils/file_utils.py,sha256=tMaCPy7ghs9x4Hxu_sviX8MXU2sBjNvohUrvt4MejoM,2853
 rust_crate_pipeline/utils/logging_utils.py,sha256=e5jG0Yd6k3exgAdbVca46kWADJ_Qz8UJ3yEJzwTqPyI,2452
-rust_crate_pipeline-1.4.1.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
-rust_crate_pipeline-1.4.1.dist-info/METADATA,sha256=OY5aKfWvpdRnLr9oKJ0SyX1N6evt1IYvu4J4GuFjwy0,17605
-rust_crate_pipeline-1.4.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rust_crate_pipeline-1.4.1.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
-rust_crate_pipeline-1.4.1.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
-rust_crate_pipeline-1.4.1.dist-info/RECORD,,
+rust_crate_pipeline-1.4.2.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
+rust_crate_pipeline-1.4.2.dist-info/METADATA,sha256=1gnViQw0U2JkgbGieFQaV6OEK6sbNKAykivgwaZ_DAo,17605
+rust_crate_pipeline-1.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rust_crate_pipeline-1.4.2.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
+rust_crate_pipeline-1.4.2.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
+rust_crate_pipeline-1.4.2.dist-info/RECORD,,