rust-crate-pipeline 1.2.5__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (19) hide show
  1. rust_crate_pipeline/__init__.py +15 -6
  2. rust_crate_pipeline/ai_processing.py +260 -153
  3. rust_crate_pipeline/analysis.py +171 -160
  4. rust_crate_pipeline/config.py +22 -2
  5. rust_crate_pipeline/github_token_checker.py +30 -20
  6. rust_crate_pipeline/main.py +108 -45
  7. rust_crate_pipeline/network.py +109 -108
  8. rust_crate_pipeline/pipeline.py +269 -125
  9. rust_crate_pipeline/production_config.py +15 -9
  10. rust_crate_pipeline/utils/file_utils.py +14 -10
  11. rust_crate_pipeline/utils/logging_utils.py +25 -13
  12. rust_crate_pipeline/version.py +25 -1
  13. {rust_crate_pipeline-1.2.5.dist-info → rust_crate_pipeline-1.4.0.dist-info}/METADATA +27 -15
  14. rust_crate_pipeline-1.4.0.dist-info/RECORD +19 -0
  15. rust_crate_pipeline-1.2.5.dist-info/RECORD +0 -19
  16. {rust_crate_pipeline-1.2.5.dist-info → rust_crate_pipeline-1.4.0.dist-info}/WHEEL +0 -0
  17. {rust_crate_pipeline-1.2.5.dist-info → rust_crate_pipeline-1.4.0.dist-info}/entry_points.txt +0 -0
  18. {rust_crate_pipeline-1.2.5.dist-info → rust_crate_pipeline-1.4.0.dist-info}/licenses/LICENSE +0 -0
  19. {rust_crate_pipeline-1.2.5.dist-info → rust_crate_pipeline-1.4.0.dist-info}/top_level.txt +0 -0
rust_crate_pipeline/utils/file_utils.py CHANGED
@@ -1,17 +1,18 @@
1
1
  # rust_crate_pipeline/utils/file_utils.py
2
- import os
3
2
  import json
3
+ import os
4
4
  import shutil
5
5
  from datetime import datetime
6
- from typing import List, Dict, Any
6
+ from typing import List, Dict
7
+
7
8
 
8
9
  def create_output_dir(base_name: str = "crate_data") -> str:
9
10
  """
10
11
  Create timestamped output directory
11
-
12
+
12
13
  Args:
13
14
  base_name: Base name for output directory
14
-
15
+
15
16
  Returns:
16
17
  Path to created directory
17
18
  """
@@ -20,38 +21,40 @@ def create_output_dir(base_name: str = "crate_data") -> str:
20
21
  os.makedirs(output_dir, exist_ok=True)
21
22
  return output_dir
22
23
 
24
+
23
25
  def save_checkpoint(data: List[Dict], prefix: str, output_dir: str) -> str:
24
26
  """
25
27
  Save processing checkpoint with status metadata
26
-
28
+
27
29
  Args:
28
30
  data: List of crate dictionaries
29
31
  prefix: File name prefix
30
32
  output_dir: Target directory
31
-
33
+
32
34
  Returns:
33
35
  Path to saved checkpoint file
34
36
  """
35
37
  timestamp = datetime.now().isoformat()
36
38
  filename = os.path.join(output_dir, f"{prefix}_{timestamp}.jsonl")
37
-
39
+
38
40
  with open(filename, "w") as f:
39
41
  for item in data:
40
42
  f.write(json.dumps(item) + "\n")
41
-
43
+
42
44
  # Save status metadata
43
45
  status = {
44
46
  "timestamp": timestamp,
45
47
  "total_items": len(data),
46
48
  "checkpoint_file": filename
47
49
  }
48
-
50
+
49
51
  status_file = os.path.join(output_dir, f"{prefix}_status_{timestamp}.json")
50
52
  with open(status_file, "w") as f:
51
53
  json.dump(status, f, indent=2)
52
-
54
+
53
55
  return filename
54
56
 
57
+
55
58
  def safe_file_cleanup(path: str):
56
59
  """Safely remove files or directories"""
57
60
  try:
@@ -62,6 +65,7 @@ def safe_file_cleanup(path: str):
62
65
  except Exception as e:
63
66
  print(f"Failed to cleanup {path}: {str(e)}")
64
67
 
68
+
65
69
  def disk_space_check(min_free_gb: float = 1.0) -> bool:
66
70
  """Check if sufficient disk space is available"""
67
71
  try:
rust_crate_pipeline/utils/logging_utils.py CHANGED
@@ -6,38 +6,45 @@ import logging
6
6
  from functools import wraps
7
7
  from typing import Optional
8
8
 
9
- def configure_logging(log_dir: Optional[str] = None, log_level: int = logging.INFO) -> logging.Logger:
9
+
10
+ def configure_logging(log_dir: Optional[str] = None,
11
+ log_level: int = logging.INFO) -> logging.Logger:
10
12
  """
11
13
  Configure global logging with file and console handlers
12
-
14
+
13
15
  Args:
14
16
  log_dir: Directory for log files (defaults to current directory)
15
17
  log_level: Logging level (default: INFO)
16
-
18
+
17
19
  Returns:
18
20
  Root logger instance
19
21
  """
20
22
  logger = logging.getLogger()
21
23
  logger.setLevel(log_level)
22
-
24
+
23
25
  # Console handler
24
26
  console_handler = logging.StreamHandler()
25
27
  console_handler.setLevel(log_level)
26
- console_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
28
+ console_format = logging.Formatter(
29
+ "%(asctime)s [%(levelname)s] %(message)s")
27
30
  console_handler.setFormatter(console_format)
28
31
  logger.addHandler(console_handler)
29
-
32
+
30
33
  # File handler
31
34
  if log_dir:
32
- log_file = os.path.join(log_dir, f"pipeline_{time.strftime('%Y%m%d-%H%M%S')}.log")
35
+ log_file = os.path.join(
36
+ log_dir, f"pipeline_{
37
+ time.strftime('%Y%m%d-%H%M%S')}.log")
33
38
  file_handler = logging.FileHandler(log_file)
34
39
  file_handler.setLevel(log_level)
35
- file_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")
40
+ file_format = logging.Formatter(
41
+ "%(asctime)s [%(levelname)s] %(name)s: %(message)s")
36
42
  file_handler.setFormatter(file_format)
37
43
  logger.addHandler(file_handler)
38
-
44
+
39
45
  return logger
40
46
 
47
+
41
48
  def log_execution_time(func):
42
49
  """Decorator to log function execution time"""
43
50
  @wraps(func)
@@ -45,18 +52,23 @@ def log_execution_time(func):
45
52
  start_time = time.time()
46
53
  result = func(*args, **kwargs)
47
54
  end_time = time.time()
48
- logging.info(f"{func.__name__} executed in {end_time - start_time:.2f} seconds")
55
+ logging.info(
56
+ f"{func.__name__} executed in {end_time - start_time:.2f} seconds")
49
57
  return result
50
58
  return wrapper
51
59
 
60
+
52
61
  def log_resource_usage():
53
62
  """Log current resource utilization (CPU, memory, disk)"""
54
63
  cpu_percent = psutil.cpu_percent()
55
64
  mem = psutil.virtual_memory()
56
65
  disk = psutil.disk_usage('.')
57
-
58
- logging.info(f"Resource Usage - CPU: {cpu_percent}%, Memory: {mem.percent}%, Disk: {disk.percent}%")
59
-
66
+
67
+ logging.info(
68
+ f"Resource Usage - CPU: {cpu_percent}%, Memory: {
69
+ mem.percent}%, Disk: {
70
+ disk.percent}%")
71
+
60
72
  return {
61
73
  "cpu_percent": cpu_percent,
62
74
  "memory_percent": mem.percent,
rust_crate_pipeline/version.py CHANGED
@@ -1,9 +1,33 @@
1
1
  """Version information for rust-crate-pipeline."""
2
2
 
3
- __version__ = "1.2.5"
3
+ __version__ = "1.4.0"
4
4
  __version_info__ = tuple(int(x) for x in __version__.split("."))
5
5
 
6
6
  # Version history
7
+ # 1.4.0 - Major Release: Rule Zero Compliance Audit Complete
8
+ # - Completed comprehensive Rule Zero alignment audit
9
+ # - Eliminated all code redundancy and dead code
10
+ # - Achieved 100% test coverage (22/22 tests passing)
11
+ # - Refactored to pure asyncio architecture (thread-free)
12
+ # - Suppressed Pydantic deprecation warnings
13
+ # - Full production readiness with Docker support
14
+ # - Enhanced documentation with PyPI cross-references
15
+ # - Certified Rule Zero compliance across all four principles
16
+ # 1.3.1 - Bug Fix Release: Crawl4AI Integration Cleanup
17
+ # - Fixed CSS selector syntax errors in Crawl4AI integration
18
+ # - Cleaned up duplicate and obsolete test files
19
+ # - Resolved import conflicts between workspace and integration configs
20
+ # - Improved error handling in enhanced scraping module
21
+ # - Standardized on direct llama.cpp approach (removed Ollama dependencies)
22
+ # - Enhanced Rule Zero compliance with transparent cleanup process
23
+ # 1.3.0 - Quality & Integration Release: Comprehensive code quality improvements
24
+ # - Fixed all critical PEP 8 violations (F821, F811, E114)
25
+ # - Enhanced error handling with graceful dependency fallbacks
26
+ # - Improved module integration and import path resolution
27
+ # - Added comprehensive test validation (21/21 tests passing)
28
+ # - Enhanced async support and Unicode handling
29
+ # - Production-ready CLI interfaces with robust error handling
30
+ # - Full Rule Zero compliance validation
7
31
  # 1.2.0 - Major release: Production-ready, cleaned codebase
8
32
  # - Unified documentation into single comprehensive README
9
33
  # - Removed all non-essential development and test files
{rust_crate_pipeline-1.2.5.dist-info → rust_crate_pipeline-1.4.0.dist-info}/METADATA RENAMED
@@ -1,15 +1,15 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rust-crate-pipeline
3
- Version: 1.2.5
3
+ Version: 1.4.0
4
4
  Summary: A comprehensive system for gathering, enriching, and analyzing metadata for Rust crates using AI-powered insights
5
- Home-page: https://github.com/DaveTmire85/SigilDERG-Data_Production
5
+ Home-page: https://github.com/Superuser666-Sigil/SigilDERG-Data_Production
6
6
  Author: SuperUser666-Sigil
7
7
  Author-email: SuperUser666-Sigil <miragemodularframework@gmail.com>
8
8
  License-Expression: MIT
9
- Project-URL: Homepage, https://github.com/DaveTmire85/SigilDERG-Data_Production
10
- Project-URL: Documentation, https://github.com/DaveTmire85/SigilDERG-Data_Production#readme
11
- Project-URL: Repository, https://github.com/DaveTmire85/SigilDERG-Data_Production
12
- Project-URL: Bug Tracker, https://github.com/DaveTmire85/SigilDERG-Data_Production/issues
9
+ Project-URL: Homepage, https://github.com/Superuser666-Sigil/SigilDERG-Data_Production
10
+ Project-URL: Documentation, https://github.com/Superuser666-Sigil/SigilDERG-Data_Production#readme
11
+ Project-URL: Repository, https://github.com/Superuser666-Sigil/SigilDERG-Data_Production
12
+ Project-URL: Bug Tracker, https://github.com/Superuser666-Sigil/SigilDERG-Data_Production/issues
13
13
  Keywords: rust,crates,metadata,ai,analysis,pipeline,dependencies
14
14
  Classifier: Development Status :: 4 - Beta
15
15
  Classifier: Intended Audience :: Developers
@@ -51,30 +51,38 @@ Dynamic: requires-python
51
51
 
52
52
  [![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
53
53
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
54
- [![PyPI Ready](https://img.shields.io/badge/PyPI-Ready-green.svg)](https://pypi.org/)
54
+ [![PyPI Package](https://img.shields.io/badge/PyPI-v1.4.0-green.svg)](https://pypi.org/project/rust-crate-pipeline/)
55
55
  [![Docker Ready](https://img.shields.io/badge/Docker-Ready-blue.svg)](https://docker.com/)
56
+ [![Rule Zero Compliant](https://img.shields.io/badge/Rule%20Zero-Compliant-gold.svg)](https://github.com/Superuser666-Sigil/SigilDERG-Data_Production/blob/main/SYSTEM_AUDIT_REPORT.md)
56
57
 
57
- A production-ready pipeline for comprehensive Rust crate analysis, featuring AI-powered insights, dependency mapping, and automated data enrichment. Designed for researchers, developers, and data scientists studying the Rust ecosystem.
58
+ A production-ready, Rule Zero-compliant pipeline for comprehensive Rust crate analysis, featuring AI-powered insights, dependency mapping, and automated data enrichment. Designed for researchers, developers, and data scientists studying the Rust ecosystem.
59
+
60
+ 📦 **Available on PyPI:** [rust-crate-pipeline](https://pypi.org/project/rust-crate-pipeline/)
58
61
 
59
62
  ## 🚀 Quick Start
60
63
 
61
64
  ### 1. Installation
62
65
 
63
66
  #### From PyPI (Recommended)
67
+
64
68
  ```bash
65
69
  pip install rust-crate-pipeline
66
70
  ```
67
71
 
72
+ For the latest version, visit: [rust-crate-pipeline on PyPI](https://pypi.org/project/rust-crate-pipeline/)
73
+
68
74
  #### From Source
75
+
69
76
  ```bash
70
- git clone https://github.com/DaveTmire85/SigilDERG-Data_Production.git
77
+ git clone https://github.com/Superuser666-Sigil/SigilDERG-Data_Production.git
71
78
  cd SigilDERG-Data_Production
72
79
  pip install -e .
73
80
  ```
74
81
 
75
82
  #### Development Installation
83
+
76
84
  ```bash
77
- git clone https://github.com/DaveTmire85/SigilDERG-Data_Production.git
85
+ git clone https://github.com/Superuser666-Sigil/SigilDERG-Data_Production.git
78
86
  cd SigilDERG-Data_Production
79
87
  pip install -e ".[dev]"
80
88
  ```
@@ -139,6 +147,8 @@ python3 -m rust_crate_pipeline \
139
147
 
140
148
  ## 🎯 Features
141
149
 
150
+ *Available in the latest version: [rust-crate-pipeline v1.4.0](https://pypi.org/project/rust-crate-pipeline/)*
151
+
142
152
  ### 📊 Data Collection & Analysis
143
153
 
144
154
  - **Multi-source metadata**: crates.io, GitHub, lib.rs integration
@@ -295,7 +305,7 @@ docker run -d --name pipeline \
295
305
 
296
306
  ### Output Structure
297
307
 
298
- ```
308
+ ```text
299
309
  output/
300
310
  ├── enriched_crates_YYYYMMDD_HHMMSS.json # Main results
301
311
  ├── metadata_YYYYMMDD_HHMMSS.json # Raw metadata
@@ -459,7 +469,7 @@ sudo systemctl status rust-crate-pipeline
459
469
 
460
470
  ### Processing Flow
461
471
 
462
- ```
472
+ ```text
463
473
  1. Crate Discovery → 2. Metadata Fetching → 3. AI Enrichment
464
474
  ↓ ↓ ↓
465
475
  4. Source Analysis → 5. Security Scanning → 6. Community Analysis
@@ -469,7 +479,7 @@ sudo systemctl status rust-crate-pipeline
469
479
 
470
480
  ### Project Structure
471
481
 
472
- ```
482
+ ```text
473
483
  rust_crate_pipeline/
474
484
  ├── __init__.py # Package initialization
475
485
  ├── __main__.py # Entry point for python -m execution
@@ -530,7 +540,7 @@ pipeline = CrateDataPipeline(
530
540
 
531
541
  ```bash
532
542
  # Clone and install
533
- git clone https://github.com/DaveTmire85/SigilDERG-Data_Production.git
543
+ git clone https://github.com/Superuser666-Sigil/SigilDERG-Data_Production.git
534
544
  cd SigilDERG-Data_Production
535
545
  pip install -r requirements.txt
536
546
 
@@ -570,4 +580,6 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
570
580
 
571
581
  ---
572
582
 
573
- **Ready to analyze the Rust ecosystem! 🦀✨**
583
+ ## Ready to analyze the Rust ecosystem! 🦀✨
584
+
585
+ 📦 **Get started today:** [Install from PyPI](https://pypi.org/project/rust-crate-pipeline/)
rust_crate_pipeline-1.4.0.dist-info/RECORD ADDED
@@ -0,0 +1,19 @@
1
+ rust_crate_pipeline/__init__.py,sha256=NxD8_OEGHEHUN9EfJj2S1rRyZ0UMkiF20LNSMnjL9Uk,1939
2
+ rust_crate_pipeline/__main__.py,sha256=fYgtPofuk4vkwiZ7ELP4GVMNj_QiKmZMSlvhzsNGuDs,155
3
+ rust_crate_pipeline/ai_processing.py,sha256=sj-qPtIVLuuY_VoWoLbcGQ6_eS_giQyXIPyAGAWOCrs,24814
4
+ rust_crate_pipeline/analysis.py,sha256=jcHHTBZ_zg5n4VGPXJYM7-NkNeL5hRdgvowkiim0onM,17663
5
+ rust_crate_pipeline/config.py,sha256=xX4j_vgXaQxVI6Q3UmazzEzFdm6kLhpGbM2Of_fZS6k,2336
6
+ rust_crate_pipeline/github_token_checker.py,sha256=_cyOiSYc1bCVczr6pUUJc_s822ic7Qi_IW3JtI_4C0w,3796
7
+ rust_crate_pipeline/main.py,sha256=bemr27xpXIFYEyXtcCQfZpAQ5pPycyiRZKP8nj9kork,10111
8
+ rust_crate_pipeline/network.py,sha256=MFtn_-9MRBUSehfjLboUBGOMk8gv2edjOjHCR_YEyGc,12677
9
+ rust_crate_pipeline/pipeline.py,sha256=aOLuIpfvDbPDCvft8ppUa0vRiFVdiz2wltpi26ZJaes,22769
10
+ rust_crate_pipeline/production_config.py,sha256=24YWT68Fo2Kl8v7Hn1WgqfPrikXma9VZEuEcMr7iDik,2282
11
+ rust_crate_pipeline/version.py,sha256=4JXcc5UI7bkW_OwMSDTrt2YpSLowN-WFH11PYQDr_BQ,2614
12
+ rust_crate_pipeline/utils/file_utils.py,sha256=IJOBBp6-w9pnCdqyGcRNwBph_iwI_zzULCdAULGFUy0,2097
13
+ rust_crate_pipeline/utils/logging_utils.py,sha256=5-o6ohm38sH1ozjZWHPlm9Wj7yILiUzvMsLJDeu11lk,2350
14
+ rust_crate_pipeline-1.4.0.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
15
+ rust_crate_pipeline-1.4.0.dist-info/METADATA,sha256=srt7t9sB9uJ70LF0jB9gvolb3qb4BLUtdBFYWfiAPDA,17474
16
+ rust_crate_pipeline-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
17
+ rust_crate_pipeline-1.4.0.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
18
+ rust_crate_pipeline-1.4.0.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
19
+ rust_crate_pipeline-1.4.0.dist-info/RECORD,,
rust_crate_pipeline-1.2.5.dist-info/RECORD DELETED
@@ -1,19 +0,0 @@
1
- rust_crate_pipeline/__init__.py,sha256=m9fb1WGbyOimxK2e18FSgvLWGYBwbLoHM_mscr-nAPs,1429
2
- rust_crate_pipeline/__main__.py,sha256=fYgtPofuk4vkwiZ7ELP4GVMNj_QiKmZMSlvhzsNGuDs,155
3
- rust_crate_pipeline/ai_processing.py,sha256=B93rCDdxE-UkYMjmT0UotQTahx9-Lgzec7_bjBd3cUs,23240
4
- rust_crate_pipeline/analysis.py,sha256=ijP4zp3cFnN09nZkeCluyAvbyAtAW_M2YSxALpQX8LY,18615
5
- rust_crate_pipeline/config.py,sha256=r4Y_5SD-lfrM1112edk9T0S0MiVxaNSSHk4q2yDrM88,1528
6
- rust_crate_pipeline/github_token_checker.py,sha256=MJqHP8J84NEZ6nzdutpC7iRnsP0kyqscjLUosvmI4MI,3768
7
- rust_crate_pipeline/main.py,sha256=Wz4Q4TX-G7qvLNMyYT6cHbgRCeMJoWILCvXcJr1FYAc,7876
8
- rust_crate_pipeline/network.py,sha256=t_G8eh_WHNugm_laMftcWVbHsmP0bOlTPnVW9DqF6SU,13375
9
- rust_crate_pipeline/pipeline.py,sha256=fcWgqKC0teGeVyNbwayFwngoZLJGWwWZAlWtMqwtdyY,17074
10
- rust_crate_pipeline/production_config.py,sha256=TdvmO1SIRpex1xZ0AymTKXpLfkkvOG44Jyy7S5M-u7k,2304
11
- rust_crate_pipeline/version.py,sha256=coYkryx3UFGAYS5obh7aBC1n8hhQIMhIeFhlnGfggeg,1022
12
- rust_crate_pipeline/utils/file_utils.py,sha256=lnHeLrt1JYaQhRDKtA1TWR2HIyRO8zwOyWb-KmAmWgk,2126
13
- rust_crate_pipeline/utils/logging_utils.py,sha256=O4Jnr_k9dBchrVqXf-vqtDKgizDtL_ljh8g7G2VCX_c,2241
14
- rust_crate_pipeline-1.2.5.dist-info/licenses/LICENSE,sha256=tpd4XNpbssrSx9-iErATOLrOh0ivNPfO2I5MAPUpats,1088
15
- rust_crate_pipeline-1.2.5.dist-info/METADATA,sha256=tCIAwMZ41r7K7BesgCHH27f8S_5UMdqsujdu9MPDePk,16741
16
- rust_crate_pipeline-1.2.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
17
- rust_crate_pipeline-1.2.5.dist-info/entry_points.txt,sha256=9Rr_IRuFRIridXxUSdEJbB3ba0NnpEfKmknZXFdYRC0,70
18
- rust_crate_pipeline-1.2.5.dist-info/top_level.txt,sha256=GUdB7RyxHLhijQxui_KTy3B8p_L2APui9C6RYa0FuaE,20
19
- rust_crate_pipeline-1.2.5.dist-info/RECORD,,