nuvu-scan 2.0.0__tar.gz → 2.0.1__tar.gz

{nuvu_scan-2.0.0 → nuvu_scan-2.0.1}/.github/workflows/ci.yml

@@ -9,6 +9,8 @@ on:
 jobs:
   lint:
     runs-on: ubuntu-latest
+    # Only run lint on PRs - already passed before merge
+    if: github.event_name == 'pull_request'
     steps:
       - uses: actions/checkout@v4
 
@@ -28,37 +30,44 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
+    # On PRs: wait for lint. On push to main: run directly
     needs: lint
+    if: always() && (needs.lint.result == 'success' || needs.lint.result == 'skipped')
     strategy:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
-
+    
     steps:
       - uses: actions/checkout@v4
-
+      
       - name: Install uv
         uses: astral-sh/setup-uv@v4
         with:
           version: "latest"
-
+      
       - name: Set up Python ${{ matrix.python-version }}
         run: uv python install ${{ matrix.python-version }}
-
+      
       - name: Install dependencies
         run: |
           uv sync --dev
-
+      
+      - name: Run linter
+        run: |
+          uv run ruff check .
+          uv run black --check .
+      
       - name: Run type checker
         run: |
           uv run mypy nuvu_scan || true  # Allow failures for now
-
+      
       - name: Run tests
         run: |
           uv run pytest --cov=nuvu_scan --cov-report=xml
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
+      
       - name: Upload coverage
         uses: codecov/codecov-action@v3
         with:
@@ -69,22 +78,22 @@ jobs:
     runs-on: ubuntu-latest
     needs: test
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-
+    
     steps:
       - uses: actions/checkout@v4
-
+      
       - name: Install uv
         uses: astral-sh/setup-uv@v4
         with:
           version: "latest"
-
+      
       - name: Set up Python
         run: uv python install 3.11
-
+      
       - name: Build package
         run: |
           uv build
-
+      
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:

{nuvu_scan-2.0.0 → nuvu_scan-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nuvu-scan
-Version: 2.0.0
+Version: 2.0.1
 Summary: Multi-Cloud Data Asset Control - Discover, govern, and optimize your cloud data assets across AWS and GCP
 Project-URL: Homepage, https://nuvu.dev
 Project-URL: Documentation, https://github.com/nuvudev/nuvu-scan#readme
@@ -35,6 +35,7 @@ Requires-Dist: google-cloud-dataproc>=5.4.0
 Requires-Dist: google-cloud-monitoring>=2.16.0
 Requires-Dist: google-cloud-pubsub>=2.18.0
 Requires-Dist: google-cloud-storage>=2.10.0
+Requires-Dist: httpx>=0.25.0
 Provides-Extra: dev
 Requires-Dist: bandit>=1.7.0; extra == 'dev'
 Requires-Dist: boto3-stubs[essential]>=1.28.0; extra == 'dev'
@@ -195,40 +196,16 @@ nuvu scan --provider gcp --gcp-credentials /path/to/service-account-key.json --g
 nuvu scan --provider gcp --gcp-project your-project-id --output-format json --output-file gcp-report.json
 ```
 
-### Selective Scanning (Collectors)
+### Push to Remote API (Optional)
 
-Run focused scans on specific services instead of a full scan:
+You can optionally push scan results to a remote API for centralized tracking:
 
 ```bash
-# List available collectors for a provider
-nuvu scan --provider aws --list-collectors
-# Output: athena, glue, iam, mwaa, redshift, s3
-
-nuvu scan --provider gcp --list-collectors
-# Output: bigquery, dataproc, gcs, gemini, iam, pubsub
-
-# Scan only Redshift
-nuvu scan --provider aws -c redshift --region us-west-2
-
-# Scan multiple specific collectors
-nuvu scan --provider aws -c redshift -c glue --region us-west-2
-
-# Scan only S3 buckets
-nuvu scan --provider aws -c s3 --output-format html
-
-# Full scan (default - all collectors)
-nuvu scan --provider aws  # Runs all collectors
-
-# GCP: Scan only BigQuery
-nuvu scan --provider gcp -c bigquery --gcp-project your-project
+# Push results to a remote endpoint
+nuvu scan --provider aws --push --api-key your-api-key --api-url https://your-api.example.com
 ```
 
-**Benefits of selective scanning:**
-- **Faster scans** - Focus on services you care about
-- **Reduced API calls** - Only query the services you need
-- **Targeted reports** - Generate reports for specific areas
-
----
+This is useful for integrating with your own data governance platforms or CI/CD pipelines.
 
 ## Features
 

{nuvu_scan-2.0.0 → nuvu_scan-2.0.1}/README.md

@@ -148,40 +148,16 @@ nuvu scan --provider gcp --gcp-credentials /path/to/service-account-key.json --g
 nuvu scan --provider gcp --gcp-project your-project-id --output-format json --output-file gcp-report.json
 ```
 
-### Selective Scanning (Collectors)
+### Push to Remote API (Optional)
 
-Run focused scans on specific services instead of a full scan:
+You can optionally push scan results to a remote API for centralized tracking:
 
 ```bash
-# List available collectors for a provider
-nuvu scan --provider aws --list-collectors
-# Output: athena, glue, iam, mwaa, redshift, s3
-
-nuvu scan --provider gcp --list-collectors
-# Output: bigquery, dataproc, gcs, gemini, iam, pubsub
-
-# Scan only Redshift
-nuvu scan --provider aws -c redshift --region us-west-2
-
-# Scan multiple specific collectors
-nuvu scan --provider aws -c redshift -c glue --region us-west-2
-
-# Scan only S3 buckets
-nuvu scan --provider aws -c s3 --output-format html
-
-# Full scan (default - all collectors)
-nuvu scan --provider aws  # Runs all collectors
-
-# GCP: Scan only BigQuery
-nuvu scan --provider gcp -c bigquery --gcp-project your-project
+# Push results to a remote endpoint
+nuvu scan --provider aws --push --api-key your-api-key --api-url https://your-api.example.com
 ```
 
-**Benefits of selective scanning:**
-- **Faster scans** - Focus on services you care about
-- **Reduced API calls** - Only query the services you need
-- **Targeted reports** - Generate reports for specific areas
-
----
+This is useful for integrating with your own data governance platforms or CI/CD pipelines.
 
 ## Features
 

{nuvu_scan-2.0.0 → nuvu_scan-2.0.1}/nuvu_scan/cli/commands/scan.py

@@ -98,24 +98,18 @@ from ..formatters.json import JSONFormatter
 @click.option(
     "--push",
     is_flag=True,
-    help="Push scan results to Nuvu Cloud (requires API key)",
-)
-@click.option(
-    "--nuvu-cloud-url",
-    envvar="NUVU_CLOUD_URL",
-    default="https://nuvu.dev",
-    show_default=True,
-    help="Nuvu Cloud base URL",
+    help="Push scan results to Nuvu Cloud (requires --api-key)",
 )
 @click.option(
     "--api-key",
     envvar="NUVU_API_KEY",
-    help="Nuvu Cloud API key (from dashboard account settings)",
+    help="Nuvu Cloud API key for pushing results (default: from NUVU_API_KEY env var)",
 )
 @click.option(
-    "--list-collectors",
-    is_flag=True,
-    help="List available collectors for the specified provider and exit",
+    "--api-url",
+    envvar="NUVU_API_URL",
+    default="https://nuvu.dev",
+    help="Nuvu Cloud API URL (default: https://nuvu.dev)",
 )
 def scan_command(
     provider: str,
@@ -134,8 +128,8 @@ def scan_command(
     gcp_credentials: str | None,
     gcp_project: str | None,
     push: bool,
-    nuvu_cloud_url: str | None,
     api_key: str | None,
+    api_url: str,
     list_collectors: bool,
 ):
     """Scan cloud provider for data assets."""
@@ -311,41 +305,71 @@ def scan_command(
             f.write(content)
         click.echo(f"Report written to {output_file}", err=True)
 
+    # Push to Nuvu Cloud if requested
     if push:
-        if not nuvu_cloud_url:
-            click.echo("Error: --nuvu-cloud-url or NUVU_CLOUD_URL is required for --push", err=True)
-            sys.exit(1)
         if not api_key:
-            click.echo("Error: --api-key or NUVU_API_KEY is required for --push", err=True)
+            click.echo(
+                "Error: --api-key required for pushing results (or set NUVU_API_KEY env var)",
+                err=True,
+            )
             sys.exit(1)
 
-        payload = json.loads(JSONFormatter().format(result))
-        payload["scan_regions"] = list(region) if region else None
-        payload["scan_all_regions"] = False if region else True
+        click.echo(f"Pushing results to Nuvu Cloud ({api_url})...", err=True)
+        try:
+            import httpx
 
-        import_url = nuvu_cloud_url.rstrip("/") + "/api/scans/import"
-        request = Request(
-            import_url,
-            data=json.dumps(payload).encode("utf-8"),
-            headers={
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {api_key}",
-            },
-            method="POST",
-        )
+            # Prepare payload matching ScanImport schema
+            # Extract unique regions from assets
+            scan_regions = list(set(asset.region for asset in result.assets if asset.region))
 
-        try:
-            with urlopen(request) as response:
-                response_body = response.read().decode("utf-8")
-                click.echo(f"Scan uploaded to Nuvu Cloud: {response.status}", err=True)
-                if response_body:
-                    click.echo(response_body, err=True)
-        except HTTPError as e:
-            error_body = e.read().decode("utf-8")
-            click.echo(f"Failed to upload scan: {e.code} {e.reason}", err=True)
-            if error_body:
-                click.echo(error_body, err=True)
+            payload = {
+                "provider": provider,
+                "account_id": result.account_id or "unknown",
+                "scan_timestamp": result.scan_timestamp or datetime.utcnow().isoformat(),
+                "total_cost_estimate_usd": result.total_cost_estimate_usd,
+                "scan_regions": scan_regions if scan_regions else None,
+                "scan_all_regions": not bool(region),
+                "assets": [
+                    {
+                        "provider": asset.provider,
+                        "asset_type": asset.asset_type,
+                        "normalized_category": asset.normalized_category.value if asset.normalized_category else "unknown",
+                        "service": asset.service or asset.asset_type.split("_")[0] if asset.asset_type else "unknown",
+                        "region": asset.region,
+                        "arn": asset.arn,
+                        "name": asset.name,
+                        "created_at": asset.created_at,
+                        "last_activity_at": asset.last_activity_at,
+                        "size_bytes": asset.size_bytes,
+                        "tags": asset.tags,
+                        "cost_estimate_usd": asset.cost_estimate_usd,
+                        "risk_flags": asset.risk_flags,
+                        "ownership_confidence": asset.ownership_confidence or "unknown",
+                        "suggested_owner": asset.suggested_owner,
+                    }
+                    for asset in result.assets
+                ],
+            }
+
+            # Push to API using the /api/scans/import endpoint
+            with httpx.Client(timeout=60) as client:
+                response = client.post(
+                    f"{api_url.rstrip('/')}/api/scans/import",
+                    json=payload,
+                    headers={
+                        "Authorization": f"Bearer {api_key}",
+                        "Content-Type": "application/json",
+                    },
+                )
+                response.raise_for_status()
+                result_data = response.json()
+                click.echo(
+                    f"✓ Pushed {len(result.assets)} assets to Nuvu Cloud (scan_id: {result_data.get('id', 'N/A')})",
+                    err=True,
+                )
+        except httpx.HTTPStatusError as e:
+            click.echo(f"Error pushing to Nuvu Cloud: {e.response.status_code} - {e.response.text}", err=True)
             sys.exit(1)
-        except URLError as e:
-            click.echo(f"Failed to upload scan: {e.reason}", err=True)
+        except Exception as e:
+            click.echo(f"Error pushing to Nuvu Cloud: {e}", err=True)
             sys.exit(1)

{nuvu_scan-2.0.0 → nuvu_scan-2.0.1}/nuvu_scan/core/providers/aws/aws_scanner.py

@@ -33,55 +33,19 @@ class AWSScanner(CloudProviderScan):
     def __init__(self, config: ScanConfig):
         super().__init__(config)
         self.session = self._create_session()
-        if not self.config.regions:
-            self.config.regions = self._resolve_regions()
-        self.collectors = self._initialize_collectors()
-        self.cost_explorer = CostExplorerCollector(self.session, self.config.regions)
-
         # Auto-detect account ID if not provided
         if not self.config.account_id:
             self.config.account_id = self._get_account_id()
+        self.collectors = self._initialize_collectors()
+        self.cost_explorer = CostExplorerCollector(self.session, self.config.regions)
 
-    def scan(self):
-        """Execute a full scan with AWS-specific cost handling."""
-        from datetime import datetime
-
-        # Discover assets
-        assets = self.list_assets()
-
-        # Analyze each asset
-        total_estimated_cost = 0.0
-        actual_total_cost = None
-        actual_service_costs = None
-
-        for asset in assets:
-            if asset.asset_type == "cost_summary":
-                actual_total_cost = asset.usage_metrics.get("total_actual_cost_30d")
-                actual_service_costs = asset.usage_metrics.get("actual_costs_30d")
-                continue
-
-            asset.usage_metrics = self.get_usage_metrics(asset)
-            asset.cost_estimate_usd = self.get_cost_estimate(asset)
-            total_estimated_cost += asset.cost_estimate_usd or 0.0
-
-        # Build summary
-        summary = self._build_summary(assets)
-        if actual_total_cost is not None:
-            summary["total_actual_cost_30d"] = actual_total_cost
-            summary["actual_costs_30d"] = actual_service_costs or {}
-            summary["estimated_assets_cost_total"] = total_estimated_cost
-
-        # Use actual 30-day cost if available, otherwise fallback to estimates
-        total_cost = actual_total_cost if actual_total_cost is not None else total_estimated_cost
-
-        return ScanResult(
-            provider=self.provider,
-            account_id=self.config.account_id or "unknown",
-            scan_timestamp=datetime.utcnow().isoformat(),
-            assets=assets,
-            total_cost_estimate_usd=total_cost,
-            summary=summary,
-        )
+    def _get_account_id(self) -> str:
+        """Get the AWS account ID using STS."""
+        try:
+            sts_client = self.session.client("sts")
+            return sts_client.get_caller_identity()["Account"]
+        except Exception:
+            return "unknown"
 
     def _create_session(self) -> boto3.Session:
         """

{nuvu_scan-2.0.0 → nuvu_scan-2.0.1}/nuvu_scan/core/providers/gcp/gcp_scanner.py

@@ -27,6 +27,9 @@ class GCPScanner(CloudProviderScan):
         super().__init__(config)
         self.credentials = self._create_credentials()
         self.project_id = self._get_project_id()
+        # Set account_id to project_id for consistency in ScanResult
+        if not self.config.account_id:
+            self.config.account_id = self.project_id
         self.collectors = self._initialize_collectors()
 
     def _create_credentials(self):

{nuvu_scan-2.0.0 → nuvu_scan-2.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "nuvu-scan"
-version = "2.0.0"
+version = "2.0.1"
 description = "Multi-Cloud Data Asset Control - Discover, govern, and optimize your cloud data assets across AWS and GCP"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -43,6 +43,7 @@ dependencies = [
     "boto3>=1.28.0",
     "botocore>=1.31.0",
     "click>=8.0.0",
+    "httpx>=0.25.0",
     "google-cloud-storage>=2.10.0",
     "google-cloud-bigquery>=3.11.0",
     "google-cloud-dataproc>=5.4.0",
@@ -70,6 +71,7 @@ dev = [
     "pytest-cov>=4.0.0",
     "ruff>=0.8.0",
     "mypy>=1.0.0",
+    "pre-commit>=4.5.1",
     "boto3-stubs[essential]>=1.28.0",
     "pre-commit>=3.5.0",
     "bandit>=1.7.0",