syncvey 0.11.0__py3-none-any.whl

syncvey/__init__.py

@@ -0,0 +1,3 @@
+"""SyncVey: AWS inventory and drift detection ledger."""
+
+__version__ = "0.11.0"

syncvey/asset_manager/admin.py

@@ -0,0 +1,263 @@
+import json
+
+from django.contrib import admin
+from django.utils.html import format_html
+from django.utils.translation import gettext_lazy
+
+from .models import (
+    Organization, Membership, UserProfile,
+    System, Environment,
+    Asset,
+    ScanJob,
+    Application, AppEnvConfig, AppDependency,
+    AuditLog,
+    EolSnapshot,
+    DriftSnapshot,
+)
+
+
+# ---------------------------------------------------------------------------
+# Inline helpers
+# ---------------------------------------------------------------------------
+
+class MembershipInline(admin.TabularInline):
+    model = Membership
+    extra = 0
+    fields = ('user', 'role')
+    autocomplete_fields = ('user',)
+
+
+class EnvironmentInline(admin.TabularInline):
+    model = Environment
+    extra = 0
+    fields = ('name', 'env_type', 'tfstate_filename')
+    show_change_link = True
+
+
+class AppEnvConfigInline(admin.TabularInline):
+    model = AppEnvConfig
+    extra = 0
+    fields = ('environment', 'language_version', 'framework_version', 'deploy_method', 'runtime', 'runtime_version')
+    show_change_link = True
+
+
+class AppDependencyInline(admin.TabularInline):
+    model = AppDependency
+    extra = 0
+    fields = ('name', 'version', 'dep_type')
+
+
+# ---------------------------------------------------------------------------
+# Organization
+# ---------------------------------------------------------------------------
+
+@admin.register(Organization)
+class OrganizationAdmin(admin.ModelAdmin):
+    list_display  = ('name', 'slug', 'created_at')
+    search_fields = ('name', 'slug')
+    prepopulated_fields = {'slug': ('name',)}
+    readonly_fields = ('created_at', 'updated_at')
+    inlines = [MembershipInline]
+
+
+# ---------------------------------------------------------------------------
+# Membership
+# ---------------------------------------------------------------------------
+
+@admin.register(Membership)
+class MembershipAdmin(admin.ModelAdmin):
+    list_display  = ('user', 'organization', 'role', 'created_at')
+    list_filter   = ('role', 'organization')
+    search_fields = ('user__username', 'user__email', 'organization__name')
+    readonly_fields = ('created_at', 'updated_at')
+    autocomplete_fields = ('user', 'organization')
+
+
+# ---------------------------------------------------------------------------
+# UserProfile
+# ---------------------------------------------------------------------------
+
+@admin.register(UserProfile)
+class UserProfileAdmin(admin.ModelAdmin):
+    list_display  = ('user', 'two_factor_enabled', 'created_at')
+    list_filter   = ('two_factor_enabled',)
+    search_fields = ('user__username', 'user__email')
+    readonly_fields = ('created_at', 'updated_at', 'totp_secret')
+
+
+# ---------------------------------------------------------------------------
+# System
+# ---------------------------------------------------------------------------
+
+@admin.register(System)
+class SystemAdmin(admin.ModelAdmin):
+    list_display  = ('name', 'code', 'organization', 'aws_configured_badge', 'created_at')
+    list_filter   = ('organization',)
+    search_fields = ('name', 'code', 'organization__name')
+    readonly_fields = ('created_at', 'updated_at')
+    inlines = [EnvironmentInline]
+
+    @admin.display(description='AWS', boolean=True)
+    def aws_configured_badge(self, obj):
+        return bool(obj.aws_role_arn)
+
+
+# ---------------------------------------------------------------------------
+# Environment
+# ---------------------------------------------------------------------------
+
+@admin.register(Environment)
+class EnvironmentAdmin(admin.ModelAdmin):
+    list_display  = ('__str__', 'system', 'env_type', 'tfstate_filename', 'created_at')
+    list_filter   = ('env_type', 'system__organization')
+    search_fields = ('name', 'system__name')
+    readonly_fields = ('created_at', 'updated_at')
+
+
+# ---------------------------------------------------------------------------
+# Asset（中心モデル・JSONプロパティ表示）
+# ---------------------------------------------------------------------------
+
+@admin.register(Asset)
+class AssetAdmin(admin.ModelAdmin):
+    list_display         = ('name', 'asset_type', 'asset_category', 'provider', 'cloud_id', 'region', 'environment', 'created_at')
+    list_filter          = ('asset_type', 'asset_category', 'provider', 'environment__env_type', 'environment__system__organization')
+    search_fields        = ('name', 'cloud_id', 'region', 'environment__name', 'environment__system__name')
+    readonly_fields      = ('created_at', 'updated_at', 'raw_data_pretty')
+    list_select_related  = ('environment', 'environment__system')
+
+    @admin.display(description=gettext_lazy('Raw data (JSON)'))
+    def raw_data_pretty(self, obj):
+        if not obj.raw_data:
+            return '—'
+        formatted = json.dumps(obj.raw_data, ensure_ascii=False, indent=2)
+        return format_html(
+            '<pre style="font-size:11px;max-height:300px;overflow:auto;'
+            'background:#f8f9fa;padding:8px;border-radius:4px;white-space:pre-wrap;">'
+            '{}</pre>',
+            formatted,
+        )
+
+
+# ---------------------------------------------------------------------------
+# ScanJob
+# ---------------------------------------------------------------------------
+
+@admin.register(ScanJob)
+class ScanJobAdmin(admin.ModelAdmin):
+    list_display    = ('system', 'status', 'created_count', 'updated_count', 'started_at', 'finished_at', 'created_at')
+    list_filter     = ('status', 'system__organization')
+    search_fields   = ('system__name', 'error_message')
+    readonly_fields = ('created_at', 'updated_at', 'started_at', 'finished_at', 'created_count', 'updated_count', 'error_message')
+
+
+# ---------------------------------------------------------------------------
+# Application
+# ---------------------------------------------------------------------------
+
+@admin.register(Application)
+class ApplicationAdmin(admin.ModelAdmin):
+    list_display  = ('name', 'system', 'language', 'framework', 'repository_url', 'created_at')
+    list_filter   = ('language', 'framework', 'system__organization')
+    search_fields = ('name', 'system__name', 'description', 'repository_url')
+    readonly_fields = ('created_at', 'updated_at')
+    inlines = [AppEnvConfigInline]
+
+
+# ---------------------------------------------------------------------------
+# AppEnvConfig
+# ---------------------------------------------------------------------------
+
+@admin.register(AppEnvConfig)
+class AppEnvConfigAdmin(admin.ModelAdmin):
+    list_display  = ('application', 'environment', 'language_version', 'framework_version', 'deploy_method', 'runtime', 'runtime_version')
+    list_filter   = ('deploy_method', 'runtime', 'environment__env_type')
+    search_fields = ('application__name', 'environment__name', 'language_version', 'framework_version', 'branch')
+    readonly_fields = ('created_at', 'updated_at')
+    inlines = [AppDependencyInline]
+
+
+# ---------------------------------------------------------------------------
+# AppDependency
+# ---------------------------------------------------------------------------
+
+@admin.register(AppDependency)
+class AppDependencyAdmin(admin.ModelAdmin):
+    list_display  = ('name', 'version', 'dep_type', 'app_env_config')
+    list_filter   = ('dep_type',)
+    search_fields = ('name', 'version', 'app_env_config__application__name')
+    readonly_fields = ('created_at', 'updated_at')
+
+
+# ---------------------------------------------------------------------------
+# AuditLog (read-only)
+# ---------------------------------------------------------------------------
+
+@admin.register(AuditLog)
+class AuditLogAdmin(admin.ModelAdmin):
+    list_display  = ('created_at', 'user', 'action_badge', 'model_name', 'object_repr', 'diff_summary')
+    list_filter   = ('action', 'model_name')
+    search_fields = ('user__username', 'object_repr', 'model_name')
+    readonly_fields = ('user', 'action', 'model_name', 'object_id', 'object_repr', 'diff', 'created_at')
+    date_hierarchy = 'created_at'
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+    @admin.display(description='Action')
+    def action_badge(self, obj):
+        colors = {'create': '#16a34a', 'update': '#2563eb', 'delete': '#dc2626'}
+        color = colors.get(obj.action, '#64748b')
+        return format_html(
+            '<span style="background:{};color:#fff;padding:2px 8px;border-radius:9999px;font-size:11px;font-weight:700">{}</span>',
+            color, obj.action.upper()
+        )
+
+    @admin.display(description='Changes')
+    def diff_summary(self, obj):
+        if not obj.diff:
+            return '—'
+        parts = [f'{k}: {v["old"]} → {v["new"]}' for k, v in list(obj.diff.items())[:3]]
+        return ', '.join(parts)
+
+
+# ---------------------------------------------------------------------------
+# EOL Snapshot（読み取り専用・観測用）
+# ---------------------------------------------------------------------------
+
+@admin.register(EolSnapshot)
+class EolSnapshotAdmin(admin.ModelAdmin):
+    list_display    = ('fetched_at', 'source', 'product_count')
+    readonly_fields = ('fetched_at', 'source', 'data')
+
+    @admin.display(description=gettext_lazy('Products'))
+    def product_count(self, obj):
+        return len(obj.data or {})
+
+    def has_add_permission(self, request):
+        # スナップショットはジョブ/コマンドが作る。手動追加は不可。
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Drift Snapshot（読み取り専用・観測用）
+# ---------------------------------------------------------------------------
+
+@admin.register(DriftSnapshot)
+class DriftSnapshotAdmin(admin.ModelAdmin):
+    list_display    = ('environment', 'source', 'changed_count', 'added_count', 'removed_count', 'unchanged_count', 'detected_at')
+    list_filter     = ('source', 'environment__env_type')
+    search_fields   = ('environment__name', 'environment__system__name')
+    readonly_fields = ('environment', 'source', 'changed_count', 'added_count', 'removed_count', 'unchanged_count', 'detail', 'detected_at')
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False

syncvey/asset_manager/apps.py

@@ -0,0 +1,24 @@
+from django.apps import AppConfig
+
+
+class AssetManagerConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'asset_manager'
+    verbose_name = 'SyncVey'
+
+    def ready(self):
+        from django.db.models.signals import pre_save, post_save, post_delete
+        from .signals import handle_pre_save, handle_post_save, handle_post_delete
+        from .models import Asset, System, Environment, Application, AppEnvConfig, AppDependency
+
+        for model in [Asset, System, Environment, Application, AppEnvConfig, AppDependency]:
+            pre_save.connect(handle_pre_save, sender=model, weak=False)
+            post_save.connect(handle_post_save, sender=model, weak=False)
+            post_delete.connect(handle_post_delete, sender=model, weak=False)
+
+        # runserver は2プロセス起動するので子プロセス(RUN_MAIN=true)のみで起動
+        # pytest / manage.py test では起動しない
+        from .scheduler_guard import should_start_scheduler
+        if should_start_scheduler():
+            from .scheduler import start
+            start()

syncvey/asset_manager/autoscaling.py

@@ -0,0 +1,68 @@
+"""
+autoscaling.py
+--------------
+One source of truth for the question "is this resource owned by an Auto Scaling
+group?" — and therefore whether its coming and going is *churn*, not *drift*.
+
+The cry-wolf problem this fixes: an Auto Scaling group launches and terminates
+instances on its own. Every scale-out shows up in a live scan as a brand-new
+resource the previous scan didn't have — which the drift report counts as
+"added." That's a false alarm: the autoscaler owning an instance is the system
+working as designed, not configuration drifting from Terraform.
+
+The nice part is we need **no new API call and no new IAM permission** to know
+this. EC2 stamps every ASG-launched instance with the reserved tag
+`aws:autoscaling:groupName`, and we already read instance tags in the scanner.
+So ownership is right there in the data we've had all along.
+
+We suppress only the *existence* dimension — an instance the autoscaler
+created appearing, and one it terminated vanishing. An actual attribute change
+on a persistent instance — a security group opened, an AMI swapped — is still
+real drift and still reported.
+"""
+
+from django.conf import settings
+
+# EC2 auto-applies this reserved tag to every instance an ASG launches. Users
+# can't set it themselves, so its presence is a reliable ownership signal.
+ASG_TAG = 'aws:autoscaling:groupName'
+
+
+def autoscaling_group_of(raw_data):
+    """
+    Name of the Auto Scaling group that owns this resource, or None.
+
+    Prefers an explicit `autoscaling_group` key (set by the current scanner),
+    then falls back to the `aws:autoscaling:groupName` tag — so assets scanned
+    before this feature existed are classified correctly too.
+    """
+    if not raw_data:
+        return None
+    explicit = raw_data.get('autoscaling_group')
+    if explicit:
+        return explicit
+    tags = raw_data.get('tags') or {}
+    return tags.get(ASG_TAG) or None
+
+
+def is_autoscaling_managed(raw_data):
+    """True if an Auto Scaling group owns this resource."""
+    return autoscaling_group_of(raw_data) is not None
+
+
+def suppression_enabled():
+    """Whether autoscaling churn is suppressed from drift (default on)."""
+    return getattr(settings, 'DRIFT_SUPPRESS_AUTOSCALING', True)
+
+
+def is_autoscaling_churn(raw_data):
+    """
+    True if a resource *appearing or disappearing* should be counted as
+    autoscaling churn instead of drift.
+
+    Both directions belong to the same existence dimension: a scale-out makes
+    an instance show up, a scale-in makes one vanish, and neither is Terraform
+    drifting. Callers apply this to the "added" and "removed" branches only —
+    an attribute change on a persistent instance is never suppressed.
+    """
+    return suppression_enabled() and is_autoscaling_managed(raw_data)

syncvey/asset_manager/context_processors.py

@@ -0,0 +1,19 @@
+"""
+context_processors.py
+---------------------
+Expose the feature-flag / plugin seams to every template, so the core layout
+can show or hide UI based on which features/plugins are present without any
+hard dependency on optional apps.
+
+    {% if features.drift_history %} ... {% endif %}
+    {% for item in plugin_nav_items %} ... {% endfor %}
+"""
+
+from . import plugins
+
+
+def features(request):
+    return {
+        'features': plugins.available_features(),
+        'plugin_nav_items': plugins.plugin_nav_items(request),
+    }

syncvey/asset_manager/dependency_import.py

@@ -0,0 +1,25 @@
+"""ロックファイルの内容を、環境（AppEnvConfig）のライブラリとして台帳へ入れる。元のファイルは保存しない。"""
+from django.db import transaction
+
+from .lockfiles import parse
+
+
+def import_lockfile(cfg, filename: str, raw: bytes) -> dict:
+    """解析して、同じエコシステムのライブラリを置き換える。
+
+    ロックファイルは「いま入っているライブラリの全部」なので、同じエコシステムの行は丸ごと入れ替える
+    （消えたライブラリが台帳に残らない）。手で登録した行（エコシステム空）は触らない。
+    Returns: {'ecosystem', 'total', 'added', 'removed', 'skipped'}
+    """
+    from .models import AppDependency
+    eco, found, skipped = parse(filename, raw)
+    wanted = set(found)
+    with transaction.atomic():
+        existing = {(d.name, d.version): d for d in cfg.dependencies.filter(ecosystem=eco)}
+        stale = [d.pk for key, d in existing.items() if key not in wanted]
+        AppDependency.objects.filter(pk__in=stale).delete()
+        new = [AppDependency(app_env_config=cfg, name=n, version=v, ecosystem=eco,
+                             dep_type=AppDependency.DepType.LIBRARY)
+               for n, v in found if (n, v) not in existing]
+        AppDependency.objects.bulk_create(new)
+    return {'ecosystem': eco, 'total': len(wanted), 'added': len(new), 'removed': len(stale), 'skipped': skipped}

syncvey/asset_manager/drift.py

@@ -0,0 +1,108 @@
+"""
+drift.py
+--------
+What counts as drift, decided in one place.
+
+Two halves live here:
+
+  * ``_compute_raw_diff`` — the *attribute* question: did the values change?
+  * ``classify``          — the *existence* question (did it appear? disappear?)
+    and, crucially, the order between the two.
+
+The order is the part that kept getting rewritten. Four callers used to answer
+it independently — the environment badge, the snapshot writer, the drift report
+and the CLI — and every time the core learned a new category, some of those
+copies kept the old answer. ``removed`` shipped in 0.1.0; two copies were
+brought up to date in #24; the fourth was still calling a *deleted* resource an
+*addition* when #31 caught it. The failure mode is always the same: a resource
+that vanished has no ``raw_data_prev`` either, so judging on that field alone
+makes a deletion look like a first sighting.
+
+So callers no longer decide. They ask, and keep only their own output shape:
+counts for the badge, JSON for the CLI, ORM rows for the report.
+"""
+
+from .autoscaling import is_autoscaling_churn
+
+# 判定結果。呼び出し側はこの5つだけを知っていればよい。
+CHANGED = 'changed'
+ADDED = 'added'
+REMOVED = 'removed'
+AUTOSCALING = 'autoscaling'
+UNCHANGED = 'unchanged'
+
+# raw_data のうち diff 表示から除外するキー
+# （Terraform 内部フィールド・AWS 側が自動更新する動的パラメータ）
+_DIFF_EXCLUDE = frozenset({
+    # Terraform 内部
+    '_resource_type', '_scan_source', 'tags_all', 'timeouts', 'tags',
+    # 識別子（変わらない）
+    'arn', 'id',
+    # ECS — AWS が自動更新する動的値
+    'running_count', 'pending_count',
+    'deployment_maximum_percent', 'deployment_minimum_healthy_percent',
+    'deployments',
+    # RDS — 自動バックアップ・メンテナンス系
+    'latest_restorable_time', 'status',
+    'pending_modified_values', 'read_replica_db_instance_identifiers',
+    # EC2 — 起動・停止で変わる値
+    'launch_time', 'state_transition_reason',
+    'state_reason',
+    # 汎用タイムスタンプ
+    'last_modified', 'last_modified_time', 'created_time',
+    'created_at', 'updated_at',
+    # DynamoDB / ElastiCache / EFS / Route53 — AWS が自動更新する動的値
+    'item_count', 'table_status', 'cache_cluster_status',
+    'lifecycle_state', 'number_of_mount_targets', 'record_count',
+    'size_in_bytes',
+})
+
+
+def _compute_raw_diff(old: dict, new: dict) -> list:
+    """
+    old / new の raw_data を比較し、変更フィールドのリストを返す。
+    [{'field': str, 'old': str, 'new': str}, ...]
+
+    キーは「両方に存在するもの」の積集合(&)だけを比較する。
+    tfstate インポートは全属性(50+キー)を保存する一方、ライブスキャン
+    (scanner.py)は厳選した互換キー(~11)のみを出力するため、和集合(|)で
+    比較すると scanner が出さない tfstate 固有キーが全て「削除」と誤検知
+    される。実ドリフトは共通キー上で起きるので積集合で比較するのが正しい。
+    新規/削除リソースの検出は classify() が存在次元として別途行う。
+    """
+    changes = []
+    keys = (set(old.keys()) & set(new.keys())) - _DIFF_EXCLUDE
+    for key in sorted(keys):
+        ov = old.get(key)
+        nv = new.get(key)
+        if ov != nv:
+            changes.append({
+                'field': key,
+                'old': str(ov) if ov is not None else '',
+                'new': str(nv) if nv is not None else '',
+            })
+    return changes
+
+
+def classify(asset) -> tuple[str, list]:
+    """
+    1つの資産のドリフト区分を返す: ``(category, changes)``.
+
+    ``changes`` は CHANGED のときだけ中身が入る（それ以外は空リスト）。
+
+    存在を先に、属性を後に見る。この順序が本体で、逆にすると
+    「AWS から消えた資産」が「初めて見た資産」と同じ枝に落ちる
+    （どちらも raw_data_prev が空のため）。
+
+    Auto Scaling が湧かせた／消したインスタンスは、出現も消滅も churn として
+    AUTOSCALING に逃がす。抑制するのは存在次元だけで、生き続けている
+    ASG インスタンスの属性変更は今も本物のドリフトとして CHANGED になる。
+    """
+    if asset.missing_since:
+        # AWS から返って来なくなった。スケールインなら churn。
+        return (AUTOSCALING if is_autoscaling_churn(asset.raw_data) else REMOVED), []
+    if not asset.raw_data_prev:
+        # 初回検出。スケールアウトなら churn。
+        return (AUTOSCALING if is_autoscaling_churn(asset.raw_data) else ADDED), []
+    changes = _compute_raw_diff(asset.raw_data_prev, asset.raw_data)
+    return (CHANGED, changes) if changes else (UNCHANGED, [])