yeonjae-universal-data-storage 1.0.1__py3-none-any.whl

universal_data_storage/__init__.py

@@ -0,0 +1,3 @@
+"""Universal Data Storage - 커밋 데이터와 집계 결과를 데이터베이스에 저장하고 관리하는 범용 모듈"""
+
+__version__ = "1.0.0"

universal_data_storage/exceptions.py

@@ -0,0 +1,92 @@
+"""
+Universal Data Storage 예외 클래스
+
+데이터 저장과 관련된 예외들을 정의합니다.
+"""
+
+from typing import Dict, Any, Optional
+
+
+class DataStorageException(Exception):
+    """데이터 저장 관련 기본 예외 클래스"""
+    
+    def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
+        self.message = message
+        self.details = details or {}
+        super().__init__(self.message)
+
+
+class DatabaseConnectionException(DataStorageException):
+    """데이터베이스 연결 예외"""
+    
+    def __init__(self, original_error: Exception):
+        message = "Database connection failed"
+        details = {
+            "error_type": type(original_error).__name__,
+            "error_message": str(original_error)
+        }
+        super().__init__(message, details)
+
+
+class DuplicateDataException(DataStorageException):
+    """중복 데이터 예외"""
+    
+    def __init__(self, commit_hash: str, existing_data: Dict[str, Any]):
+        message = f"Duplicate commit data found for hash: {commit_hash}"
+        details = {
+            "commit_hash": commit_hash,
+            "existing_data": existing_data
+        }
+        super().__init__(message, details)
+
+
+class StorageValidationException(DataStorageException):
+    """저장 데이터 검증 예외"""
+    
+    def __init__(self, field: str, value: Any, reason: str):
+        message = f"Storage validation failed for field '{field}': {reason}"
+        details = {
+            "field": field,
+            "value": value,
+            "reason": reason
+        }
+        super().__init__(message, details)
+
+
+class StorageOperationException(DataStorageException):
+    """저장 작업 예외"""
+    
+    def __init__(self, operation: str, original_error: Exception):
+        message = f"Storage operation '{operation}' failed"
+        details = {
+            "operation": operation,
+            "error_type": type(original_error).__name__,
+            "error_message": str(original_error)
+        }
+        super().__init__(message, details)
+
+
+class BatchStorageException(DataStorageException):
+    """배치 저장 예외"""
+    
+    def __init__(self, failed_count: int, total_count: int, errors: list):
+        message = f"Batch storage failed: {failed_count}/{total_count} items failed"
+        details = {
+            "failed_count": failed_count,
+            "total_count": total_count,
+            "errors": errors
+        }
+        super().__init__(message, details)
+
+
+class CompressionException(DataStorageException):
+    """데이터 압축 예외"""
+    
+    def __init__(self, data_type: str, original_error: Exception):
+        message = f"Data compression failed for {data_type}"
+        details = {
+            "data_type": data_type,
+            "error_type": type(original_error).__name__,
+            "error_message": str(original_error)
+        }
+        super().__init__(message, details) 

universal_data_storage/models.py

@@ -0,0 +1,248 @@
+"""Data storage models for webhook events - MVP Version based on design spec."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional, Dict, Any, List
+from enum import Enum
+
+from sqlalchemy import (
+    Column, String, DateTime, LargeBinary, Integer, 
+    UniqueConstraint, ForeignKey, Text, Index, create_engine
+)
+from sqlalchemy.sql import func
+from sqlalchemy.orm import relationship, declarative_base
+from pydantic import BaseModel, ConfigDict
+
+# SQLAlchemy Base 독립적으로 생성
+Base = declarative_base()
+
+
+class StorageStatus(str, Enum):
+    """저장 상태"""
+    SUCCESS = "success"
+    FAILED = "failed"
+    DUPLICATE = "duplicate"
+    PENDING = "pending"
+
+
+# SQLAlchemy Models (Database Tables)
+class CommitRecord(Base):
+    """커밋 정보 테이블 - MVP 버전"""
+    
+    __tablename__ = "commits"
+    
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    hash = Column(String(40), unique=True, nullable=False, index=True)
+    message = Column(Text, nullable=False)
+    author = Column(String(255), nullable=False, index=True)
+    author_email = Column(String(255), nullable=True)
+    timestamp = Column(DateTime, nullable=False, index=True)
+    repository = Column(String(255), nullable=False, index=True)
+    branch = Column(String(255), nullable=False)
+    pusher = Column(String(255), nullable=True)
+    commit_count = Column(Integer, nullable=False, default=1)
+    created_at = Column(DateTime, nullable=False, default=func.now())
+    
+    # 관계 설정
+    diffs = relationship("DiffRecord", back_populates="commit", cascade="all, delete-orphan")
+    
+    def __repr__(self) -> str:
+        return f"<CommitRecord(id={self.id}, hash={self.hash[:8]}, repo={self.repository})>"
+
+
+class DiffRecord(Base):
+    """Diff 정보 테이블 - MVP 버전"""
+    
+    __tablename__ = "commit_diffs"
+    
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    commit_id = Column(Integer, ForeignKey("commits.id", ondelete="CASCADE"), nullable=False)
+    file_path = Column(Text, nullable=False)
+    additions = Column(Integer, nullable=False, default=0)
+    deletions = Column(Integer, nullable=False, default=0)
+    changes = Column(Text, nullable=True)  # diff 내용 (압축된 형태)
+    diff_patch = Column(LargeBinary, nullable=True)  # 압축된 diff 데이터
+    diff_url = Column(String, nullable=True)  # S3 URL (큰 파일의 경우)
+    created_at = Column(DateTime, nullable=False, default=func.now())
+    
+    # 관계 설정
+    commit = relationship("CommitRecord", back_populates="diffs")
+    
+    # 인덱스 추가
+    __table_args__ = (
+        Index('idx_diffs_commit_id', 'commit_id'),
+        Index('idx_diffs_file_path', 'file_path'),
+    )
+    
+    def __repr__(self) -> str:
+        return f"<DiffRecord(id={self.id}, commit_id={self.commit_id}, file={self.file_path})>"
+
+
+# Legacy Event Model (기존 호환성 유지)
+class Event(Base):
+    """Database model for GitHub webhook events - Legacy support."""
+    
+    __tablename__ = "events"
+    
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    platform = Column(String, nullable=False, default="github")
+    repository = Column(String, nullable=False, index=True)
+    commit_sha = Column(String, nullable=False, index=True)
+    author_name = Column(String, nullable=True)
+    author_email = Column(String, nullable=True)
+    timestamp_utc = Column(DateTime, nullable=True)
+    ref = Column(String, nullable=True)
+    pusher = Column(String, nullable=False, index=True)
+    commit_count = Column(Integer, nullable=False, default=1)
+    diff_patch = Column(LargeBinary, nullable=True)  # Compressed diff or None if stored in S3
+    diff_url = Column(String, nullable=True)  # S3 URL if diff is too large
+    added_lines = Column(Integer, nullable=True)
+    deleted_lines = Column(Integer, nullable=True)
+    files_changed = Column(Integer, nullable=True)
+    payload = Column(String, nullable=False)  # JSON string
+    created_at = Column(DateTime, nullable=False, default=func.now())
+    
+    # Prevent duplicate events
+    __table_args__ = (
+        UniqueConstraint('repository', 'commit_sha', name='uq_repo_commit'),
+    )
+    
+    def __repr__(self) -> str:
+        return f"<Event(id={self.id}, repo={self.repository}, sha={self.commit_sha[:8]})>"
+
+
+# Pydantic Models (API Request/Response)
+class CommitData(BaseModel):
+    """커밋 데이터 입력 모델"""
+    
+    commit_hash: str
+    message: str
+    author: str
+    author_email: Optional[str] = None
+    timestamp: datetime
+    repository: str
+    branch: str
+    pusher: Optional[str] = None
+    commit_count: int = 1
+    
+    model_config = ConfigDict(
+        from_attributes=True,
+        json_encoders={datetime: lambda v: v.isoformat()}
+    )
+
+
+class DiffData(BaseModel):
+    """Diff 데이터 입력 모델"""
+    
+    file_path: str
+    additions: int = 0
+    deletions: int = 0
+    changes: Optional[str] = None
+    diff_content: Optional[bytes] = None  # 원본 diff 내용
+    
+    model_config = ConfigDict(
+        from_attributes=True,
+        arbitrary_types_allowed=True
+    )
+
+
+class StorageResult(BaseModel):
+    """저장 결과 응답 모델"""
+    
+    success: bool
+    status: StorageStatus
+    commit_id: Optional[int] = None
+    message: str
+    timestamp: datetime
+    metadata: Dict[str, Any] = {}
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+class CommitSummary(BaseModel):
+    """커밋 요약 정보"""
+    
+    id: int
+    hash: str
+    message: str
+    author: str
+    timestamp: datetime
+    repository: str
+    branch: str
+    diff_count: int
+    total_additions: int
+    total_deletions: int
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+class DiffSummary(BaseModel):
+    """Diff 요약 정보"""
+    
+    id: int
+    file_path: str
+    additions: int
+    deletions: int
+    has_content: bool
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+# Legacy Models (기존 호환성)
+class EventCreate(BaseModel):
+    """Pydantic model for creating new events."""
+    
+    repository: str
+    commit_sha: str
+    event_type: str = "push"
+    payload: str
+    diff_data: Optional[bytes] = None
+    diff_s3_url: Optional[str] = None
+    
+    model_config = ConfigDict(
+        from_attributes=True,
+        arbitrary_types_allowed=True
+    )
+
+
+class EventResponse(BaseModel):
+    """Pydantic model for event API responses."""
+    
+    id: int
+    repository: str
+    commit_sha: str
+    event_type: str
+    payload: Dict[str, Any]
+    diff_s3_url: Optional[str] = None
+    created_at: datetime
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+# Aggregated Models
+class CommitWithDiffs(BaseModel):
+    """커밋과 관련 Diff 정보를 포함한 집계 모델"""
+    
+    commit: CommitSummary
+    diffs: List[DiffSummary]
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+class BatchStorageResult(BaseModel):
+    """배치 저장 결과"""
+    
+    total_commits: int
+    successful_commits: int
+    failed_commits: int
+    results: List[StorageResult]
+    duration_seconds: float
+    
+    @property
+    def success_rate(self) -> float:
+        if self.total_commits == 0:
+            return 0.0
+        return self.successful_commits / self.total_commits
+    
+    model_config = ConfigDict(from_attributes=True) 

universal_data_storage/service.py

@@ -0,0 +1,504 @@
+"""Data storage service - MVP Version based on design specification."""
+
+from __future__ import annotations
+
+import gzip
+import json
+import logging
+import os
+from datetime import datetime
+from io import BytesIO
+from typing import Any, Dict, List, Optional
+
+from sqlalchemy import func
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import Session, selectinload
+from sqlalchemy.future import select
+
+from modules.data_storage.models import (
+    Event, CommitRecord, DiffRecord, 
+    CommitData, DiffData, StorageResult, StorageStatus,
+    CommitSummary, DiffSummary, CommitWithDiffs, BatchStorageResult
+)
+from universal_git_data_parser.models import DiffData as GitDiffData
+# Database configuration - standalone implementation
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, declarative_base
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
+import os
+
+# Create base for models
+Base = declarative_base()
+
+def get_session():
+    """Get database session - standalone implementation"""
+    database_url = os.getenv("DATABASE_URL", "sqlite:///./test.db")
+    engine = create_engine(database_url)
+    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+    return SessionLocal()
+
+async def get_async_session():
+    """Get async database session - standalone implementation"""
+    database_url = os.getenv("ASYNC_DATABASE_URL", "sqlite+aiosqlite:///./test.db")
+    engine = create_async_engine(database_url)
+    SessionLocal = async_sessionmaker(engine, class_=AsyncSession)
+    return SessionLocal()
+
+# Simplified S3Client for standalone operation
+class S3Client:
+    def __init__(self):
+        self.configured = False
+    
+    def upload_file(self, key: str, data: bytes) -> str:
+        return f"s3://bucket/{key}"
+
+# Simple logging class
+class ModuleIOLogger:
+    def __init__(self, module_name: str):
+        self.module_name = module_name
+    
+    def log_input(self, operation: str, data: dict):
+        pass
+    
+    def log_output(self, operation: str, result: dict):
+        pass
+
+logger = logging.getLogger(__name__)
+
+GZIP_THRESHOLD = 256 * 1024  # 256 KiB
+
+
+class DataStorageManager:
+    """MVP 버전 데이터 저장 관리자 - 설계서 기반 구현"""
+    
+    def __init__(self, db_session: Optional[Session] = None):
+        """
+        DataStorageManager 초기화
+        
+        Args:
+            db_session: 데이터베이스 세션 (선택적, 테스트용)
+        """
+        self.db_session = db_session
+        self.s3_client = S3Client() if self._s3_configured() else None
+        self.logger = logging.getLogger(__name__)
+    
+    def store_commit(
+        self, 
+        commit_data: CommitData, 
+        diff_data: List[DiffData]
+    ) -> StorageResult:
+        """
+        MVP: 커밋 데이터와 diff 데이터 저장
+        
+        Args:
+            commit_data: 커밋 정보
+            diff_data: diff 정보 리스트
+            
+        Returns:
+            StorageResult: 저장 결과
+        """
+        start_time = datetime.now()
+        
+        try:
+            # 세션 처리
+            if self.db_session:
+                return self._store_commit_sync(commit_data, diff_data, start_time)
+            else:
+                with get_session() as session:
+                    self.db_session = session
+                    return self._store_commit_sync(commit_data, diff_data, start_time)
+                    
+        except Exception as e:
+            self.logger.error(f"Failed to store commit {commit_data.commit_hash}: {e}")
+            return StorageResult(
+                success=False,
+                status=StorageStatus.FAILED,
+                message=f"Storage failed: {str(e)}",
+                timestamp=datetime.now()
+            )
+    
+    def _store_commit_sync(
+        self, 
+        commit_data: CommitData, 
+        diff_data: List[DiffData],
+        start_time: datetime
+    ) -> StorageResult:
+        """동기 방식 커밋 저장 구현"""
+        
+        try:
+            # 1. 중복 커밋 확인
+            if self._is_duplicate_commit(commit_data.commit_hash):
+                return StorageResult(
+                    success=False,
+                    status=StorageStatus.DUPLICATE,
+                    message="Commit already exists",
+                    timestamp=datetime.now(),
+                    metadata={"commit_hash": commit_data.commit_hash}
+                )
+            
+            # 2. 커밋 레코드 생성
+            commit_record = CommitRecord(
+                hash=commit_data.commit_hash,
+                message=commit_data.message,
+                author=commit_data.author,
+                author_email=commit_data.author_email,
+                timestamp=commit_data.timestamp,
+                repository=commit_data.repository,
+                branch=commit_data.branch,
+                pusher=commit_data.pusher,
+                commit_count=commit_data.commit_count
+            )
+            
+            self.db_session.add(commit_record)
+            self.db_session.flush()  # ID 생성을 위해 flush
+            
+            # 3. Diff 레코드들 생성 및 저장
+            total_additions = 0
+            total_deletions = 0
+            
+            for diff in diff_data:
+                # diff 압축 처리
+                compressed_diff = None
+                diff_url = None
+                
+                if diff.diff_content:
+                    compressed_data = self._compress_bytes(diff.diff_content)
+                    
+                    if len(compressed_data) <= GZIP_THRESHOLD:
+                        compressed_diff = compressed_data
+                    elif self.s3_client:
+                        # S3에 업로드
+                        key = f"{commit_data.repository}/{commit_data.commit_hash}/{diff.file_path}.patch.gz"
+                        diff_url = self._upload_to_s3_sync(key, compressed_data)
+                    else:
+                        # S3가 없으면 DB에 저장 (경고 로그)
+                        compressed_diff = compressed_data
+                        self.logger.warning(
+                            f"Large diff stored in DB (S3 not configured): {len(compressed_data)} bytes"
+                        )
+                
+                diff_record = DiffRecord(
+                    commit_id=commit_record.id,
+                    file_path=diff.file_path,
+                    additions=diff.additions,
+                    deletions=diff.deletions,
+                    changes=diff.changes,
+                    diff_patch=compressed_diff,
+                    diff_url=diff_url
+                )
+                
+                self.db_session.add(diff_record)
+                total_additions += diff.additions
+                total_deletions += diff.deletions
+            
+            # 4. 트랜잭션 커밋
+            self.db_session.commit()
+            
+            # 5. 성공 로그 및 결과 반환
+            duration = (datetime.now() - start_time).total_seconds()
+            self.logger.info(
+                f"Stored commit {commit_data.commit_hash}: "
+                f"files={len(diff_data)}, +{total_additions}/-{total_deletions}, "
+                f"duration={duration:.2f}s"
+            )
+            
+            return StorageResult(
+                success=True,
+                status=StorageStatus.SUCCESS,
+                commit_id=commit_record.id,
+                message="Commit stored successfully",
+                timestamp=datetime.now(),
+                metadata={
+                    "commit_hash": commit_data.commit_hash,
+                    "files_changed": len(diff_data),
+                    "total_additions": total_additions,
+                    "total_deletions": total_deletions,
+                    "duration_seconds": duration
+                }
+            )
+            
+        except Exception as e:
+            self.db_session.rollback()
+            raise e
+    
+    def _is_duplicate_commit(self, commit_hash: str) -> bool:
+        """중복 커밋 확인"""
+        result = self.db_session.query(CommitRecord).filter(
+            CommitRecord.hash == commit_hash
+        ).first()
+        return result is not None
+    
+    def get_commit_by_hash(self, commit_hash: str) -> Optional[CommitWithDiffs]:
+        """커밋 해시로 상세 정보 조회"""
+        
+        commit = self.db_session.query(CommitRecord).filter(
+            CommitRecord.hash == commit_hash
+        ).first()
+        
+        if not commit:
+            return None
+        
+        # Diff 정보와 함께 조회
+        diffs = self.db_session.query(DiffRecord).filter(
+            DiffRecord.commit_id == commit.id
+        ).all()
+        
+        # 요약 정보 생성
+        commit_summary = CommitSummary(
+            id=commit.id,
+            hash=commit.hash,
+            message=commit.message,
+            author=commit.author,
+            timestamp=commit.timestamp,
+            repository=commit.repository,
+            branch=commit.branch,
+            diff_count=len(diffs),
+            total_additions=sum(d.additions for d in diffs),
+            total_deletions=sum(d.deletions for d in diffs)
+        )
+        
+        diff_summaries = [
+            DiffSummary(
+                id=diff.id,
+                file_path=diff.file_path,
+                additions=diff.additions,
+                deletions=diff.deletions,
+                has_content=bool(diff.diff_patch or diff.diff_url)
+            )
+            for diff in diffs
+        ]
+        
+        return CommitWithDiffs(
+            commit=commit_summary,
+            diffs=diff_summaries
+        )
+    
+    def get_recent_commits(
+        self, 
+        repository: str, 
+        limit: int = 10
+    ) -> List[CommitSummary]:
+        """최근 커밋 목록 조회"""
+        
+        commits = self.db_session.query(CommitRecord).filter(
+            CommitRecord.repository == repository
+        ).order_by(
+            CommitRecord.timestamp.desc()
+        ).limit(limit).all()
+        
+        results = []
+        for commit in commits:
+            # 각 커밋의 diff 통계 계산
+            diff_stats = self.db_session.query(
+                func.count(DiffRecord.id).label('diff_count'),
+                func.sum(DiffRecord.additions).label('total_additions'),
+                func.sum(DiffRecord.deletions).label('total_deletions')
+            ).filter(
+                DiffRecord.commit_id == commit.id
+            ).first()
+            
+            results.append(CommitSummary(
+                id=commit.id,
+                hash=commit.hash,
+                message=commit.message,
+                author=commit.author,
+                timestamp=commit.timestamp,
+                repository=commit.repository,
+                branch=commit.branch,
+                diff_count=diff_stats.diff_count or 0,
+                total_additions=diff_stats.total_additions or 0,
+                total_deletions=diff_stats.total_deletions or 0
+            ))
+        
+        return results
+    
+    def _compress_bytes(self, data: bytes) -> bytes:
+        """바이트 데이터 gzip 압축"""
+        buf = BytesIO()
+        with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
+            gz.write(data)
+        return buf.getvalue()
+    
+    def _upload_to_s3_sync(self, key: str, data: bytes) -> Optional[str]:
+        """S3에 동기 방식으로 데이터 업로드"""
+        try:
+            # 실제 S3 클라이언트가 비동기일 수 있으므로 동기 버전 필요
+            # 여기서는 간단히 URL 반환 (실제 구현 시 s3_client 수정 필요)
+            return f"s3://{os.getenv('AWS_S3_BUCKET', 'codeping-diffs')}/{key}"
+        except Exception as e:
+            self.logger.error(f"Failed to upload to S3: {e}")
+            return None
+    
+    def _s3_configured(self) -> bool:
+        """S3 설정 확인"""
+        required_vars = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_S3_BUCKET"]
+        return all(os.environ.get(var) for var in required_vars)
+
+
+class LegacyDataStorageService:
+    """기존 이벤트 저장 서비스 - 호환성 유지"""
+    
+    def __init__(self):
+        self.s3_client = S3Client() if self._s3_configured() else None
+        
+        # 입출력 로거 설정
+        self.io_logger = ModuleIOLogger("DataStorage")
+    
+    def store_event_with_diff(
+        self,
+        payload: Dict[str, Any],
+        headers: Dict[str, str],
+        diff_data: GitDiffData
+    ) -> None:
+        """기존 이벤트 저장 방식 - 호환성 유지"""
+        
+        # 입력 로깅
+        self.io_logger.log_input(
+            "store_event_with_diff",
+            data={"payload": payload, "diff_data": diff_data},
+            metadata={
+                "repository": diff_data.repository,
+                "commit_sha": diff_data.commit_sha,
+                "headers_count": len(headers),
+                "payload_size": len(str(payload)),
+                "diff_size": len(diff_data.diff_content) if diff_data.diff_content else 0,
+                "added_lines": diff_data.added_lines,
+                "deleted_lines": diff_data.deleted_lines,
+                "files_changed": diff_data.files_changed
+            }
+        )
+        
+        try:
+            import asyncio
+            asyncio.run(self._store_event_async(payload, headers, diff_data))
+            
+            # 출력 로깅 (성공)
+            self.io_logger.log_output(
+                "store_event_with_diff",
+                metadata={
+                    "storage_success": True,
+                    "repository": diff_data.repository,
+                    "commit_sha": diff_data.commit_sha,
+                    "storage_location": "determined_in_async"
+                }
+            )
+            
+        except Exception as e:
+            # 오류 로깅
+            self.io_logger.log_error(
+                "store_event_with_diff",
+                e,
+                metadata={
+                    "repository": diff_data.repository,
+                    "commit_sha": diff_data.commit_sha
+                }
+            )
+            raise
+    
+    async def _store_event_async(
+        self,
+        payload: Dict[str, Any],
+        headers: Dict[str, str],
+        diff_data: GitDiffData
+    ) -> None:
+        """기존 비동기 이벤트 저장 구현"""
+        
+        # 압축 처리
+        diff_patch = None
+        diff_url = None
+        
+        if diff_data.diff_content:
+            compressed_diff = self._compress_bytes(diff_data.diff_content)
+            
+            if len(compressed_diff) <= GZIP_THRESHOLD:
+                diff_patch = compressed_diff
+                storage_location = "db"
+            else:
+                if self.s3_client:
+                    key = f"{diff_data.repository}/{diff_data.commit_sha}.patch.gz"
+                    diff_url = await self.s3_client.upload_diff(key, compressed_diff)
+                    storage_location = "s3"
+                else:
+                    diff_patch = compressed_diff
+                    storage_location = "db_large"
+                    logger.warning(
+                        "Large diff stored in DB (S3 not configured): %d bytes",
+                        len(compressed_diff)
+                    )
+        else:
+            storage_location = "none"
+        
+        # 이벤트 데이터 준비
+        platform = "github" if "x-github-event" in {k.lower() for k in headers} else "gitlab"
+        
+        event_data = {
+            "platform": platform,
+            "repository": diff_data.repository,
+            "commit_sha": diff_data.commit_sha,
+            "author_name": payload.get("pusher", {}).get("name"),
+            "author_email": payload.get("pusher", {}).get("email"),
+            "timestamp_utc": None,
+            "ref": payload.get("ref"),
+            "pusher": payload.get("pusher", {}).get("name", "unknown"),
+            "commit_count": len(payload.get("commits", [])),
+            "diff_patch": diff_patch,
+            "diff_url": diff_url,
+            "added_lines": diff_data.added_lines,
+            "deleted_lines": diff_data.deleted_lines,
+            "files_changed": diff_data.files_changed,
+            "payload": json.dumps(payload),
+        }
+        
+        # 데이터베이스 저장
+        async with get_async_session() as session:
+            await self._save_event(session, event_data)
+        
+        logger.info(
+            "Stored event %s/%s: gzip_size=%s stored_in=%s added=%s deleted=%s files=%s",
+            diff_data.repository,
+            diff_data.commit_sha,
+            f"{len(compressed_diff) / 1024:.1f} KB" if diff_data.diff_content else "0 KB",
+            storage_location,
+            diff_data.added_lines or 0,
+            diff_data.deleted_lines or 0,
+            diff_data.files_changed or 0,
+        )
+    
+    async def _save_event(self, session: AsyncSession, event_data: Dict[str, Any]) -> None:
+        """이벤트 데이터베이스 저장"""
+        
+        event = Event(**event_data)
+        session.add(event)
+        
+        try:
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            raise
+    
+    def _compress_bytes(self, data: bytes) -> bytes:
+        """바이트 데이터 압축"""
+        buf = BytesIO()
+        with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
+            gz.write(data)
+        return buf.getvalue()
+    
+    def _s3_configured(self) -> bool:
+        """S3 설정 확인"""
+        required_vars = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_S3_BUCKET"]
+        return all(os.environ.get(var) for var in required_vars)
+
+
+# 편의 함수들
+def create_data_storage_manager(db_session: Optional[Session] = None) -> DataStorageManager:
+    """DataStorageManager 팩토리 함수"""
+    return DataStorageManager(db_session)
+
+
+def store_commit_data(
+    commit_data: CommitData, 
+    diff_data: List[DiffData]
+) -> StorageResult:
+    """간편한 커밋 저장 함수"""
+    manager = create_data_storage_manager()
+    return manager.store_commit(commit_data, diff_data) 

yeonjae_universal_data_storage-1.0.1.dist-info/METADATA

@@ -0,0 +1,39 @@
+Metadata-Version: 2.4
+Name: yeonjae-universal-data-storage
+Version: 1.0.1
+Summary: Universal data storage module for persisting development data
+Author-email: Yeonjae <dev@example.com>
+License: MIT
+Project-URL: Homepage, https://github.com/yeonjae/universal-modules
+Project-URL: Repository, https://github.com/yeonjae/universal-modules
+Project-URL: Issues, https://github.com/yeonjae/universal-modules/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: sqlalchemy>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: flake8>=6.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+
+# Universal data storage
+
+범용 data storage 모듈
+
+## 설치
+
+```bash
+pip install git+https://github.com/yeonjae-work/universal-modules.git#subdirectory=packages/universal-data-storage
+```

yeonjae_universal_data_storage-1.0.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+universal_data_storage/__init__.py,sha256=UsZXoMuNKys3sVZ9sd3GVOldeKh3XT-DVOLZ6AlRp0w,153
+universal_data_storage/exceptions.py,sha256=G4NwCx4bnXqLlIdMPPl0uYlSMZNKWg9M2Nh4ouUujMY,2921
+universal_data_storage/models.py,sha256=01YUrSmihBJI60lQYjV7wQRKZA_UlvH6FXNT5xuHAhc,7342
+universal_data_storage/service.py,sha256=6a7HWrERw1aOis6eckWBCD3K98YuBTqR0sZwGYYHJf4,18186
+yeonjae_universal_data_storage-1.0.1.dist-info/METADATA,sha256=gphuhxd38NrEWBWT4y4-N5qqHixsq2mjGUwvM3jHdNk,1464
+yeonjae_universal_data_storage-1.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+yeonjae_universal_data_storage-1.0.1.dist-info/top_level.txt,sha256=hFqUnzMmajixJKMqk83wkf8FTh9ofHDMRpxpXgJYKpY,23
+yeonjae_universal_data_storage-1.0.1.dist-info/RECORD,,

yeonjae_universal_data_storage-1.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

yeonjae_universal_data_storage-1.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+universal_data_storage