yeonjae-universal-data-storage 1.0.1__tar.gz

yeonjae_universal_data_storage-1.0.1/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.4
+Name: yeonjae-universal-data-storage
+Version: 1.0.1
+Summary: Universal data storage module for persisting development data
+Author-email: Yeonjae <dev@example.com>
+License: MIT
+Project-URL: Homepage, https://github.com/yeonjae/universal-modules
+Project-URL: Repository, https://github.com/yeonjae/universal-modules
+Project-URL: Issues, https://github.com/yeonjae/universal-modules/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: sqlalchemy>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: flake8>=6.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+
+# Universal data storage
+
+범용 data storage 모듈
+
+## 설치
+
+```bash
+pip install git+https://github.com/yeonjae-work/universal-modules.git#subdirectory=packages/universal-data-storage
+```

yeonjae_universal_data_storage-1.0.1/README.md

@@ -0,0 +1,9 @@
+# Universal data storage
+
+범용 data storage 모듈
+
+## 설치
+
+```bash
+pip install git+https://github.com/yeonjae-work/universal-modules.git#subdirectory=packages/universal-data-storage
+```

yeonjae_universal_data_storage-1.0.1/pyproject.toml

@@ -0,0 +1,81 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "yeonjae-universal-data-storage"
+version = "1.0.1"
+description = "Universal data storage module for persisting development data"
+authors = [{name = "Yeonjae", email = "dev@example.com"}]
+license = {text = "MIT"}
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+    "pydantic>=2.0.0",
+    "sqlalchemy>=2.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "pytest-asyncio>=0.21.0",
+    "black>=23.0.0",
+    "isort>=5.12.0",
+    "flake8>=6.0.0",
+    "mypy>=1.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/yeonjae/universal-modules"
+Repository = "https://github.com/yeonjae/universal-modules"
+Issues = "https://github.com/yeonjae/universal-modules/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+universal_data_storage = ["py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = "-v --tb=short"
+
+[tool.coverage.run]
+source = ["src/universal_data_storage"]
+omit = ["tests/*"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+]
+
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+
+[tool.black]
+line-length = 100
+target-version = ['py39']
+
+[tool.isort]
+profile = "black"
+line_length = 100 

yeonjae_universal_data_storage-1.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

yeonjae_universal_data_storage-1.0.1/src/universal_data_storage/__init__.py

@@ -0,0 +1,3 @@
+"""Universal Data Storage - 커밋 데이터와 집계 결과를 데이터베이스에 저장하고 관리하는 범용 모듈"""
+
+__version__ = "1.0.0"

yeonjae_universal_data_storage-1.0.1/src/universal_data_storage/exceptions.py

@@ -0,0 +1,92 @@
+"""
+Universal Data Storage 예외 클래스
+
+데이터 저장과 관련된 예외들을 정의합니다.
+"""
+
+from typing import Dict, Any, Optional
+
+
+class DataStorageException(Exception):
+    """데이터 저장 관련 기본 예외 클래스"""
+    
+    def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
+        self.message = message
+        self.details = details or {}
+        super().__init__(self.message)
+
+
+class DatabaseConnectionException(DataStorageException):
+    """데이터베이스 연결 예외"""
+    
+    def __init__(self, original_error: Exception):
+        message = "Database connection failed"
+        details = {
+            "error_type": type(original_error).__name__,
+            "error_message": str(original_error)
+        }
+        super().__init__(message, details)
+
+
+class DuplicateDataException(DataStorageException):
+    """중복 데이터 예외"""
+    
+    def __init__(self, commit_hash: str, existing_data: Dict[str, Any]):
+        message = f"Duplicate commit data found for hash: {commit_hash}"
+        details = {
+            "commit_hash": commit_hash,
+            "existing_data": existing_data
+        }
+        super().__init__(message, details)
+
+
+class StorageValidationException(DataStorageException):
+    """저장 데이터 검증 예외"""
+    
+    def __init__(self, field: str, value: Any, reason: str):
+        message = f"Storage validation failed for field '{field}': {reason}"
+        details = {
+            "field": field,
+            "value": value,
+            "reason": reason
+        }
+        super().__init__(message, details)
+
+
+class StorageOperationException(DataStorageException):
+    """저장 작업 예외"""
+    
+    def __init__(self, operation: str, original_error: Exception):
+        message = f"Storage operation '{operation}' failed"
+        details = {
+            "operation": operation,
+            "error_type": type(original_error).__name__,
+            "error_message": str(original_error)
+        }
+        super().__init__(message, details)
+
+
+class BatchStorageException(DataStorageException):
+    """배치 저장 예외"""
+    
+    def __init__(self, failed_count: int, total_count: int, errors: list):
+        message = f"Batch storage failed: {failed_count}/{total_count} items failed"
+        details = {
+            "failed_count": failed_count,
+            "total_count": total_count,
+            "errors": errors
+        }
+        super().__init__(message, details)
+
+
+class CompressionException(DataStorageException):
+    """데이터 압축 예외"""
+    
+    def __init__(self, data_type: str, original_error: Exception):
+        message = f"Data compression failed for {data_type}"
+        details = {
+            "data_type": data_type,
+            "error_type": type(original_error).__name__,
+            "error_message": str(original_error)
+        }
+        super().__init__(message, details) 

yeonjae_universal_data_storage-1.0.1/src/universal_data_storage/models.py

@@ -0,0 +1,248 @@
+"""Data storage models for webhook events - MVP Version based on design spec."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional, Dict, Any, List
+from enum import Enum
+
+from sqlalchemy import (
+    Column, String, DateTime, LargeBinary, Integer, 
+    UniqueConstraint, ForeignKey, Text, Index, create_engine
+)
+from sqlalchemy.sql import func
+from sqlalchemy.orm import relationship, declarative_base
+from pydantic import BaseModel, ConfigDict
+
+# SQLAlchemy Base 독립적으로 생성
+Base = declarative_base()
+
+
+class StorageStatus(str, Enum):
+    """저장 상태"""
+    SUCCESS = "success"
+    FAILED = "failed"
+    DUPLICATE = "duplicate"
+    PENDING = "pending"
+
+
+# SQLAlchemy Models (Database Tables)
+class CommitRecord(Base):
+    """커밋 정보 테이블 - MVP 버전"""
+    
+    __tablename__ = "commits"
+    
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    hash = Column(String(40), unique=True, nullable=False, index=True)
+    message = Column(Text, nullable=False)
+    author = Column(String(255), nullable=False, index=True)
+    author_email = Column(String(255), nullable=True)
+    timestamp = Column(DateTime, nullable=False, index=True)
+    repository = Column(String(255), nullable=False, index=True)
+    branch = Column(String(255), nullable=False)
+    pusher = Column(String(255), nullable=True)
+    commit_count = Column(Integer, nullable=False, default=1)
+    created_at = Column(DateTime, nullable=False, default=func.now())
+    
+    # 관계 설정
+    diffs = relationship("DiffRecord", back_populates="commit", cascade="all, delete-orphan")
+    
+    def __repr__(self) -> str:
+        return f"<CommitRecord(id={self.id}, hash={self.hash[:8]}, repo={self.repository})>"
+
+
+class DiffRecord(Base):
+    """Diff 정보 테이블 - MVP 버전"""
+    
+    __tablename__ = "commit_diffs"
+    
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    commit_id = Column(Integer, ForeignKey("commits.id", ondelete="CASCADE"), nullable=False)
+    file_path = Column(Text, nullable=False)
+    additions = Column(Integer, nullable=False, default=0)
+    deletions = Column(Integer, nullable=False, default=0)
+    changes = Column(Text, nullable=True)  # diff 내용 (압축된 형태)
+    diff_patch = Column(LargeBinary, nullable=True)  # 압축된 diff 데이터
+    diff_url = Column(String, nullable=True)  # S3 URL (큰 파일의 경우)
+    created_at = Column(DateTime, nullable=False, default=func.now())
+    
+    # 관계 설정
+    commit = relationship("CommitRecord", back_populates="diffs")
+    
+    # 인덱스 추가
+    __table_args__ = (
+        Index('idx_diffs_commit_id', 'commit_id'),
+        Index('idx_diffs_file_path', 'file_path'),
+    )
+    
+    def __repr__(self) -> str:
+        return f"<DiffRecord(id={self.id}, commit_id={self.commit_id}, file={self.file_path})>"
+
+
+# Legacy Event Model (기존 호환성 유지)
+class Event(Base):
+    """Database model for GitHub webhook events - Legacy support."""
+    
+    __tablename__ = "events"
+    
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    platform = Column(String, nullable=False, default="github")
+    repository = Column(String, nullable=False, index=True)
+    commit_sha = Column(String, nullable=False, index=True)
+    author_name = Column(String, nullable=True)
+    author_email = Column(String, nullable=True)
+    timestamp_utc = Column(DateTime, nullable=True)
+    ref = Column(String, nullable=True)
+    pusher = Column(String, nullable=False, index=True)
+    commit_count = Column(Integer, nullable=False, default=1)
+    diff_patch = Column(LargeBinary, nullable=True)  # Compressed diff or None if stored in S3
+    diff_url = Column(String, nullable=True)  # S3 URL if diff is too large
+    added_lines = Column(Integer, nullable=True)
+    deleted_lines = Column(Integer, nullable=True)
+    files_changed = Column(Integer, nullable=True)
+    payload = Column(String, nullable=False)  # JSON string
+    created_at = Column(DateTime, nullable=False, default=func.now())
+    
+    # Prevent duplicate events
+    __table_args__ = (
+        UniqueConstraint('repository', 'commit_sha', name='uq_repo_commit'),
+    )
+    
+    def __repr__(self) -> str:
+        return f"<Event(id={self.id}, repo={self.repository}, sha={self.commit_sha[:8]})>"
+
+
+# Pydantic Models (API Request/Response)
+class CommitData(BaseModel):
+    """커밋 데이터 입력 모델"""
+    
+    commit_hash: str
+    message: str
+    author: str
+    author_email: Optional[str] = None
+    timestamp: datetime
+    repository: str
+    branch: str
+    pusher: Optional[str] = None
+    commit_count: int = 1
+    
+    model_config = ConfigDict(
+        from_attributes=True,
+        json_encoders={datetime: lambda v: v.isoformat()}
+    )
+
+
+class DiffData(BaseModel):
+    """Diff 데이터 입력 모델"""
+    
+    file_path: str
+    additions: int = 0
+    deletions: int = 0
+    changes: Optional[str] = None
+    diff_content: Optional[bytes] = None  # 원본 diff 내용
+    
+    model_config = ConfigDict(
+        from_attributes=True,
+        arbitrary_types_allowed=True
+    )
+
+
+class StorageResult(BaseModel):
+    """저장 결과 응답 모델"""
+    
+    success: bool
+    status: StorageStatus
+    commit_id: Optional[int] = None
+    message: str
+    timestamp: datetime
+    metadata: Dict[str, Any] = {}
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+class CommitSummary(BaseModel):
+    """커밋 요약 정보"""
+    
+    id: int
+    hash: str
+    message: str
+    author: str
+    timestamp: datetime
+    repository: str
+    branch: str
+    diff_count: int
+    total_additions: int
+    total_deletions: int
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+class DiffSummary(BaseModel):
+    """Diff 요약 정보"""
+    
+    id: int
+    file_path: str
+    additions: int
+    deletions: int
+    has_content: bool
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+# Legacy Models (기존 호환성)
+class EventCreate(BaseModel):
+    """Pydantic model for creating new events."""
+    
+    repository: str
+    commit_sha: str
+    event_type: str = "push"
+    payload: str
+    diff_data: Optional[bytes] = None
+    diff_s3_url: Optional[str] = None
+    
+    model_config = ConfigDict(
+        from_attributes=True,
+        arbitrary_types_allowed=True
+    )
+
+
+class EventResponse(BaseModel):
+    """Pydantic model for event API responses."""
+    
+    id: int
+    repository: str
+    commit_sha: str
+    event_type: str
+    payload: Dict[str, Any]
+    diff_s3_url: Optional[str] = None
+    created_at: datetime
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+# Aggregated Models
+class CommitWithDiffs(BaseModel):
+    """커밋과 관련 Diff 정보를 포함한 집계 모델"""
+    
+    commit: CommitSummary
+    diffs: List[DiffSummary]
+    
+    model_config = ConfigDict(from_attributes=True)
+
+
+class BatchStorageResult(BaseModel):
+    """배치 저장 결과"""
+    
+    total_commits: int
+    successful_commits: int
+    failed_commits: int
+    results: List[StorageResult]
+    duration_seconds: float
+    
+    @property
+    def success_rate(self) -> float:
+        if self.total_commits == 0:
+            return 0.0
+        return self.successful_commits / self.total_commits
+    
+    model_config = ConfigDict(from_attributes=True)