kodit 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

kodit/log.py

@@ -1,20 +1,33 @@
 """Logging configuration for kodit."""
 
 import logging
+import platform
+import re
+import shutil
+import subprocess
 import sys
 import uuid
 from enum import Enum
 from functools import lru_cache
+from pathlib import Path
 from typing import Any
 
+import rudderstack.analytics as rudder_analytics
 import structlog
-from posthog import Posthog
 from structlog.types import EventDict
 
+from kodit import _version
 from kodit.config import AppContext
 
+_MAC_RE = re.compile(r"(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}")
+
 log = structlog.get_logger(__name__)
 
+rudder_analytics.write_key = "2wm1RmV2GnO92NGSs8yYtmSI0mi"
+rudder_analytics.dataPlaneUrl = (
+    "https://danbmedefzavzlslreyxjgcjwlf.dataplane.rudderstack.com"
+)
+
 
 def drop_color_message_key(_, __, event_dict: EventDict) -> EventDict:  # noqa: ANN001
     """Drop the `color_message` key from the event dict."""
@@ -131,35 +144,124 @@ def configure_logging(app_context: AppContext) -> None:
     sys.excepthook = handle_exception
 
 
-posthog = Posthog(
-    project_api_key="phc_JsX0yx8NLPcIxamfp4Zc7xyFykXjwmekKUQz060cSt3",
-    host="https://eu.i.posthog.com",
-)
+def configure_telemetry(app_context: AppContext) -> None:
+    """Configure telemetry for the application."""
+    if app_context.disable_telemetry:
+        structlog.stdlib.get_logger(__name__).info("Telemetry has been disabled")
+        rudder_analytics.send = False
 
+    rudder_analytics.identify(
+        anonymous_id=get_stable_mac_str(),
+        traits={},
+    )
 
-@lru_cache(maxsize=1)
-def get_mac_address() -> str:
-    """Get the MAC address of the primary network interface.
 
-    Returns:
-        str: The MAC address or a fallback UUID if not available
+def log_event(event: str, properties: dict[str, Any] | None = None) -> None:
+    """Log an event to Rudderstack."""
+    p = properties or {}
+    # Set default posthog properties
+    p["$app_name"] = "kodit"
+    p["$app_version"] = _version.version
+    p["$os"] = sys.platform
+    p["$os_version"] = sys.version
+    rudder_analytics.track(
+        anonymous_id=get_stable_mac_str(),
+        event=event,
+        properties=properties or {},
+    )
+
 
+# ----------------------------------------------------------------------
+# Helper functions
+# ----------------------------------------------------------------------
+def _mac_int(mac: str) -> int:
+    return int(mac.replace(":", "").replace("-", ""), 16)
+
+
+def _is_globally_administered(mac_int: int) -> bool:
+    first_octet = (mac_int >> 40) & 0xFF
+    return not (first_octet & 0b11)  # both bits must be 0
+
+
+def _from_sysfs() -> list[int]:
+    base = Path("/sys/class/net")
+    if not base.is_dir():
+        return []
+    macs: list[int] = []
+    for iface in base.iterdir():
+        try:
+            with (base / iface / "address").open() as f:
+                content = f.read().strip()
+            if _MAC_RE.fullmatch(content):
+                macs.append(_mac_int(content))
+        except (FileNotFoundError, PermissionError):
+            pass
+    return macs
+
+
+def _from_command(cmd: str) -> list[int]:
+    try:
+        out = subprocess.check_output(  # noqa: S602
+            cmd,
+            shell=True,
+            text=True,
+            stderr=subprocess.DEVNULL,
+            encoding="utf-8",
+        )
+    except Exception:  # noqa: BLE001
+        return []
+    return [_mac_int(m.group()) for m in _MAC_RE.finditer(out)]
+
+
+@lru_cache(maxsize=1)
+def get_stable_mac_int() -> int | None:
+    """Return a *hardware* MAC as an int, or None if none can be found.
+
+    Search order:
+        1. /sys/class/net (Linux)
+        2. `ip link show` (Linux), `ifconfig -a` (Linux+macOS)
+        3. `getmac` and `wmic nic` (Windows)
+    The first globally-administered, non-multicast address wins.
     """
-    # Get the MAC address of the primary network interface
-    mac = uuid.getnode()
-    return f"{mac:012x}" if mac != uuid.getnode() else str(uuid.uuid4())
+    system = platform.system()
+    candidates: list[int] = []
+
+    if system == "Linux":
+        candidates += _from_sysfs()
+        if not candidates and shutil.which("ip"):
+            candidates += _from_command("ip link show")
+        if not candidates:  # fall back to ifconfig
+            candidates += _from_command("ifconfig -a")
+
+    elif system == "Darwin":  # macOS
+        candidates += _from_command("ifconfig -a")
+
+    elif system == "Windows":
+        # getmac is present on every supported Windows version
+        candidates += _from_command("getmac /v /fo list")
+        # wmic still exists through at least Win 11
+        candidates += _from_command(
+            'wmic nic where "MACAddress is not null" get MACAddress /format:list'
+        )
 
+    # Prefer globally administered, non-multicast addresses
+    for mac in candidates:
+        if _is_globally_administered(mac):
+            return mac
 
-def configure_telemetry(app_context: AppContext) -> None:
-    """Configure telemetry for the application."""
-    if app_context.disable_telemetry:
-        structlog.stdlib.get_logger(__name__).info("Telemetry has been disabled")
-        posthog.disabled = True
+    # If all we saw were locally-administered MACs, just return the first one
+    if candidates:
+        return candidates[0]
 
+    return None
 
-def log_event(event: str, properties: dict[str, Any] | None = None) -> None:
-    """Log an event to PostHog."""
-    log.debug(
-        "Logging event", id=get_mac_address(), ph_event=event, ph_properties=properties
-    )
-    posthog.capture(get_mac_address(), event, properties or {})
+
+def get_stable_mac_str() -> str:
+    """Return a *stable* 12-digit hex string (lower-case, no separators).
+
+    Falls back to uuid.getnode() if necessary, so it never raises.
+    """
+    mac_int = get_stable_mac_int()
+    if mac_int is None:
+        mac_int = uuid.getnode()  # may still be random in VMs
+    return f"{mac_int:012x}"

kodit/migrations/versions/9e53ea8bb3b0_add_authors.py

@@ -0,0 +1,103 @@
+# ruff: noqa
+"""add authors
+
+Revision ID: 9e53ea8bb3b0
+Revises: c3f5137d30f5
+Create Date: 2025-06-14 10:50:36.058114
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "9e53ea8bb3b0"
+down_revision: Union[str, None] = "c3f5137d30f5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # Define the enum type separately so we can explicitly create it when needed
+    source_type = sa.Enum("UNKNOWN", "FOLDER", "GIT", name="sourcetype")
+
+    # Explicitly create the enum type for PostgreSQL (no-op on SQLite)
+    source_type.create(op.get_bind(), checkfirst=True)
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "authors",
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("email", sa.String(length=255), nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", "email", name="uix_author"),
+    )
+    op.create_index(op.f("ix_authors_email"), "authors", ["email"], unique=False)
+    op.create_index(op.f("ix_authors_name"), "authors", ["name"], unique=False)
+    op.create_table(
+        "author_file_mappings",
+        sa.Column("author_id", sa.Integer(), nullable=False),
+        sa.Column("file_id", sa.Integer(), nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["author_id"],
+            ["authors.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["file_id"],
+            ["files.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("author_id", "file_id", name="uix_author_file_mapping"),
+    )
+    op.create_index(
+        op.f("ix_author_file_mappings_author_id"),
+        "author_file_mappings",
+        ["author_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_author_file_mappings_file_id"),
+        "author_file_mappings",
+        ["file_id"],
+        unique=False,
+    )
+    op.add_column(
+        "files", sa.Column("extension", sa.String(length=255), nullable=False)
+    )
+    op.create_index(op.f("ix_files_extension"), "files", ["extension"], unique=False)
+    op.add_column("sources", sa.Column("type", source_type, nullable=False))
+    op.create_index(op.f("ix_sources_type"), "sources", ["type"], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_sources_type"), table_name="sources")
+    op.drop_column("sources", "type")
+    op.drop_index(op.f("ix_files_extension"), table_name="files")
+    op.drop_column("files", "extension")
+    op.drop_index(
+        op.f("ix_author_file_mappings_file_id"), table_name="author_file_mappings"
+    )
+    op.drop_index(
+        op.f("ix_author_file_mappings_author_id"), table_name="author_file_mappings"
+    )
+    op.drop_table("author_file_mappings")
+    op.drop_index(op.f("ix_authors_name"), table_name="authors")
+    op.drop_index(op.f("ix_authors_email"), table_name="authors")
+    op.drop_table("authors")
+
+    # Explicitly drop the enum type (PostgreSQL)
+    source_type = sa.Enum("UNKNOWN", "FOLDER", "GIT", name="sourcetype")
+    source_type.drop(op.get_bind(), checkfirst=True)
+    # ### end Alembic commands ###

kodit/source/source_factories.py

@@ -0,0 +1,356 @@
+"""Source factories for creating different types of sources.
+
+This module provides factory classes for creating sources, improving cohesion by
+separating the concerns of different source types.
+"""
+
+import mimetypes
+import shutil
+import tempfile
+from abc import ABC, abstractmethod
+from datetime import UTC, datetime
+from hashlib import sha256
+from pathlib import Path
+from typing import Protocol
+
+import aiofiles
+import git
+import structlog
+from tqdm import tqdm
+
+from kodit.source.ignore import IgnorePatterns
+from kodit.source.source_models import (
+    Author,
+    AuthorFileMapping,
+    File,
+    Source,
+    SourceType,
+)
+from kodit.source.source_repository import SourceRepository
+
+
+class WorkingCopyProvider(Protocol):
+    """Protocol for providing working copies of sources."""
+
+    async def prepare(self, uri: str) -> Path:
+        """Prepare a working copy and return its path."""
+        ...
+
+
+class FileMetadataExtractor(Protocol):
+    """Protocol for extracting file metadata."""
+
+    async def extract(self, path: Path, source: Source) -> File:
+        """Extract metadata from a file."""
+        ...
+
+
+class AuthorExtractor(Protocol):
+    """Protocol for extracting author information."""
+
+    async def extract(self, path: Path, source: Source) -> list[Author]:
+        """Extract authors for a file."""
+        ...
+
+
+class SourceFactory(ABC):
+    """Abstract base class for source factories."""
+
+    def __init__(
+        self,
+        working_copy: WorkingCopyProvider,
+        metadata_extractor: FileMetadataExtractor,
+        author_extractor: AuthorExtractor,
+        repository: SourceRepository,
+    ) -> None:
+        """Initialize the source factory."""
+        self.working_copy = working_copy
+        self.metadata_extractor = metadata_extractor
+        self.author_extractor = author_extractor
+        self.repository = repository
+        self.log = structlog.get_logger(__name__)
+
+    @abstractmethod
+    async def create(self, uri: str) -> Source:
+        """Create a source from a URI."""
+        ...
+
+    async def _process_files(self, source: Source, files: list[Path]) -> None:
+        """Process files for a source."""
+        for path in tqdm(files, total=len(files), leave=False):
+            if not path.is_file():
+                continue
+
+            # Extract file metadata
+            file_record = await self.metadata_extractor.extract(path, source)
+            await self.repository.create_file(file_record)
+
+            # Extract authors
+            authors = await self.author_extractor.extract(path, source)
+            for author in authors:
+                await self.repository.upsert_author_file_mapping(
+                    AuthorFileMapping(
+                        author_id=author.id,
+                        file_id=file_record.id,
+                    )
+                )
+
+
+class GitSourceFactory(SourceFactory):
+    """Factory for creating Git sources."""
+
+    async def create(self, uri: str) -> Source:
+        """Create a git source from a URI."""
+        # Normalize the URI
+        self.log.debug("Normalising git uri", uri=uri)
+        with tempfile.TemporaryDirectory() as temp_dir:
+            git.Repo.clone_from(uri, temp_dir)
+            remote = git.Repo(temp_dir).remote()
+            uri = remote.url
+
+        # Check if source already exists
+        self.log.debug("Checking if source already exists", uri=uri)
+        source = await self.repository.get_source_by_uri(uri)
+
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+            return source
+
+        # Prepare working copy
+        clone_path = await self.working_copy.prepare(uri)
+
+        # Create source record
+        self.log.debug("Creating source", uri=uri, clone_path=str(clone_path))
+        source = await self.repository.create_source(
+            Source(
+                uri=uri,
+                cloned_path=str(clone_path),
+                source_type=SourceType.GIT,
+            )
+        )
+
+        # Get files to process using ignore patterns
+        ignore_patterns = IgnorePatterns(clone_path)
+        files = [
+            f
+            for f in clone_path.rglob("*")
+            if f.is_file() and not ignore_patterns.should_ignore(f)
+        ]
+
+        # Process files
+        self.log.info("Inspecting files", source_id=source.id, num_files=len(files))
+        await self._process_files(source, files)
+
+        return source
+
+
+class FolderSourceFactory(SourceFactory):
+    """Factory for creating folder sources."""
+
+    async def create(self, uri: str) -> Source:
+        """Create a folder source from a path."""
+        directory = Path(uri).expanduser().resolve()
+
+        # Check if source already exists
+        source = await self.repository.get_source_by_uri(directory.as_uri())
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+            return source
+
+        # Validate directory exists
+        if not directory.exists():
+            msg = f"Folder does not exist: {directory}"
+            raise ValueError(msg)
+
+        # Prepare working copy
+        clone_path = await self.working_copy.prepare(directory.as_uri())
+
+        # Create source record
+        source = await self.repository.create_source(
+            Source(
+                uri=directory.as_uri(),
+                cloned_path=str(clone_path),
+                source_type=SourceType.FOLDER,
+            )
+        )
+
+        # Get all files to process
+        files = [f for f in clone_path.rglob("*") if f.is_file()]
+
+        # Process files
+        await self._process_files(source, files)
+
+        return source
+
+
+class GitWorkingCopyProvider:
+    """Working copy provider for Git repositories."""
+
+    def __init__(self, clone_dir: Path) -> None:
+        """Initialize the provider."""
+        self.clone_dir = clone_dir
+        self.log = structlog.get_logger(__name__)
+
+    async def prepare(self, uri: str) -> Path:
+        """Prepare a Git working copy."""
+        # Create a unique directory name for the clone
+        clone_path = self.clone_dir / uri.replace("/", "_").replace(":", "_")
+        clone_path.mkdir(parents=True, exist_ok=True)
+
+        try:
+            self.log.info("Cloning repository", uri=uri, clone_path=str(clone_path))
+            git.Repo.clone_from(uri, clone_path)
+        except git.GitCommandError as e:
+            if "already exists and is not an empty directory" not in str(e):
+                msg = f"Failed to clone repository: {e}"
+                raise ValueError(msg) from e
+            self.log.info("Repository already exists, reusing...", uri=uri)
+
+        return clone_path
+
+
+class FolderWorkingCopyProvider:
+    """Working copy provider for local folders."""
+
+    def __init__(self, clone_dir: Path) -> None:
+        """Initialize the provider."""
+        self.clone_dir = clone_dir
+
+    async def prepare(self, uri: str) -> Path:
+        """Prepare a folder working copy."""
+        # Handle file:// URIs
+        if uri.startswith("file://"):
+            from urllib.parse import urlparse
+
+            parsed = urlparse(uri)
+            directory = Path(parsed.path).expanduser().resolve()
+        else:
+            directory = Path(uri).expanduser().resolve()
+
+        # Clone into a local directory
+        clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
+        clone_path.mkdir(parents=True, exist_ok=True)
+
+        # Copy all files recursively, preserving directory structure, ignoring
+        # hidden files
+        shutil.copytree(
+            directory,
+            clone_path,
+            ignore=shutil.ignore_patterns(".*"),
+            dirs_exist_ok=True,
+        )
+
+        return clone_path
+
+
+class BaseFileMetadataExtractor:
+    """Base class for file metadata extraction with common functionality."""
+
+    async def extract(self, path: Path, source: Source) -> File:
+        """Extract metadata from a file."""
+        # Get timestamps - to be implemented by subclasses
+        created_at, updated_at = await self._get_timestamps(path, source)
+
+        # Read file content and calculate metadata
+        async with aiofiles.open(path, "rb") as f:
+            content = await f.read()
+            mime_type = mimetypes.guess_type(path)
+            sha = sha256(content).hexdigest()
+
+            return File(
+                created_at=created_at,
+                updated_at=updated_at,
+                source_id=source.id,
+                cloned_path=str(path),
+                mime_type=mime_type[0]
+                if mime_type and mime_type[0]
+                else "application/octet-stream",
+                uri=path.as_uri(),
+                sha256=sha,
+                size_bytes=len(content),
+            )
+
+    async def _get_timestamps(
+        self, path: Path, source: Source
+    ) -> tuple[datetime, datetime]:
+        """Get creation and modification timestamps. To be implemented by subclasses."""
+        raise NotImplementedError
+
+
+class GitFileMetadataExtractor(BaseFileMetadataExtractor):
+    """Git-specific implementation for extracting file metadata."""
+
+    async def _get_timestamps(
+        self, path: Path, source: Source
+    ) -> tuple[datetime, datetime]:
+        """Get timestamps from Git history."""
+        git_repo = git.Repo(source.cloned_path)
+        commits = list(git_repo.iter_commits(paths=str(path), all=True))
+
+        if commits:
+            last_modified_at = commits[0].committed_datetime
+            first_modified_at = commits[-1].committed_datetime
+            return first_modified_at, last_modified_at
+        # Fallback to current time if no commits found
+        now = datetime.now(UTC)
+        return now, now
+
+
+class FolderFileMetadataExtractor(BaseFileMetadataExtractor):
+    """Folder-specific implementation for extracting file metadata."""
+
+    async def _get_timestamps(
+        self,
+        path: Path,
+        source: Source,  # noqa: ARG002
+    ) -> tuple[datetime, datetime]:
+        """Get timestamps from file system."""
+        stat = path.stat()
+        file_created_at = datetime.fromtimestamp(stat.st_ctime, UTC)
+        file_modified_at = datetime.fromtimestamp(stat.st_mtime, UTC)
+        return file_created_at, file_modified_at
+
+
+class GitAuthorExtractor:
+    """Author extractor for Git repositories."""
+
+    def __init__(self, repository: SourceRepository) -> None:
+        """Initialize the extractor."""
+        self.repository = repository
+
+    async def extract(self, path: Path, source: Source) -> list[Author]:
+        """Extract authors from a Git file."""
+        authors: list[Author] = []
+        git_repo = git.Repo(source.cloned_path)
+
+        try:
+            # Get the file's blame
+            blames = git_repo.blame("HEAD", str(path))
+
+            # Extract the blame's authors
+            actors = [
+                commit.author
+                for blame in blames or []
+                for commit in blame
+                if isinstance(commit, git.Commit)
+            ]
+
+            # Get or create the authors in the database
+            for actor in actors:
+                if actor.email:
+                    author = Author.from_actor(actor)
+                    author = await self.repository.upsert_author(author)
+                    authors.append(author)
+        except git.GitCommandError:
+            # Handle cases where file might not be tracked
+            pass
+
+        return authors
+
+
+class NoOpAuthorExtractor:
+    """No-op author extractor for sources that don't have author information."""
+
+    async def extract(self, path: Path, source: Source) -> list[Author]:  # noqa: ARG002
+        """Return empty list of authors."""
+        return []

kodit/source/source_models.py

@@ -8,7 +8,8 @@ folders) and their relationships.
 import datetime
 from enum import Enum as EnumType
 
-from sqlalchemy import Enum, ForeignKey, Integer, String
+from git import Actor
+from sqlalchemy import Enum, ForeignKey, Integer, String, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column
 
 from kodit.database import Base, CommonMixin
@@ -60,8 +61,15 @@ class Author(Base, CommonMixin):
 
     __tablename__ = "authors"
 
-    name: Mapped[str] = mapped_column(String(255), index=True, unique=True)
-    email: Mapped[str] = mapped_column(String(255), index=True, unique=True)
+    __table_args__ = (UniqueConstraint("name", "email", name="uix_author"),)
+
+    name: Mapped[str] = mapped_column(String(255), index=True)
+    email: Mapped[str] = mapped_column(String(255), index=True)
+
+    @staticmethod
+    def from_actor(actor: Actor) -> "Author":
+        """Create an Author from an Actor."""
+        return Author(name=actor.name, email=actor.email)
 
 
 class AuthorFileMapping(Base, CommonMixin):
@@ -69,8 +77,12 @@ class AuthorFileMapping(Base, CommonMixin):
 
     __tablename__ = "author_file_mappings"
 
-    author_id: Mapped[int] = mapped_column(ForeignKey("authors.id"))
-    file_id: Mapped[int] = mapped_column(ForeignKey("files.id"))
+    __table_args__ = (
+        UniqueConstraint("author_id", "file_id", name="uix_author_file_mapping"),
+    )
+
+    author_id: Mapped[int] = mapped_column(ForeignKey("authors.id"), index=True)
+    file_id: Mapped[int] = mapped_column(ForeignKey("files.id"), index=True)
 
 
 class File(Base, CommonMixin):