orgdatacore 2.0.0__py3-none-any.whl

orgdatacore/_internal/testing/filesource.py

@@ -0,0 +1,147 @@
+"""
+FileDataSource - Internal testing utility.
+
+INTERNAL USE ONLY: This is kept for test infrastructure.
+Production code should use GCSDataSource or implement a custom DataSource.
+
+This module is NOT part of the public API and may change without notice.
+"""
+
+import os
+import threading
+import time
+from collections.abc import Callable
+from io import BytesIO
+from typing import BinaryIO
+
+from orgdatacore._exceptions import FileSourceError
+from orgdatacore._log import get_logger
+
+
+class FileDataSource:
+    """
+    FileDataSource loads organizational data from local files.
+
+    INTERNAL USE ONLY: This should only be used in test code.
+    Production code should use GCSDataSource or implement a custom DataSource.
+
+    For production deployments, file-based data sources are not recommended
+    for security reasons. Use GCS or implement your own DataSource (e.g., S3).
+    """
+
+    def __init__(
+        self, file_paths: list[str] | str, poll_interval: float = 60.0
+    ) -> None:
+        """
+        Create a new file-based data source for testing.
+
+        INTERNAL USE ONLY.
+
+        Args:
+            file_paths: Path(s) to the data file(s). If multiple provided,
+                       the last one is used.
+            poll_interval: Interval in seconds for polling file changes.
+        """
+        if isinstance(file_paths, str):
+            file_paths = [file_paths]
+        self.file_paths = file_paths
+        self.poll_interval = poll_interval
+        self._stop_event = threading.Event()
+
+    def load(self) -> BinaryIO:
+        """Load and return a reader for the organizational data file.
+
+        Returns:
+            File-like object containing the data.
+
+        Raises:
+            FileSourceError: If loading fails.
+        """
+        logger = get_logger()
+
+        if not self.file_paths:
+            raise FileSourceError("no file paths provided")
+
+        # Load the primary data file (use the last path if multiple provided)
+        file_path = self.file_paths[-1]
+        logger.debug("Loading from file", extra={"path": file_path})
+
+        try:
+            with open(file_path, "rb") as f:
+                content = f.read()
+            return BytesIO(content)
+        except FileNotFoundError:
+            raise FileSourceError(f"file not found: {file_path}")
+        except PermissionError:
+            raise FileSourceError(f"permission denied: {file_path}")
+        except OSError as e:
+            raise FileSourceError(f"failed to read file {file_path}: {e}")
+
+    def watch(self, callback: Callable[[], Exception | None]) -> Exception | None:
+        """
+        Monitor for file changes and call callback when data is updated.
+
+        This starts a background thread that polls for file changes.
+
+        Args:
+            callback: Function to call when files change.
+
+        Returns:
+            Exception if setup fails, None otherwise.
+        """
+        logger = get_logger()
+
+        if not self.file_paths:
+            return FileSourceError("no file paths to watch")
+
+        # Get initial modification times
+        mod_times: dict[str, float] = {}
+        for path in self.file_paths:
+            try:
+                mod_times[path] = os.path.getmtime(path)
+            except OSError:
+                pass
+
+        logger.debug(
+            "Starting file watcher",
+            extra={"paths": self.file_paths, "poll_interval": self.poll_interval},
+        )
+
+        def watcher() -> None:
+            while not self._stop_event.is_set():
+                time.sleep(self.poll_interval)
+                if self._stop_event.is_set():
+                    break
+
+                # Check if any files have changed
+                changed = False
+                for path in self.file_paths:
+                    try:
+                        current_mtime = os.path.getmtime(path)
+                        last_mtime = mod_times.get(path)
+                        if last_mtime is None or current_mtime > last_mtime:
+                            mod_times[path] = current_mtime
+                            changed = True
+                    except OSError:
+                        pass
+
+                if changed:
+                    logger.debug("File change detected, triggering reload")
+                    try:
+                        callback()
+                    except Exception as e:
+                        logger.error("Reload callback failed", extra={"error": str(e)})
+
+        thread = threading.Thread(target=watcher, daemon=True, name="file-watcher")
+        thread.start()
+        return None
+
+    def stop_watching(self) -> None:
+        """Stop the file watcher."""
+        self._stop_event.set()
+
+    def __str__(self) -> str:
+        """Return a description of this data source."""
+        if len(self.file_paths) == 1:
+            return f"file:{self.file_paths[0]}"
+        return f"files:{','.join(self.file_paths)}"

orgdatacore/_internal/testing/helpers.py

@@ -0,0 +1,192 @@
+"""Test helpers for orgdatacore."""
+
+import json
+from collections.abc import Callable
+from io import BytesIO
+from typing import BinaryIO
+
+from orgdatacore._serialization import data_to_dict
+from orgdatacore._types import (
+    Component,
+    Data,
+    Employee,
+    GitHubIDMappings,
+    Group,
+    GroupType,
+    Indexes,
+    JiraOwnerInfo,
+    Lookups,
+    MembershipIndex,
+    MembershipInfo,
+    Metadata,
+    Org,
+    ParentInfo,
+    Pillar,
+    SlackIDMappings,
+    Team,
+    TeamGroup,
+)
+
+
+class FakeDataSource:
+    """FakeDataSource implements DataSource for testing with controllable data."""
+
+    def __init__(
+        self,
+        data: str = "",
+        load_error: Exception | None = None,
+        watch_error: Exception | None = None,
+        description: str = "fake-data-source",
+    ) -> None:
+        """
+        Create a fake data source for testing.
+
+        Args:
+            data: JSON string data to return from load().
+            load_error: Error to raise from load().
+            watch_error: Error to return from watch().
+            description: Description string for __str__.
+        """
+        self.data = data
+        self.load_error = load_error
+        self.watch_error = watch_error
+        self.description = description
+        self.watch_called = False
+
+    def load(self) -> BinaryIO:
+        """Return the test data."""
+        if self.load_error:
+            raise self.load_error
+        return BytesIO(self.data.encode("utf-8"))
+
+    def watch(self, callback: Callable[[], Exception | None]) -> Exception | None:
+        """Track that watch was called but don't actually watch."""
+        self.watch_called = True
+        if self.watch_error:
+            return self.watch_error
+        return None
+
+    def __str__(self) -> str:
+        """Return the description."""
+        return self.description
+
+
+def create_test_data() -> Data:
+    """Create comprehensive test data for testing."""
+    return Data(
+        metadata=Metadata(
+            generated_at="2024-01-01T00:00:00Z",
+            data_version="test-v1.0",
+            total_employees=2,
+            total_orgs=1,
+        ),
+        lookups=Lookups(
+            employees={
+                "testuser1": Employee(
+                    uid="testuser1",
+                    full_name="Test User One",
+                    email="testuser1@example.com",
+                    job_title="Test Engineer",
+                    slack_uid="U111111",
+                    github_id="ghuser1",
+                    manager_uid="testuser2",
+                ),
+                "testuser2": Employee(
+                    uid="testuser2",
+                    full_name="Test User Two",
+                    email="testuser2@example.com",
+                    job_title="Test Manager",
+                    slack_uid="U222222",
+                    github_id="ghuser2",
+                ),
+            },
+            teams={
+                "test-squad": Team(
+                    name="test-squad",
+                    parent=ParentInfo(name="test-team-group", type="team_group"),
+                    group=Group(
+                        type=GroupType(name="team"),
+                        resolved_people_uid_list=["testuser1", "testuser2"],
+                    ),
+                ),
+            },
+            orgs={
+                "test-division": Org(
+                    name="test-division",
+                    group=Group(
+                        type=GroupType(name="organization"),
+                        resolved_people_uid_list=["testuser1", "testuser2"],
+                    ),
+                ),
+            },
+            pillars={
+                "test-pillar": Pillar(
+                    name="test-pillar",
+                    parent=ParentInfo(name="test-division", type="org"),
+                    group=Group(
+                        type=GroupType(name="pillar"),
+                        resolved_people_uid_list=["testuser1", "testuser2"],
+                    ),
+                ),
+            },
+            team_groups={
+                "test-team-group": TeamGroup(
+                    name="test-team-group",
+                    parent=ParentInfo(name="test-pillar", type="pillar"),
+                    group=Group(
+                        type=GroupType(name="team_group"),
+                        resolved_people_uid_list=["testuser1", "testuser2"],
+                    ),
+                ),
+            },
+            components={
+                "test-component": Component(
+                    name="test-component",
+                    description="Test component",
+                ),
+            },
+        ),
+        indexes=Indexes(
+            membership=MembershipIndex(
+                membership_index={
+                    "testuser1": (
+                        MembershipInfo(name="test-squad", type="team"),
+                        MembershipInfo(name="test-division", type="org"),
+                    ),
+                    "testuser2": (
+                        MembershipInfo(name="test-squad", type="team"),
+                        MembershipInfo(name="test-division", type="org"),
+                    ),
+                },
+            ),
+            slack_id_mappings=SlackIDMappings(
+                slack_uid_to_uid={
+                    "U111111": "testuser1",
+                    "U222222": "testuser2",
+                },
+            ),
+            github_id_mappings=GitHubIDMappings(
+                github_id_to_uid={
+                    "ghuser1": "testuser1",
+                    "ghuser2": "testuser2",
+                },
+            ),
+            jira={
+                "TEST": {
+                    "Core": [JiraOwnerInfo(name="test-squad", type="team")],
+                    "_project_level": [
+                        JiraOwnerInfo(name="test-squad", type="team"),
+                    ],
+                },
+                "PLAT": {
+                    "API": [JiraOwnerInfo(name="test-squad", type="team")],
+                },
+            },
+        ),
+    )
+
+
+def create_test_data_json() -> str:
+    """Create test data as JSON string."""
+    data = create_test_data()
+    return json.dumps(data_to_dict(data))

orgdatacore/_log.py

@@ -0,0 +1,98 @@
+"""Logging configuration for orgdatacore.
+
+This module provides structured logging support matching the Go library's
+logging capabilities.
+
+Usage:
+    from orgdatacore import get_logger, set_logger
+
+    # Use the default logger
+    logger = get_logger()
+    logger.info("Data loaded", extra={"employee_count": 100})
+
+    # Set a custom logger
+    import logging
+    custom_logger = logging.getLogger("my_app.orgdata")
+    set_logger(custom_logger)
+
+    # Or configure the default logger
+    import logging
+    logging.getLogger("orgdatacore").setLevel(logging.DEBUG)
+"""
+
+import logging
+
+# Default logger for the library
+_logger: logging.Logger | None = None
+
+# Default logger name
+LOGGER_NAME = "orgdatacore"
+
+
+def get_logger() -> logging.Logger:
+    """Get the current logger for orgdatacore.
+
+    Returns the custom logger if set, otherwise returns the default
+    library logger.
+
+    Returns:
+        The configured logger instance.
+    """
+    global _logger
+    if _logger is not None:
+        return _logger
+    return logging.getLogger(LOGGER_NAME)
+
+
+def set_logger(logger: logging.Logger | None) -> None:
+    """Set a custom logger for orgdatacore.
+
+    This allows integrating orgdatacore logging with your application's
+    logging infrastructure.
+
+    Args:
+        logger: Custom logger to use, or None to reset to default.
+
+    Example:
+        import logging
+        from orgdatacore import set_logger
+
+        # Use your application's logger
+        app_logger = logging.getLogger("myapp.orgdata")
+        set_logger(app_logger)
+    """
+    global _logger
+    _logger = logger
+
+
+def configure_default_logging(
+    level: int = logging.INFO,
+    format_string: str | None = None,
+) -> None:
+    """Configure the default orgdatacore logger.
+
+    Convenience function to set up basic logging configuration.
+
+    Args:
+        level: Logging level (default: INFO)
+        format_string: Custom format string (default: structured format)
+
+    Example:
+        from orgdatacore import configure_default_logging
+        import logging
+
+        configure_default_logging(level=logging.DEBUG)
+    """
+    logger = logging.getLogger(LOGGER_NAME)
+    logger.setLevel(level)
+
+    if not logger.handlers:
+        handler = logging.StreamHandler()
+        handler.setLevel(level)
+
+        if format_string is None:
+            format_string = "[%(name)s] %(levelname)s: %(message)s"
+
+        formatter = logging.Formatter(format_string)
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)

orgdatacore/_redaction.py

@@ -0,0 +1,215 @@
+"""Data source wrappers for PII redaction."""
+
+import json
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, BinaryIO
+
+from ._serialization import data_to_json_bytes
+from ._service import parse_data
+from ._types import (
+    Data,
+    DataSource,
+    GitHubIDMappings,
+    Group,
+    PIIMode,
+    SlackIDMappings,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+class RedactingDataSource:
+    """
+    DataSource decorator that redacts PII from loaded data.
+
+    When pii_mode is REDACTED, this wrapper intercepts the JSON stream
+    from the inner data source and strips PII fields before the data
+    is parsed by Service, so PII is not propagated to downstream consumers.
+
+    PII fields redacted (emitted as empty values in the JSON, not omitted):
+    - full_name -> "[REDACTED]"
+    - email -> "[REDACTED]"
+    - slack_uid -> ""
+    - github_id -> ""
+    - possible_github_ids -> []
+
+    PII indexes cleared:
+    - slack_id_mappings.slack_uid_to_uid
+    - github_id_mappings.github_id_to_uid
+    """
+
+    def __init__(
+        self,
+        source: DataSource,
+        pii_mode: PIIMode = PIIMode.FULL,
+    ) -> None:
+        """
+        Initialize the redacting data source.
+
+        Args:
+            source: The underlying data source to wrap.
+            pii_mode: The PII handling mode. Defaults to FULL (no redaction).
+        """
+        if pii_mode not in (PIIMode.FULL, PIIMode.REDACTED):
+            raise ValueError(
+                f"RedactingDataSource only supports FULL and REDACTED modes, got {pii_mode!r}"
+            )
+        self._source = source
+        self._pii_mode = pii_mode
+
+    def load(self) -> BinaryIO:
+        """
+        Load data, optionally redacting PII fields.
+
+        Returns:
+            Binary stream containing the (possibly redacted) JSON data.
+        """
+        reader = self._source.load()
+
+        if self._pii_mode == PIIMode.FULL:
+            return reader  # Pass through unchanged
+
+        # Parse, redact, re-serialize
+        try:
+            raw = json.load(reader)
+        finally:
+            reader.close()
+        data = parse_data(raw)
+        redacted = _redact(data)
+        return BytesIO(data_to_json_bytes(redacted))
+
+    def watch(self, callback: "Callable[[], Exception | None]") -> Exception | None:
+        """
+        Monitor for changes and call the callback when data is updated.
+
+        Delegates to the underlying data source.
+
+        Args:
+            callback: Function to call when data changes.
+
+        Returns:
+            Exception if watch setup failed, None otherwise.
+        """
+        return self._source.watch(callback)
+
+    def __str__(self) -> str:
+        """Return a description of this data source."""
+        mode_suffix = " [PII redacted]" if self._pii_mode == PIIMode.REDACTED else ""
+        return f"{self._source}{mode_suffix}"
+
+
+class AsyncRedactingDataSource:
+    """Async DataSource decorator that redacts PII from loaded data.
+
+    Same redaction as RedactingDataSource but with async load().
+    Use with AsyncService and async inner sources.
+    """
+
+    def __init__(
+        self,
+        source: Any,
+        pii_mode: PIIMode = PIIMode.FULL,
+    ) -> None:
+        if pii_mode not in (PIIMode.FULL, PIIMode.REDACTED):
+            raise ValueError(
+                f"AsyncRedactingDataSource only supports FULL and REDACTED modes, got {pii_mode!r}"
+            )
+        self._source = source
+        self._pii_mode = pii_mode
+
+    async def load(self) -> BinaryIO:
+        """Load data asynchronously, optionally redacting PII fields."""
+        reader: BinaryIO = await self._source.load()
+
+        if self._pii_mode == PIIMode.FULL:
+            return reader
+
+        try:
+            raw = reader.read()
+        finally:
+            reader.close()
+        text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
+        raw_data = json.loads(text)
+        data = parse_data(raw_data)
+        redacted = _redact(data)
+        return BytesIO(data_to_json_bytes(redacted))
+
+    async def watch(self, callback: Any) -> Any:
+        """Delegate to the underlying data source."""
+        return await self._source.watch(callback)
+
+    def __str__(self) -> str:
+        mode_suffix = " [PII redacted]" if self._pii_mode == PIIMode.REDACTED else ""
+        return f"{self._source}{mode_suffix}"
+
+
+def _redact(data: Data) -> Data:
+    """Return a new Data with PII fields redacted."""
+    new_employees = {
+        uid: emp.model_copy(
+            update={
+                "full_name": "[REDACTED]",
+                "email": "[REDACTED]",
+                "slack_uid": "",
+                "github_id": "",
+                "possible_github_ids": [],
+            }
+        )
+        for uid, emp in data.lookups.employees.items()
+    }
+
+    new_teams = {
+        name: team.model_copy(update={"group": _redact_group(team.group)})
+        for name, team in data.lookups.teams.items()
+    }
+    new_orgs = {
+        name: org.model_copy(update={"group": _redact_group(org.group)})
+        for name, org in data.lookups.orgs.items()
+    }
+    new_pillars = {
+        name: pillar.model_copy(update={"group": _redact_group(pillar.group)})
+        for name, pillar in data.lookups.pillars.items()
+    }
+    new_team_groups = {
+        name: tg.model_copy(update={"group": _redact_group(tg.group)})
+        for name, tg in data.lookups.team_groups.items()
+    }
+    new_cohorts = {
+        name: c.model_copy(update={"group": _redact_group(c.group)})
+        for name, c in data.lookups.cohorts.items()
+    }
+
+    return data.model_copy(
+        update={
+            "lookups": data.lookups.model_copy(
+                update={
+                    "employees": new_employees,
+                    "teams": new_teams,
+                    "orgs": new_orgs,
+                    "pillars": new_pillars,
+                    "team_groups": new_team_groups,
+                    "cohorts": new_cohorts,
+                }
+            ),
+            "indexes": data.indexes.model_copy(
+                update={
+                    "slack_id_mappings": SlackIDMappings(slack_uid_to_uid={}),
+                    "github_id_mappings": GitHubIDMappings(github_id_to_uid={}),
+                }
+            ),
+        }
+    )
+
+
+def _redact_group(group: Group | None) -> Group | None:
+    """Return a new Group with escalation contacts redacted (None if no group)."""
+    if group is None or not group.escalation:
+        return group
+    new_escalation = [
+        contact.model_copy(
+            update={"name": "[REDACTED]", "url": "", "description": "[REDACTED]"}
+        )
+        for contact in group.escalation
+    ]
+    return group.model_copy(update={"escalation": new_escalation})

orgdatacore/_serialization.py

@@ -0,0 +1,24 @@
+"""Internal serialization: Data model -> dict/JSON bytes.
+
+Not part of the public API. Used by PII decorators and test helpers to
+round-trip typed Data through JSON without raw dict manipulation.
+
+The canonical ``orgindex-schema`` models mirror the on-disk index exactly, so
+``model_dump`` reproduces the index shape directly — no hand-written per-model
+serializers are needed.
+"""
+
+import json
+from typing import Any
+
+from ._types import Data
+
+
+def data_to_dict(data: Data) -> dict[str, Any]:
+    """Convert Data to a JSON-serializable dict in the on-disk index shape."""
+    return data.model_dump(mode="json")
+
+
+def data_to_json_bytes(data: Data) -> bytes:
+    """Convert Data to JSON bytes."""
+    return json.dumps(data_to_dict(data)).encode("utf-8")