dsgrid-toolkit 0.3.3__cp313-cp313-win_amd64.whl

build_backend.py

@@ -0,0 +1,93 @@
+"""Custom build backend that wraps maturin but allows fallback when Rust is unavailable.
+
+This allows `pip install -e .` to succeed even without a Rust toolchain installed,
+while still building the Rust extension when building wheels for distribution.
+"""
+
+import shutil
+import sys
+import warnings
+
+# Check if Rust toolchain is available
+RUST_AVAILABLE = shutil.which("cargo") is not None
+
+
+def _warn_no_rust():
+    """Warn that Rust extension will not be built."""
+    msg = (
+        "Rust toolchain (cargo) not found. The dsgrid Rust extension (minimal_patterns) "
+        "will not be built. Some functionality will be unavailable. "
+        "To build the Rust extension, install Rust from https://rustup.rs/"
+    )
+    # Use warnings module - this is typically shown by pip
+    warnings.warn(msg, UserWarning, stacklevel=2)
+    # Also print directly to stderr for visibility
+    print(f"\n{'='*70}", file=sys.stderr)
+    print("WARNING: Rust toolchain not found.", file=sys.stderr)
+    print("Building dsgrid without Rust extension (minimal_patterns).", file=sys.stderr)
+    print("Some functionality will be unavailable.", file=sys.stderr)
+    print(f"{'='*70}\n", file=sys.stderr)
+    # Flush to ensure it appears
+    sys.stderr.flush()
+
+
+# For wheel builds, use maturin if Rust is available, otherwise fall back to setuptools
+def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
+    """Build wheel - uses Rust if available, otherwise falls back to setuptools."""
+    if RUST_AVAILABLE:
+        import maturin
+
+        return maturin.build_wheel(wheel_directory, config_settings, metadata_directory)
+    else:
+        _warn_no_rust()
+        # Fall back to setuptools for wheel build without Rust extension
+        import setuptools.build_meta
+
+        return setuptools.build_meta.build_wheel(
+            wheel_directory, config_settings, metadata_directory
+        )
+
+
+def build_sdist(sdist_directory, config_settings=None):
+    """Build source distribution."""
+    import maturin
+
+    return maturin.build_sdist(sdist_directory, config_settings)
+
+
+def get_requires_for_build_wheel(config_settings=None):
+    """Get requirements for building wheel."""
+    if RUST_AVAILABLE:
+        return ["maturin>=1.0,<2.0"]
+    else:
+        return ["setuptools>=61.0"]
+
+
+def get_requires_for_build_sdist(config_settings=None):
+    """Get requirements for building sdist."""
+    return ["maturin>=1.0,<2.0"]
+
+
+# For editable installs, allow fallback without Rust
+def build_editable(wheel_directory, config_settings=None, metadata_directory=None):
+    """Build editable install - can work without Rust."""
+    if RUST_AVAILABLE:
+        import maturin
+
+        return maturin.build_editable(wheel_directory, config_settings, metadata_directory)
+    else:
+        _warn_no_rust()
+        # Fall back to setuptools for editable install without Rust extension
+        import setuptools.build_meta
+
+        return setuptools.build_meta.build_editable(
+            wheel_directory, config_settings, metadata_directory
+        )
+
+
+def get_requires_for_build_editable(config_settings=None):
+    """Get requirements for editable install."""
+    if RUST_AVAILABLE:
+        return ["maturin>=1.0,<2.0"]
+    else:
+        return ["setuptools>=61.0"]

dsgrid/__init__.py

@@ -0,0 +1,22 @@
+import datetime as dt
+import warnings
+
+from dsgrid.dsgrid_rc import DsgridRuntimeConfig
+from dsgrid.utils.timing import timer_stats_collector  # noqa: F401
+
+__title__ = "dsgrid"
+__description__ = (
+    "Python API for registring and accessing demand-side grid model (dsgrid) datasets"
+)
+__url__ = "https://github.com/dsgrid/dsgrid"
+__version__ = "0.3.3"
+__author__ = "NREL"
+__maintainer_email__ = "elaine.hale@nrel.gov"
+__license__ = "BSD-3"
+__copyright__ = "Copyright {}, The Alliance for Sustainable Energy, LLC".format(
+    dt.date.today().year
+)
+
+warnings.filterwarnings("ignore", module="duckdb_engine")
+
+runtime_config = DsgridRuntimeConfig.load()

dsgrid/api/api_manager.py

@@ -0,0 +1,179 @@
+import logging
+import threading
+from datetime import datetime
+from pathlib import Path
+
+from dsgrid.exceptions import DSGValueNotStored
+from dsgrid.registry.registry_manager import RegistryManager
+from dsgrid.utils.files import load_data
+from .models import StoreModel, AsyncTaskModel, AsyncTaskStatus, AsyncTaskType
+
+
+MAX_CONCURRENT_ASYNC_TASKS = 4
+
+logger = logging.getLogger(__name__)
+
+
+class ApiManager:
+    """Manages API requests"""
+
+    def __init__(
+        self,
+        home_dir: str | Path,
+        registry_manager: RegistryManager,
+        max_concurrent_async_tasks=MAX_CONCURRENT_ASYNC_TASKS,
+    ):
+        self._home_dir = Path(home_dir)
+        self._store = Store.load(self._home_dir)
+        self._lock = threading.RLock()
+        self._max_concurrent_async_tasks = max_concurrent_async_tasks
+        self._cached_projects = {}
+        self._registry_mgr = registry_manager
+
+    def can_start_new_async_task(self):
+        self._lock.acquire()
+        try:
+            return len(self._store.data.outstanding_async_tasks) < self._max_concurrent_async_tasks
+        finally:
+            self._lock.release()
+
+    def initialize_async_task(self, task_type: AsyncTaskType) -> int:
+        self._lock.acquire()
+        try:
+            num_outstanding = len(self._store.data.outstanding_async_tasks)
+            # TODO: implement queueing so that we don't return an error
+            if num_outstanding > self._max_concurrent_async_tasks:
+                msg = f"Too many async tasks are already running: {num_outstanding}"
+                raise Exception(msg)
+            async_task_id = self._get_next_async_task_id()
+            task = AsyncTaskModel(
+                async_task_id=async_task_id,
+                task_type=task_type,
+                status=AsyncTaskStatus.IN_PROGRESS,
+                start_time=datetime.now(),
+            )
+            self._store.data.async_tasks[async_task_id] = task
+            self._store.data.outstanding_async_tasks.add(async_task_id)
+            self._store.persist()
+        finally:
+            self._lock.release()
+
+        logger.info("Initialized async_task_id=%s", async_task_id)
+        return async_task_id
+
+    def clear_completed_async_tasks(self):
+        self._lock.acquire()
+        try:
+            to_remove = [
+                x.async_task_id
+                for x in self._store.data.async_tasks
+                if x.status == AsyncTaskStatus.COMPLETE
+            ]
+            for async_task_id in to_remove:
+                self._store.data.async_tasks.pop(async_task_id)
+            self._store.persist()
+            logger.info("Cleared %d completed tasks", len(to_remove))
+        finally:
+            self._lock.release()
+
+    def get_async_task_status(self, async_task_id):
+        """Return the status of the async ID."""
+        self._lock.acquire()
+        try:
+            return self._store.data.async_tasks[async_task_id]
+        finally:
+            self._lock.release()
+
+    def complete_async_task(self, async_task_id, return_code: int, result=None):
+        """Complete an asynchronous operation."""
+        self._lock.acquire()
+        try:
+            task = self._store.data.async_tasks[async_task_id]
+            task.status = AsyncTaskStatus.COMPLETE
+            task.return_code = return_code
+            task.completion_time = datetime.now()
+            self._store.data.outstanding_async_tasks.remove(async_task_id)
+            if result is not None:
+                task.result = result
+            self._store.persist()
+        finally:
+            self._lock.release()
+
+        logger.info("Completed async_task_id=%s", async_task_id)
+
+    def list_async_tasks(self, async_task_ids=None, status=None) -> list[AsyncTaskModel]:
+        """Return async tasks.
+
+        Parameters
+        ----------
+        async_task_ids : list | None
+            IDs of tasks for which to return status. If not set, return all statuses.
+        status : AsyncTaskStatus | None
+            If set, filter tasks by this status.
+
+        """
+        self._lock.acquire()
+        try:
+            if async_task_ids is not None:
+                diff = set(async_task_ids).difference(self._store.data.async_tasks.keys())
+                if diff:
+                    msg = f"async_task_ids={diff} are not stored"
+                    raise DSGValueNotStored(msg)
+            tasks = (
+                self._store.data.async_tasks.keys() if async_task_ids is None else async_task_ids
+            )
+            return [
+                self._store.data.async_tasks[x]
+                for x in tasks
+                if status is None or self._store.data.async_tasks[x].status == status
+            ]
+        finally:
+            self._lock.release()
+
+    def _get_next_async_task_id(self) -> int:
+        self._lock.acquire()
+        try:
+            next_id = self._store.data.next_async_task_id
+            self._store.data.next_async_task_id += 1
+            self._store.persist()
+        finally:
+            self._lock.release()
+
+        return next_id
+
+    def get_project(self, project_id):
+        """Load a Project and cache it for future calls.
+        Loading is slow and the Project isn't being changed by this API.
+        """
+        self._lock.acquire()
+        try:
+            project = self._cached_projects.get(project_id)
+            if project is not None:
+                return project
+            project = self._registry_mgr.project_manager.load_project(project_id)
+            self._cached_projects[project_id] = project
+            return project
+        finally:
+            self._lock.release()
+
+
+class Store:
+    STORE_FILENAME = "api_server_store.json"
+
+    def __init__(self, store_file: Path, data: StoreModel):
+        self._store_file = store_file
+        self.data = data
+
+    @classmethod
+    def load(cls, path: Path):
+        # TODO: use MongoDB or some other db
+        store_file = path / cls.STORE_FILENAME
+        if store_file.exists():
+            logger.info("Load from existing store: %s", store_file)
+            store_data = load_data(store_file)
+            return cls(store_file, StoreModel(**store_data))
+        logger.info("Create new store: %s", store_file)
+        return cls(store_file, StoreModel())
+
+    def persist(self):
+        self._store_file.write_text(self.data.model_dump_json(indent=2))