dsgrid-toolkit 0.3.2__tar.gz → 0.4.0__tar.gz

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: dsgrid-toolkit
-Version: 0.3.2
+Version: 0.4.0
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: chronify~=0.6.0
+Requires-Dist: chronify~=0.7.0
 Requires-Dist: click>=8.2,<9
 Requires-Dist: dash
 Requires-Dist: dash-bootstrap-components
@@ -35,16 +35,19 @@ Requires-Dist: pyarrow ; extra == 'dev'
 Requires-Dist: maturin ; extra == 'dev'
 Requires-Dist: furo ; extra == 'doc'
 Requires-Dist: ghp-import ; extra == 'doc'
+Requires-Dist: linkify-it-py~=2.0 ; extra == 'doc'
+Requires-Dist: myst-parser~=2.0 ; extra == 'doc'
 Requires-Dist: numpydoc ; extra == 'doc'
 Requires-Dist: pandas-stubs ; extra == 'doc'
 Requires-Dist: ruff ; extra == 'doc'
 Requires-Dist: sphinx~=7.2 ; extra == 'doc'
 Requires-Dist: sphinx-click~=5.0 ; extra == 'doc'
 Requires-Dist: sphinx-copybutton~=0.5.2 ; extra == 'doc'
+Requires-Dist: sphinx-design~=0.5.0 ; extra == 'doc'
 Requires-Dist: sphinx-tabs~=3.4 ; extra == 'doc'
 Requires-Dist: sphinx-argparse~=0.4.0 ; extra == 'doc'
 Requires-Dist: sphinxcontrib-programoutput ; extra == 'doc'
-Requires-Dist: autodoc-pydantic[erdantic]~=2.0 ; extra == 'doc'
+Requires-Dist: autodoc-pydantic~=2.0 ; extra == 'doc'
 Requires-Dist: twine ; extra == 'release'
 Requires-Dist: setuptools ; extra == 'release'
 Requires-Dist: wheel ; extra == 'release'
@@ -59,8 +62,8 @@ Provides-Extra: spark
 License-File: LICENSE
 Summary: Python API for accessing demand-side grid model (dsgrid) datasets
 Keywords: dsgrid
-Author-email: Elaine Hale <elaine.hale@nrel.gov>, Lixi Liu <lixi.liu@nrel.gov>, Meghan Mooney <meghan.mooney@nrel.gov>, Daniel Thom <daniel.thom@nrel.gov>
-Maintainer-email: Elaine Hale <elaine.hale@nrel.gov>
+Author-email: Elaine Hale <elaine.hale@nlr.gov>, Daniel Thom <daniel.thom@nlr.gov>, Lixi Liu <lixi.liu@nlr.gov>, Meghan Mooney <meghan.mooney@nlr.gov>
+Maintainer-email: Elaine Hale <elaine.hale@nlr.gov>
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: Documentation, https://dsgrid.github.io/dsgrid/

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/build_backend.py

@@ -31,12 +31,21 @@ def _warn_no_rust():
     sys.stderr.flush()
 
 
-# For wheel builds (CI), always use maturin
+# For wheel builds, use maturin if Rust is available, otherwise fall back to setuptools
 def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
-    """Build wheel - requires Rust for distribution builds."""
-    import maturin
+    """Build wheel - uses Rust if available, otherwise falls back to setuptools."""
+    if RUST_AVAILABLE:
+        import maturin
+
+        return maturin.build_wheel(wheel_directory, config_settings, metadata_directory)
+    else:
+        _warn_no_rust()
+        # Fall back to setuptools for wheel build without Rust extension
+        import setuptools.build_meta
 
-    return maturin.build_wheel(wheel_directory, config_settings, metadata_directory)
+        return setuptools.build_meta.build_wheel(
+            wheel_directory, config_settings, metadata_directory
+        )
 
 
 def build_sdist(sdist_directory, config_settings=None):
@@ -48,7 +57,10 @@ def build_sdist(sdist_directory, config_settings=None):
 
 def get_requires_for_build_wheel(config_settings=None):
     """Get requirements for building wheel."""
-    return ["maturin>=1.0,<2.0"]
+    if RUST_AVAILABLE:
+        return ["maturin>=1.0,<2.0"]
+    else:
+        return ["setuptools>=61.0"]
 
 
 def get_requires_for_build_sdist(config_settings=None):

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/dsgrid/__init__.py

@@ -9,11 +9,11 @@ __description__ = (
     "Python API for registring and accessing demand-side grid model (dsgrid) datasets"
 )
 __url__ = "https://github.com/dsgrid/dsgrid"
-__version__ = "0.3.2"
-__author__ = "NREL"
-__maintainer_email__ = "elaine.hale@nrel.gov"
+__version__ = "0.4.0"
+__author__ = "National Laboratory of the Rockies (formerly NREL)"
+__maintainer_email__ = "elaine.hale@nlr.gov"
 __license__ = "BSD-3"
-__copyright__ = "Copyright {}, The Alliance for Sustainable Energy, LLC".format(
+__copyright__ = "Copyright {}, Alliance for Energy Innovation, LLC".format(
     dt.date.today().year
 )
 

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/dsgrid/api/app.py

@@ -360,13 +360,15 @@ def download_async_task_archive_file(async_task_id: int):
 
 
 def _submit_project_query(spark_query: SparkSubmitProjectQueryRequest, async_task_id):
-    with NamedTemporaryFile(mode="w", suffix=".json") as fp:
+    fp = NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+    try:
         query = spark_query.query
         fp.write(query.model_dump_json())
         fp.write("\n")
         fp.flush()
+        fp.close()
         output_dir = Path(QUERY_OUTPUT_DIR)
-        dsgrid_exec = "dsgrid-cli.py"
+        dsgrid_exec = "dsgrid-cli.exe" if sys.platform == "win32" else "dsgrid-cli.py"
         base_cmd = (
             f"--url={DSGRID_REGISTRY_DATABASE_URL} "
             f"query project run "
@@ -402,6 +404,8 @@ def _submit_project_query(spark_query: SparkSubmitProjectQueryRequest, async_tas
                 archive_file="",
                 archive_file_size_mb=0,
             )
+    finally:
+        Path(fp.name).unlink(missing_ok=True)
 
     api_mgr.complete_async_task(async_task_id, ret, result=result)
 

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/dsgrid/cli/config.py

@@ -13,7 +13,7 @@ from dsgrid.dsgrid_rc import (
     DEFAULT_BACKEND,
 )
 from dsgrid.exceptions import DSGInvalidParameter
-from dsgrid.registry.common import DatabaseConnection
+from dsgrid.registry.common import DatabaseConnection, make_sqlite_url
 
 
 logger = logging.getLogger(__name__)
@@ -146,7 +146,7 @@ def create(
     try:
         db_filename = conn.get_filename()
         if use_absolute_db_path and not db_filename.is_absolute():
-            conn.url = f"sqlite:///{db_filename.resolve()}"
+            conn.url = make_sqlite_url(db_filename.resolve())
 
     except DSGInvalidParameter as exc:
         print(str(exc), file=sys.stderr)

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/dsgrid/cli/dsgrid.py

@@ -14,7 +14,6 @@ from dsgrid.utils.timing import timer_stats_collector
 from dsgrid.cli.common import get_log_level_from_str, handle_scratch_dir
 from dsgrid.cli.config import config
 from dsgrid.cli.download import download
-from dsgrid.cli.install_notebooks import install_notebooks
 from dsgrid.cli.query import query
 from dsgrid.cli.registry import registry
 from dsgrid.loggers import setup_logging, check_log_file_size, disable_console_logging
@@ -146,12 +145,12 @@ def create_registry(url: str, data_path: Path, overwrite: bool, data_store_type:
     """Create a new registry."""
     check_overwrite(data_path, overwrite)
     conn = DatabaseConnection(url=url)
-    RegistryManager.create(conn, data_path, overwrite=overwrite, data_store_type=data_store_type)
+    mgr = RegistryManager.create(conn, data_path, overwrite=overwrite, data_store_type=data_store_type)
+    mgr.dispose()
 
 
 cli.add_command(config)
 cli.add_command(create_registry)
 cli.add_command(download)
-cli.add_command(install_notebooks)
 cli.add_command(query)
 cli.add_command(registry)

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/dsgrid/cli/registry.py

@@ -35,6 +35,53 @@ from dsgrid.utils.filters import ACCEPTED_OPS
 logger = logging.getLogger(__name__)
 
 
+class _LazyRegistryManager:
+    """Defers RegistryManager creation until first attribute access.
+
+    This allows Click to process --help on subcommands without requiring a
+    registry URL, while still failing with a clear error message when a
+    command actually needs the registry manager.
+    """
+
+    def __init__(self, ctx):
+        self._ctx = ctx
+        self._manager = None
+        self._initialized = False
+
+    def _init_manager(self):
+        if self._initialized:
+            return
+        self._initialized = True
+        url = get_value_from_context(self._ctx, "url")
+        if url is None:
+            msg = (
+                "A registry URL is required. Pass it as 'dsgrid --url <URL> registry <command>' "
+                "or set it via the DSGRID_REGISTRY_DATABASE_URL environment variable "
+                "or in ~/.dsgrid.json5."
+            )
+            raise click.UsageError(msg)
+        conn = DatabaseConnection(url=url)
+        scratch_dir = get_value_from_context(self._ctx, "scratch_dir")
+        no_prompts = self._ctx.parent.params["no_prompts"]
+        self._manager = RegistryManager.load(
+            conn,
+            REMOTE_REGISTRY,
+            offline_mode=True,
+            no_prompts=no_prompts,
+            scratch_dir=scratch_dir,
+        )
+
+    def __getattr__(self, name):
+        if name.startswith("_"):
+            raise AttributeError(name)
+        self._init_manager()
+        return getattr(self._manager, name)
+
+    def dispose(self):
+        if self._manager is not None:
+            self._manager.dispose()
+
+
 def _version_info_callback(*args) -> VersionInfo | None:
     val = args[2]
     if val is None:
@@ -61,21 +108,7 @@ Click Group Definitions
 @click.pass_context
 def registry(ctx):
     """Manage a registry."""
-    conn = DatabaseConnection(
-        url=get_value_from_context(ctx, "url"),
-    )
-    scratch_dir = get_value_from_context(ctx, "scratch_dir")
-    no_prompts = ctx.parent.params["no_prompts"]
-    if "--help" in sys.argv:
-        ctx.obj = None
-    else:
-        ctx.obj = RegistryManager.load(
-            conn,
-            REMOTE_REGISTRY,
-            offline_mode=True,
-            no_prompts=no_prompts,
-            scratch_dir=scratch_dir,
-        )
+    ctx.obj = _LazyRegistryManager(ctx)
 
 
 @registry.result_callback()
@@ -834,11 +867,11 @@ $ dsgrid registry projects register-and-submit-dataset \\ \n
     "prepend them with this path.",
 )
 @click.option(
-    "-M",
-    "--missing-associations-base-dir",
+    "-A",
+    "--associations-base-dir",
     type=click.Path(exists=True),
     callback=path_callback,
-    help="Base directory for missing associations files. If set and missing associations "
+    help="Base directory for missing/expected associations files. If set and associations "
     "paths are relative, prepend them with this path.",
 )
 @click.pass_obj
@@ -853,7 +886,7 @@ def register_and_submit_dataset(
     project_id,
     log_message,
     data_base_dir,
-    missing_associations_base_dir,
+    associations_base_dir,
 ):
     """Register a dataset and then submit it to a dsgrid project.
 
@@ -873,7 +906,7 @@ def register_and_submit_dataset(
         dimension_mapping_references_file=dimension_mapping_references_file,
         autogen_reverse_supplemental_mappings=autogen_reverse_supplemental_mappings,
         data_base_dir=data_base_dir,
-        missing_associations_base_dir=missing_associations_base_dir,
+        associations_base_dir=associations_base_dir,
     )
     if res[1] != 0:
         ctx.exit(res[1])
@@ -1430,11 +1463,11 @@ $ dsgrid registry datasets register dataset.json5 --data-base-dir /path/to/data
     "prepend them with this path.",
 )
 @click.option(
-    "-M",
-    "--missing-associations-base-dir",
+    "-A",
+    "--associations-base-dir",
     type=click.Path(exists=True),
     callback=path_callback,
-    help="Base directory for missing associations files. If set and missing associations "
+    help="Base directory for missing/expected associations files. If set and associations "
     "paths are relative, prepend them with this path.",
 )
 @click.pass_obj
@@ -1445,7 +1478,7 @@ def register_dataset(
     dataset_config_file: Path,
     log_message: str,
     data_base_dir: Path | None,
-    missing_associations_base_dir: Path | None,
+    associations_base_dir: Path | None,
 ):
     """Register a new dataset with the registry. The contents of the JSON/JSON5 file
     must match the data model defined by this documentation:
@@ -1463,7 +1496,7 @@ def register_dataset(
         submitter,
         log_message,
         data_base_dir=data_base_dir,
-        missing_associations_base_dir=missing_associations_base_dir,
+        associations_base_dir=associations_base_dir,
     )
     if res[1] != 0:
         ctx.exit(res[1])
@@ -1761,11 +1794,11 @@ $ dsgrid registry bulk-register registration.json5 -j journal__11f733f6-ac9b-4f7
     "prepend them with this path.",
 )
 @click.option(
-    "-M",
-    "--missing-associations-base-dir",
+    "-A",
+    "--associations-base-dir",
     type=click.Path(exists=True),
     callback=path_callback,
-    help="Base directory for missing associations files. If set and missing associations "
+    help="Base directory for missing/expected associations files. If set and associations "
     "paths are relative, prepend them with this path.",
 )
 @click.option(
@@ -1792,7 +1825,7 @@ def bulk_register_cli(
     registry_manager: RegistryManager,
     registration_file: Path,
     data_base_dir: Path | None,
-    missing_associations_base_dir: Path | None,
+    associations_base_dir: Path | None,
     repo_base_dir: Path | None,
     journal_file: Path | None,
 ):
@@ -1811,7 +1844,7 @@ def bulk_register_cli(
         registry_manager,
         registration_file,
         data_base_dir=data_base_dir,
-        missing_associations_base_dir=missing_associations_base_dir,
+        associations_base_dir=associations_base_dir,
         repo_base_dir=repo_base_dir,
         journal_file=journal_file,
     )

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/dsgrid/common.py

@@ -1,7 +1,10 @@
 import enum
 import os
+import sys
 from pathlib import Path
 
+IS_WINDOWS = sys.platform == "win32"
+
 # AWS_PROFILE_NAME = "nrel-aws-dsgrid"
 REMOTE_REGISTRY = "s3://nrel-dsgrid-registry"
 
@@ -27,6 +30,7 @@ SCALING_FACTOR_COLUMN = "scaling_factor"
 SYNC_EXCLUDE_LIST = ["*.DS_Store", "**/*.lock"]
 TIME_ZONE_COLUMN = "time_zone"
 VALUE_COLUMN = "value"
+TIME_COLUMN = "timestamp"
 
 
 class BackendEngine(enum.StrEnum):

{dsgrid_toolkit-0.3.2 → dsgrid_toolkit-0.4.0}/dsgrid/config/dataset_config.py

@@ -48,6 +48,11 @@ ALLOWED_LOAD_DATA_LOOKUP_FILENAMES = (
     "load_data_lookup.json",
 )
 ALLOWED_DATA_FILES = ALLOWED_LOAD_DATA_FILENAMES + ALLOWED_LOAD_DATA_LOOKUP_FILENAMES
+ALLOWED_EXPECTED_DIMENSION_ASSOCIATIONS_FILENAMES = (
+    "expected_associations.csv",
+    "expected_associations.parquet",
+)
+EXPECTED_ASSOCIATIONS_DIR_NAME = "expected_associations"
 ALLOWED_MISSING_DIMENSION_ASSOCATIONS_FILENAMES = (
     "missing_associations.csv",
     "missing_associations.parquet",
@@ -209,6 +214,15 @@ class UserDataLayout(DSGBaseModel):
         title="lookup_data_file",
         description="Defines the lookup data file. Required if the table format is 'two_table'.",
     )
+    expected_associations: list[str] = Field(
+        default=[],
+        title="expected_associations",
+        description="List of paths to expected associations files (e.g., "
+        "expected_associations.parquet) or directories of files containing expected combinations "
+        "by dimension type (e.g., geography__subsector.csv, subsector__metric.csv). "
+        "When provided, only these dimension combinations are required to be present in the "
+        "data, rather than the full cross-join of all dimension records.",
+    )
     missing_associations: list[str] = Field(
         default=[],
         title="missing_associations",
@@ -631,6 +645,34 @@ def make_unvalidated_dataset_config(
 class DatasetConfig(ConfigBase):
     """Provides an interface to a DatasetConfigModel."""
 
+    @staticmethod
+    def _resolve_path(
+        raw_path: str,
+        base_dir: Path | None,
+        config_file_parent: Path,
+    ) -> Path:
+        """Resolve a possibly-relative path against a base directory.
+
+        Parameters
+        ----------
+        raw_path
+            Path string from the config (may be relative or absolute).
+        base_dir
+            If set and the path is relative, resolve against this directory.
+        config_file_parent
+            Fallback directory when *base_dir* is None.
+
+        Returns
+        -------
+        Path
+            Resolved absolute path.
+        """
+        path = Path(raw_path)
+        if not path.is_absolute():
+            parent = base_dir if base_dir is not None else config_file_parent
+            path = (parent / path).resolve()
+        return path
+
     def __init__(self, model):
         super().__init__(model)
         self._dimensions = {}  # ConfigKey to DimensionConfig
@@ -652,7 +694,7 @@ class DatasetConfig(ConfigBase):
         cls,
         config_file: Path,
         data_base_dir: Path | None = None,
-        missing_associations_base_dir: Path | None = None,
+        associations_base_dir: Path | None = None,
     ) -> "DatasetConfig":
         """Load a dataset config from a user-provided config file.
 
@@ -666,9 +708,10 @@ class DatasetConfig(ConfigBase):
         data_base_dir : Path | None, optional
             Base directory for data files. If set and data file paths are relative,
             prepend them with this path instead of using the config file's parent directory.
-        missing_associations_base_dir : Path | None, optional
-            Base directory for missing associations files. If set and paths are relative,
-            prepend them with this path instead of using the config file's parent directory.
+        associations_base_dir : Path | None, optional
+            Base directory for missing/expected associations files. If set and paths are
+            relative, prepend them with this path instead of using the config file's
+            parent directory.
 
         Returns
         -------
@@ -694,12 +737,9 @@ class DatasetConfig(ConfigBase):
             raise DSGInvalidParameter(msg)
 
         # Resolve data file path
-        data_path = Path(user_layout.data_file.path)
-        if not data_path.is_absolute():
-            if data_base_dir is not None:
-                data_path = (data_base_dir / data_path).resolve()
-            else:
-                data_path = (config_file.parent / data_path).resolve()
+        data_path = cls._resolve_path(
+            user_layout.data_file.path, data_base_dir, config_file.parent
+        )
         if str(data_path).startswith("s3://"):
             msg = "Registering a dataset from an S3 path is not supported."
             raise DSGInvalidParameter(msg)
@@ -714,28 +754,25 @@ class DatasetConfig(ConfigBase):
             if user_layout.lookup_data_file is None:
                 msg = "Two-table format requires lookup_data_file in data_layout"
                 raise DSGInvalidParameter(msg)
-            lookup_path = Path(user_layout.lookup_data_file.path)
-            if not lookup_path.is_absolute():
-                if data_base_dir is not None:
-                    lookup_path = (data_base_dir / lookup_path).resolve()
-                else:
-                    lookup_path = (config_file.parent / lookup_path).resolve()
+            lookup_path = cls._resolve_path(
+                user_layout.lookup_data_file.path, data_base_dir, config_file.parent
+            )
             if not lookup_path.exists():
                 msg = f"Lookup data file does not exist: {lookup_path}"
                 raise DSGInvalidParameter(msg)
             user_layout.lookup_data_file.path = str(lookup_path)
 
+        # Resolve expected associations paths
+        user_layout.expected_associations = [
+            str(cls._resolve_path(p, associations_base_dir, config_file.parent))
+            for p in user_layout.expected_associations
+        ]
+
         # Resolve missing associations paths
-        resolved_missing_paths: list[str] = []
-        for missing_assoc in user_layout.missing_associations:
-            missing_path = Path(missing_assoc)
-            if not missing_path.is_absolute():
-                if missing_associations_base_dir is not None:
-                    missing_path = (missing_associations_base_dir / missing_path).resolve()
-                else:
-                    missing_path = (config_file.parent / missing_path).resolve()
-            resolved_missing_paths.append(str(missing_path))
-        user_layout.missing_associations = resolved_missing_paths
+        user_layout.missing_associations = [
+            str(cls._resolve_path(p, associations_base_dir, config_file.parent))
+            for p in user_layout.missing_associations
+        ]
 
         return config
 
@@ -758,6 +795,13 @@ class DatasetConfig(ConfigBase):
             return self.model.data_layout.lookup_data_file
         return None
 
+    @property
+    def expected_associations_paths(self) -> list[Path]:
+        """Return the list of expected associations paths if available."""
+        if self.model.data_layout is not None:
+            return [Path(p) for p in self.model.data_layout.expected_associations]
+        return []
+
     @property
     def missing_associations_paths(self) -> list[Path]:
         """Return the list of missing associations paths if available."""