dsgrid-toolkit 0.2.0__tar.gz → 0.3.1__tar.gz

{dsgrid_toolkit-0.2.0 → dsgrid_toolkit-0.3.1}/PKG-INFO

@@ -1,54 +1,16 @@
 Metadata-Version: 2.4
 Name: dsgrid-toolkit
-Version: 0.2.0
-Summary: Python API for accessing demand-side grid model (dsgrid) datasets
-Project-URL: GitHub, https://github.com/dsgrid/dsgrid
-Project-URL: Documentation, https://dsgrid.github.io/dsgrid/
-Project-URL: Homepage, https://www.nrel.gov/analysis/dsgrid
-Author-email: Elaine Hale <elaine.hale@nrel.gov>, Lixi Liu <lixi.liu@nrel.gov>, Meghan Mooney <meghan.mooney@nrel.gov>, Daniel Thom <daniel.thom@nrel.gov>
-Maintainer-email: Elaine Hale <elaine.hale@nrel.gov>
-License: BSD 3-Clause License
-        
-        Copyright (c) 2021, Alliance for Sustainable Energy, LLC
-        All rights reserved.
-        
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-        
-        1. Redistributions of source code must retain the above copyright notice, this
-           list of conditions and the following disclaimer.
-        
-        2. Redistributions in binary form must reproduce the above copyright notice,
-           this list of conditions and the following disclaimer in the documentation
-           and/or other materials provided with the distribution.
-        
-        3. Neither the name of the copyright holder nor the names of its
-           contributors may be used to endorse or promote products derived from
-           this software without specific prior written permission.
-        
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-License-File: LICENSE
-Keywords: dsgrid
+Version: 0.3.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.11
-Requires-Dist: chronify~=0.3.1
-Requires-Dist: click<9,>=8.2
+Requires-Dist: chronify~=0.6.0
+Requires-Dist: click>=8.2,<9
 Requires-Dist: dash
 Requires-Dist: dash-bootstrap-components
-Requires-Dist: duckdb<2,>=1
+Requires-Dist: duckdb>=1,<2
 Requires-Dist: fastapi
 Requires-Dist: json5
 Requires-Dist: networkx
@@ -59,41 +21,51 @@ Requires-Dist: pydantic~=2.11.6
 Requires-Dist: requests
 Requires-Dist: rich-click
 Requires-Dist: semver
-Requires-Dist: sqlalchemy<3,>=2
-Requires-Dist: tzdata
+Requires-Dist: sqlalchemy>=2,<3
 Requires-Dist: uvicorn
+Requires-Dist: tzdata
+Requires-Dist: httpx ; extra == 'dev'
+Requires-Dist: pytest ; extra == 'dev'
+Requires-Dist: pytest-cov ; extra == 'dev'
+Requires-Dist: pre-commit ; extra == 'dev'
+Requires-Dist: devtools ; extra == 'dev'
+Requires-Dist: flake8 ; extra == 'dev'
+Requires-Dist: mypy ; extra == 'dev'
+Requires-Dist: pyarrow ; extra == 'dev'
+Requires-Dist: maturin ; extra == 'dev'
+Requires-Dist: furo ; extra == 'doc'
+Requires-Dist: ghp-import ; extra == 'doc'
+Requires-Dist: numpydoc ; extra == 'doc'
+Requires-Dist: pandas-stubs ; extra == 'doc'
+Requires-Dist: ruff ; extra == 'doc'
+Requires-Dist: sphinx~=7.2 ; extra == 'doc'
+Requires-Dist: sphinx-click~=5.0 ; extra == 'doc'
+Requires-Dist: sphinx-copybutton~=0.5.2 ; extra == 'doc'
+Requires-Dist: sphinx-tabs~=3.4 ; extra == 'doc'
+Requires-Dist: sphinx-argparse~=0.4.0 ; extra == 'doc'
+Requires-Dist: sphinxcontrib-programoutput ; extra == 'doc'
+Requires-Dist: autodoc-pydantic[erdantic]~=2.0 ; extra == 'doc'
+Requires-Dist: twine ; extra == 'release'
+Requires-Dist: setuptools ; extra == 'release'
+Requires-Dist: wheel ; extra == 'release'
+Requires-Dist: chronify[spark] ; extra == 'spark'
+Requires-Dist: pyspark==4.0.0 ; extra == 'spark'
+Requires-Dist: thrift ; extra == 'spark'
+Requires-Dist: thrift-sasl ; extra == 'spark'
 Provides-Extra: dev
-Requires-Dist: devtools; extra == 'dev'
-Requires-Dist: flake8; extra == 'dev'
-Requires-Dist: httpx; extra == 'dev'
-Requires-Dist: mypy; extra == 'dev'
-Requires-Dist: pre-commit; extra == 'dev'
-Requires-Dist: pyarrow; extra == 'dev'
-Requires-Dist: pytest; extra == 'dev'
-Requires-Dist: pytest-cov; extra == 'dev'
 Provides-Extra: doc
-Requires-Dist: autodoc-pydantic[erdantic]~=2.0; extra == 'doc'
-Requires-Dist: furo; extra == 'doc'
-Requires-Dist: ghp-import; extra == 'doc'
-Requires-Dist: numpydoc; extra == 'doc'
-Requires-Dist: pandas-stubs; extra == 'doc'
-Requires-Dist: ruff; extra == 'doc'
-Requires-Dist: sphinx-argparse~=0.4.0; extra == 'doc'
-Requires-Dist: sphinx-click~=5.0; extra == 'doc'
-Requires-Dist: sphinx-copybutton~=0.5.2; extra == 'doc'
-Requires-Dist: sphinx-tabs~=3.4; extra == 'doc'
-Requires-Dist: sphinxcontrib-programoutput; extra == 'doc'
-Requires-Dist: sphinx~=7.2; extra == 'doc'
 Provides-Extra: release
-Requires-Dist: setuptools; extra == 'release'
-Requires-Dist: twine; extra == 'release'
-Requires-Dist: wheel; extra == 'release'
 Provides-Extra: spark
-Requires-Dist: chronify[spark]; extra == 'spark'
-Requires-Dist: pyspark==4.0.0; extra == 'spark'
-Requires-Dist: thrift; extra == 'spark'
-Requires-Dist: thrift-sasl; extra == 'spark'
-Description-Content-Type: text/markdown
+License-File: LICENSE
+Summary: Python API for accessing demand-side grid model (dsgrid) datasets
+Keywords: dsgrid
+Author-email: Elaine Hale <elaine.hale@nrel.gov>, Lixi Liu <lixi.liu@nrel.gov>, Meghan Mooney <meghan.mooney@nrel.gov>, Daniel Thom <daniel.thom@nrel.gov>
+Maintainer-email: Elaine Hale <elaine.hale@nrel.gov>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Documentation, https://dsgrid.github.io/dsgrid/
+Project-URL: GitHub, https://github.com/dsgrid/dsgrid
+Project-URL: Homepage, https://www.nrel.gov/analysis/dsgrid
 
 # dsgrid
 [![Documentation](https://img.shields.io/badge/docs-ready-blue.svg)](https://dsgrid.github.io/dsgrid)
@@ -114,7 +86,7 @@ Python API for contributing to and accessing demand-side grid model (dsgrid) pro
 Create a virtual environment in which to install dsgrid. Anaconda or miniconda is recommended.
 
 ```
-conda create -n dsgrid python=3.10
+conda create -n dsgrid python=3.11
 conda activate dsgrid
 ```
 
@@ -123,7 +95,7 @@ conda activate dsgrid
 dsgrid uses [Apache Spark](#https://spark.apache.org/) to manage big data. There are no separate installation steps for Apache Spark beyond installing the dsgrid package and installing:
 
 ```
-pip install git+https://github.com/apache/kyuubi.git#egg=pyhive&subdirectory=python
+pip install "dsgrid-toolkit[spark]"
 ```
 
 Otherwise installing the pyspark Python dependency handles it.
@@ -148,7 +120,11 @@ To install Apache Spark on Windows, follow [these instructions](https://towardsd
 
 ### From PIPY/pip
 
-*Not yet available*
+pip install dsgrid-toolkit
+
+or
+
+pip install "dsgrid-toolkit[spark]"
 
 ### From pip+git
 
@@ -190,7 +166,7 @@ pip install -e .
 
 **Developers:**
 ```
-pip install -e '.[dev]'
+pip install -e '.[dev,spark]'
 ```
 
 ## Usage
@@ -214,3 +190,4 @@ conda deactivate
 ## Software Record
 
 dsgrid is developed under NREL Software Record SWR-21-52, "demand-side grid model".
+

{dsgrid_toolkit-0.2.0 → dsgrid_toolkit-0.3.1}/README.md

@@ -17,7 +17,7 @@ Python API for contributing to and accessing demand-side grid model (dsgrid) pro
 Create a virtual environment in which to install dsgrid. Anaconda or miniconda is recommended.
 
 ```
-conda create -n dsgrid python=3.10
+conda create -n dsgrid python=3.11
 conda activate dsgrid
 ```
 
@@ -26,7 +26,7 @@ conda activate dsgrid
 dsgrid uses [Apache Spark](#https://spark.apache.org/) to manage big data. There are no separate installation steps for Apache Spark beyond installing the dsgrid package and installing:
 
 ```
-pip install git+https://github.com/apache/kyuubi.git#egg=pyhive&subdirectory=python
+pip install "dsgrid-toolkit[spark]"
 ```
 
 Otherwise installing the pyspark Python dependency handles it.
@@ -51,7 +51,11 @@ To install Apache Spark on Windows, follow [these instructions](https://towardsd
 
 ### From PIPY/pip
 
-*Not yet available*
+pip install dsgrid-toolkit
+
+or
+
+pip install "dsgrid-toolkit[spark]"
 
 ### From pip+git
 
@@ -93,7 +97,7 @@ pip install -e .
 
 **Developers:**
 ```
-pip install -e '.[dev]'
+pip install -e '.[dev,spark]'
 ```
 
 ## Usage

{dsgrid_toolkit-0.2.0 → dsgrid_toolkit-0.3.1}/dsgrid/__init__.py

@@ -9,7 +9,7 @@ __description__ = (
     "Python API for registring and accessing demand-side grid model (dsgrid) datasets"
 )
 __url__ = "https://github.com/dsgrid/dsgrid"
-__version__ = "0.2.0"
+__version__ = "0.3.1"
 __author__ = "NREL"
 __maintainer_email__ = "elaine.hale@nrel.gov"
 __license__ = "BSD-3"

{dsgrid_toolkit-0.2.0 → dsgrid_toolkit-0.3.1}/dsgrid/api/app.py

@@ -10,7 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import Response, FileResponse
 
 from dsgrid.common import REMOTE_REGISTRY
-from dsgrid.dataset.models import TableFormatType
+from dsgrid.dataset.models import ValueFormat
 from dsgrid.config.dimensions import create_dimension_common_model, create_project_dimension_model
 from dsgrid.dimension.base_models import DimensionType, DimensionCategory
 from dsgrid.dsgrid_rc import DsgridRuntimeConfig
@@ -44,7 +44,7 @@ from .response_models import (
     ListProjectSupplementalDimensionNames,
     ListProjectsResponse,
     ListReportTypesResponse,
-    ListTableFormatTypesResponse,
+    ListValueFormatsResponse,
     SparkSubmitProjectQueryResponse,
 )
 
@@ -280,10 +280,10 @@ async def list_report_types():
     return ListReportTypesResponse(types=_list_enums(ReportType))
 
 
-@app.get("/table_formats/types", response_model=ListTableFormatTypesResponse)
-async def list_table_format_types():
-    """List the table format types available for query results."""
-    return ListTableFormatTypesResponse(types=_list_enums(TableFormatType))
+@app.get("/value_formats", response_model=ListValueFormatsResponse)
+async def list_value_formats():
+    """List the value formats available for query results."""
+    return ListValueFormatsResponse(formats=_list_enums(ValueFormat))
 
 
 @app.post("/queries/projects", response_model=SparkSubmitProjectQueryResponse)
@@ -368,7 +368,6 @@ def _submit_project_query(spark_query: SparkSubmitProjectQueryRequest, async_tas
         output_dir = Path(QUERY_OUTPUT_DIR)
         dsgrid_exec = "dsgrid-cli.py"
         base_cmd = (
-            f"--offline "
             f"--url={DSGRID_REGISTRY_DATABASE_URL} "
             f"query project run "
             f"--output={output_dir} --zip-file --overwrite {fp.name}"

{dsgrid_toolkit-0.2.0 → dsgrid_toolkit-0.3.1}/dsgrid/api/response_models.py

@@ -2,7 +2,7 @@ from dsgrid.data_models import DSGBaseModel
 from dsgrid.config.dataset_config import DatasetConfigModel
 from dsgrid.config.dimensions import DimensionCommonModel, ProjectDimensionModel
 from dsgrid.config.project_config import ProjectConfigModel, ProjectDimensionNamesModel
-from dsgrid.dataset.models import TableFormatType
+from dsgrid.dataset.models import ValueFormat
 from dsgrid.dimension.base_models import DimensionType
 from dsgrid.query.models import ReportType
 from .models import AsyncTaskModel
@@ -92,10 +92,10 @@ class ListReportTypesResponse(DSGBaseModel):
     types: list[ReportType]
 
 
-class ListTableFormatTypesResponse(DSGBaseModel):
-    """Defines the response to the list_table_format_types command."""
+class ListValueFormatsResponse(DSGBaseModel):
+    """Defines the response to the list_value_formats command."""
 
-    types: list[TableFormatType]
+    formats: list[ValueFormat]
 
 
 class SparkSubmitProjectQueryResponse(DSGBaseModel):

{dsgrid_toolkit-0.2.0 → dsgrid_toolkit-0.3.1}/dsgrid/chronify.py

@@ -20,3 +20,13 @@ def create_store(store_file: Path) -> Generator[chronify.Store, None, None]:
         yield store
     finally:
         store.dispose()
+
+
+@contextmanager
+def create_in_memory_store() -> Generator[chronify.Store, None, None]:
+    """Create an in-memory chronify Store."""
+    store = chronify.Store.create_in_memory_db()
+    try:
+        yield store
+    finally:
+        store.dispose()

{dsgrid_toolkit-0.2.0 → dsgrid_toolkit-0.3.1}/dsgrid/cli/config.py

@@ -40,7 +40,7 @@ $ dsgrid config create sqlite:////projects/dsgrid/registries/standard-scenarios/
     "-b",
     "--backend-engine",
     type=click.Choice([x.value for x in BackendEngine]),
-    default=DEFAULT_BACKEND,
+    default=DEFAULT_BACKEND.value,
     help="Backend engine for SQL processing",
 )
 @click.option(
@@ -91,7 +91,6 @@ $ dsgrid config create sqlite:////projects/dsgrid/registries/standard-scenarios/
 # )
 # @click.option(
 #    "-o",
-#    "--offline",
 #    is_flag=True,
 #    default=False,
 #    show_default=True,

{dsgrid_toolkit-0.2.0 → dsgrid_toolkit-0.3.1}/dsgrid/cli/dsgrid.py

@@ -6,6 +6,10 @@ from pathlib import Path
 import rich_click as click
 
 import dsgrid
+from chronify.utils.path_utils import check_overwrite
+from dsgrid.common import LOCAL_REGISTRY
+from dsgrid.registry.common import DatabaseConnection, DataStoreType
+from dsgrid.registry.registry_manager import RegistryManager
 from dsgrid.utils.timing import timer_stats_collector
 from dsgrid.cli.common import get_log_level_from_str, handle_scratch_dir
 from dsgrid.cli.config import config
@@ -38,15 +42,6 @@ logger = logging.getLogger(__name__)
 @click.option(
     "-n", "--no-prompts", default=False, is_flag=True, show_default=True, help="Do not prompt."
 )
-@click.option(
-    "--offline/--online",
-    is_flag=True,
-    default=dsgrid.runtime_config.offline,
-    show_default=True,
-    help="Run registry commands in offline mode. WARNING: any commands you perform in offline "
-    "mode run the risk of being out-of-sync with the latest dsgrid registry, and any write "
-    "commands will not be officially synced with the remote registry",
-)
 @click.option(
     "--timings/--no-timings",
     default=dsgrid.runtime_config.timings,
@@ -54,24 +49,6 @@ logger = logging.getLogger(__name__)
     show_default=True,
     help="Enable tracking of function timings.",
 )
-# Server-related options are commented-out because the registry is currently only
-# supported in SQLite. If/when we add postgres support, these can be added back.
-# @click.option(
-#    "-U",
-#    "--username",
-#    type=str,
-#    default=dsgrid.runtime_config.database_user,
-#    help="Database username",
-# )
-# @click.option(
-#    "-P",
-#    "--password",
-#    prompt=True,
-#    hide_input=True,
-#    cls=OptionPromptPassword,
-#    help="dsgrid registry password. Will prompt unless it is passed or the username matches the "
-#    "runtime config file.",
-# )
 @click.option(
     "-u",
     "--url",
@@ -103,10 +80,7 @@ def cli(
     file_level,
     log_file,
     no_prompts,
-    offline,
     timings,
-    # username,
-    # password,
     url,
     reraise_exceptions,
     scratch_dir,
@@ -135,7 +109,48 @@ def callback(*args, **kwargs):
         timer_stats_collector.log_stats()
 
 
+_create_registry_epilog = """
+Examples:\n
+$ dsgrid create-registry sqlite:////projects/dsgrid/my_project/registry.db -p /projects/dsgrid/my_project/registry-data\n
+"""
+
+
+@click.command(name="create-registry", epilog=_create_registry_epilog)
+@click.argument("url")
+@click.option(
+    "-p",
+    "--data-path",
+    default=LOCAL_REGISTRY,
+    show_default=True,
+    callback=lambda *x: Path(x[2]),
+    help="Local dsgrid registry data path. Must not contain the registry file listed in URL.",
+)
+@click.option(
+    "-f",
+    "--overwrite",
+    "--force",
+    is_flag=True,
+    default=False,
+    help="Delete registry_path and the database if they already exist.",
+)
+@click.option(
+    "-t",
+    "--data-store-type",
+    type=click.Choice([x.value for x in DataStoreType]),
+    default=DataStoreType.FILESYSTEM.value,
+    show_default=True,
+    help="Type of store to use for the registry data.",
+    callback=lambda *x: DataStoreType(x[2]),
+)
+def create_registry(url: str, data_path: Path, overwrite: bool, data_store_type: DataStoreType):
+    """Create a new registry."""
+    check_overwrite(data_path, overwrite)
+    conn = DatabaseConnection(url=url)
+    RegistryManager.create(conn, data_path, overwrite=overwrite, data_store_type=data_store_type)
+
+
 cli.add_command(config)
+cli.add_command(create_registry)
 cli.add_command(download)
 cli.add_command(install_notebooks)
 cli.add_command(query)

dsgrid_toolkit-0.3.1/dsgrid/cli/dsgrid_admin.py

@@ -0,0 +1,92 @@
+"""CLI for dsgrid admin commands (testing purposes only)."""
+
+import logging
+from pathlib import Path
+
+import rich_click as click
+
+from dsgrid.config.simple_models import RegistrySimpleModel
+from dsgrid.dsgrid_rc import DsgridRuntimeConfig
+from dsgrid.loggers import setup_logging, check_log_file_size
+from dsgrid.registry.common import DatabaseConnection
+from dsgrid.registry.filter_registry_manager import FilterRegistryManager
+from dsgrid.registry.registry_manager import RegistryManager
+from dsgrid.utils.files import load_data
+
+
+logger = logging.getLogger(__name__)
+_config = DsgridRuntimeConfig.load()
+
+
+@click.group()
+@click.option("-l", "--log-file", default="dsgrid_admin.log", type=str, help="Log to this file.")
+@click.option(
+    "-n", "--no-prompts", default=False, is_flag=True, show_default=True, help="Do not prompt."
+)
+@click.option(
+    "--verbose", is_flag=True, default=False, show_default=True, help="Enable verbose log output."
+)
+def cli(log_file, no_prompts, verbose):
+    """dsgrid-admin commands (for testing purposes only)"""
+    path = Path(log_file)
+    level = logging.DEBUG if verbose else logging.INFO
+    check_log_file_size(path, no_prompts=no_prompts)
+    setup_logging("dsgrid", path, console_level=level, file_level=level, mode="a")
+
+
+@click.command()
+@click.option(
+    "--src-database-url",
+    required=True,
+    help="Source dsgrid registry database URL.",
+)
+@click.option(
+    "--dst-database-url",
+    default="dsgrid",
+    required=True,
+    help="Destination dsgrid registry database URL.",
+)
+@click.argument("dst_data_path", type=click.Path(exists=False), callback=lambda *x: Path(x[2]))
+@click.argument("config_file", type=click.Path(exists=True), callback=lambda *x: Path(x[2]))
+@click.option(
+    "-m",
+    "--mode",
+    default="data-symlinks",
+    type=click.Choice(["copy", "data-symlinks", "rsync"]),
+    show_default=True,
+    help="Controls whether to copy all data, make symlinks to data files, or sync data with the "
+    "rsync utility (not available on Windows).",
+)
+@click.option(
+    "-f",
+    "--overwrite",
+    "--force",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Overwrite dst_registry_path if it already exists. Does not apply if using rsync.",
+)
+def make_filtered_registry(
+    src_database_url,
+    dst_database_url,
+    dst_data_path: Path,
+    config_file: Path,
+    mode,
+    overwrite,
+):
+    """Make a filtered registry for testing purposes."""
+    simple_model = RegistrySimpleModel(**load_data(config_file))
+    src_conn = DatabaseConnection(url=src_database_url)
+    dst_conn = DatabaseConnection(url=dst_database_url)
+    RegistryManager.copy(
+        src_conn,
+        dst_conn,
+        dst_data_path,
+        mode=mode,
+        force=overwrite,
+    )
+    with FilterRegistryManager.load(dst_conn, offline_mode=True, use_remote_data=False) as mgr:
+        mgr.filter(simple_model=simple_model)
+
+
+cli.add_command(make_filtered_registry)