anta-database 0.4.2__tar.gz → 0.5.0__tar.gz

{anta_database-0.4.2 → anta_database-0.5.0}/PKG-INFO

@@ -1,9 +1,8 @@
 Metadata-Version: 2.4
 Name: anta_database
-Version: 0.4.2
+Version: 0.5.0
 Summary: SQLite database for the AntArchitecture Community Data
-Author: Antoine Hermant
-Author-email: antoine.hermant@etik.com
+Author-email: Antoine Hermant <antoine.hermant@etik.com>
 License-Expression: MIT
 Project-URL: Homepage, https://antoinehermant.github.io/anta_database/intro
 Project-URL: Documentation, https://antoinehermant.github.io/anta_database/quick_start
@@ -17,19 +16,29 @@ Classifier: Programming Language :: Python
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: numpy
 Requires-Dist: pandas
-Requires-Dist: geopandas
-Requires-Dist: fastparquet
 Requires-Dist: pyarrow
 Requires-Dist: xarray
 Requires-Dist: h5netcdf
 Requires-Dist: h5py
-Requires-Dist: pyproj
-Requires-Dist: scipy
 Requires-Dist: matplotlib
 Requires-Dist: colormaps
 Requires-Dist: tqdm
-Requires-Dist: pyshp
+Requires-Dist: shapely
+Provides-Extra: compile
+Requires-Dist: geopandas; extra == "compile"
+Requires-Dist: pyproj; extra == "compile"
+Requires-Dist: scipy; extra == "compile"
+Requires-Dist: netcdf4; extra == "compile"
+Requires-Dist: zarr; extra == "compile"
+Provides-Extra: pism
+Requires-Dist: scipy; extra == "pism"
+Requires-Dist: netcdf4; extra == "pism"
+Provides-Extra: publish
+Requires-Dist: boto3>=1.36; extra == "publish"
+Provides-Extra: all
+Requires-Dist: anta_database[compile,pism,publish]; extra == "all"
 Dynamic: license-file
 
 <h1 align="center">
@@ -74,18 +83,37 @@ Each dataset contains (when available):
 pip install anta_database
 ```
 
+This installs what is needed to query and plot the database. Optional extras:
+```bash
+pip install "anta_database[compile]"  # compile the database from raw data
+pip install "anta_database[pism]"     # plot comparisons with PISM outputs
+pip install "anta_database[all]"      # everything
+```
+
 For the latest development version:
 ```bash
 pip install git+https://github.com/antoinehermant/anta_database.git
 ```
 
+## Downloading the data
+
+Download the database, or update it to the latest published version (only new or changed files are downloaded):
+```python
+from anta_database import sync
+sync()                            # default local directory
+sync("/path/to/AntADatabase/")    # or a directory of your choice
+```
+or from the command line: `anta-sync --dir /path/to/AntADatabase/`. From an existing `Database`, use `db.sync()`, or `Database(sync=True)` to always sync when loading.
+
+When loading a synced database, `Database` tells you if a newer version is published (disable with `check_updates=False`). Updates are never downloaded without you asking, so your results only change when you decide to sync.
+
 ## Quick Start
 
 ```python
 from anta_database import Database
 
-# Initialize database (contact maintainer for data access)
-db = Database("/path/to/AntADatabase/")
+# Load the database (without a path, uses the default directory of sync())
+db = Database()
 
 # Query data by various criteria
 db.query(dataset="Cavitte_2020", age=10000)
@@ -100,7 +128,7 @@ Full documentation and examples are available on the [Home Page](https://antoine
 
 ## Data Access
 
-The database files are not publicly hosted. Please contact [antoine.hermant@unibe.ch](mailto:antoine.hermant@unibe.ch) to request access to the data.
+The AntADatabase is free to download with `sync()` (see [Downloading the data](#downloading-the-data)). Any data used must be cited at source using the DOIs provided in the metadata.
 
 ## Support
 

{anta_database-0.4.2 → anta_database-0.5.0}/README.md

@@ -40,18 +40,37 @@ Each dataset contains (when available):
 pip install anta_database
 ```
 
+This installs what is needed to query and plot the database. Optional extras:
+```bash
+pip install "anta_database[compile]"  # compile the database from raw data
+pip install "anta_database[pism]"     # plot comparisons with PISM outputs
+pip install "anta_database[all]"      # everything
+```
+
 For the latest development version:
 ```bash
 pip install git+https://github.com/antoinehermant/anta_database.git
 ```
 
+## Downloading the data
+
+Download the database, or update it to the latest published version (only new or changed files are downloaded):
+```python
+from anta_database import sync
+sync()                            # default local directory
+sync("/path/to/AntADatabase/")    # or a directory of your choice
+```
+or from the command line: `anta-sync --dir /path/to/AntADatabase/`. From an existing `Database`, use `db.sync()`, or `Database(sync=True)` to always sync when loading.
+
+When loading a synced database, `Database` tells you if a newer version is published (disable with `check_updates=False`). Updates are never downloaded without you asking, so your results only change when you decide to sync.
+
 ## Quick Start
 
 ```python
 from anta_database import Database
 
-# Initialize database (contact maintainer for data access)
-db = Database("/path/to/AntADatabase/")
+# Load the database (without a path, uses the default directory of sync())
+db = Database()
 
 # Query data by various criteria
 db.query(dataset="Cavitte_2020", age=10000)
@@ -66,7 +85,7 @@ Full documentation and examples are available on the [Home Page](https://antoine
 
 ## Data Access
 
-The database files are not publicly hosted. Please contact [antoine.hermant@unibe.ch](mailto:antoine.hermant@unibe.ch) to request access to the data.
+The AntADatabase is free to download with `sync()` (see [Downloading the data](#downloading-the-data)). Any data used must be cited at source using the DOIs provided in the metadata.
 
 ## Support
 

anta_database-0.5.0/anta_database/__init__.py

@@ -0,0 +1,68 @@
+"""
+Anta Database Package
+
+Main package for isochrone database access and processing.
+
+The query, indexing and plotting tools only need the core dependencies.
+Compiling and PISM plotting need optional extras and are imported lazily:
+    pip install "anta_database[compile]"
+    pip install "anta_database[pism]"
+
+Download or update the published database with anta_database.sync().
+"""
+
+from typing import TYPE_CHECKING
+
+# Database modules
+from anta_database.database.database import Database, MetadataResult
+
+# Indexing modules
+from anta_database.indexing.index_database import IndexDatabase
+
+# Plotting modules
+from anta_database.plotting.plotting import Plotting
+
+# Downloading and updating the published database
+from anta_database.cloud.sync import sync
+
+if TYPE_CHECKING:
+    from anta_database.compiling.zarr_converter_simple import FlightLineZarrConverter
+    from anta_database.compiling.compile import CompileDatabase
+    from anta_database.plotting.plotting_pism import PISMPlotting
+
+# Optional classes, imported on first access
+_LAZY_IMPORTS = {
+    "FlightLineZarrConverter": "anta_database.compiling",
+    "CompileDatabase": "anta_database.compiling",
+    "PISMPlotting": "anta_database.plotting.plotting_pism",
+}
+
+
+def __getattr__(name):
+    if name in _LAZY_IMPORTS:
+        import importlib
+
+        module = importlib.import_module(_LAZY_IMPORTS[name])
+        return getattr(module, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__():
+    return sorted(list(globals()) + list(_LAZY_IMPORTS))
+
+
+__all__ = [
+    # Database classes
+    "Database",
+    "MetadataResult",
+    # Compiling classes
+    "FlightLineZarrConverter",
+    "CompileDatabase",
+    # Indexing functions
+    "IndexDatabase",
+    # Plotting functions
+    "Plotting",
+    "PISMPlotting",
+    # Cloud functions
+    "sync",
+]

anta_database-0.5.0/anta_database/cloud/__init__.py

@@ -0,0 +1,24 @@
+"""
+Cloud distribution of the AntADatabase.
+
+sync() only needs the standard library. publish() is for maintainers and needs
+the [publish] extra: pip install "anta_database[publish]"
+"""
+
+from typing import TYPE_CHECKING
+
+from anta_database.cloud.sync import default_database_dir, sync
+
+if TYPE_CHECKING:
+    from anta_database.cloud.publish import publish
+
+
+def __getattr__(name):
+    if name == "publish":
+        from anta_database.cloud.publish import publish
+
+        return publish
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = ["sync", "publish", "default_database_dir"]

anta_database-0.5.0/anta_database/cloud/manifest.py

@@ -0,0 +1,98 @@
+"""
+Manifest shared by publish (maintainer) and sync (users).
+
+A manifest lists every file of a published database with its checksum:
+    {
+        "format": 1,
+        "db_version": "2026.10.27-1530",
+        "created": "2026-10-27T15:30:00+00:00",
+        "files": {"AntADatabase.db": {"sha256": "...", "size": 123}, ...}
+    }
+Paths are relative to the database directory and always use "/".
+"""
+
+import glob
+import hashlib
+import json
+import os
+from datetime import datetime, timezone
+from typing import Dict, Optional
+
+MANIFEST_NAME = "manifest.json"
+MANIFEST_FORMAT = 1
+DB_FILE = "AntADatabase.db"
+
+# Files of a database directory that are published, relative to the directory.
+# Raw input data and temporary compile files are not published.
+PUBLISHED_PATTERNS = [
+    DB_FILE,
+    "*/raw_md.json",
+    "*/h5/*.h5",
+]
+
+
+def sha256sum(path: str, chunk_size: int = 1 << 20) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(chunk_size), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def list_published_files(database_dir: str) -> list:
+    """Relative paths of all files to publish from a local database directory."""
+    files = set()
+    for pattern in PUBLISHED_PATTERNS:
+        for path in glob.glob(os.path.join(database_dir, pattern)):
+            if os.path.isfile(path):
+                rel = os.path.relpath(path, database_dir)
+                files.add(rel.replace(os.sep, "/"))
+    return sorted(files)
+
+
+def build_manifest(database_dir: str, db_version: Optional[str] = None) -> Dict:
+    """Hash all published files of a local database directory."""
+    from tqdm import tqdm
+
+    if not os.path.isfile(os.path.join(database_dir, DB_FILE)):
+        raise FileNotFoundError(f"{DB_FILE} not found in {database_dir}")
+
+    now = datetime.now(timezone.utc)
+    files = {}
+    for rel in tqdm(list_published_files(database_dir), desc="Hashing", unit="file"):
+        path = os.path.join(database_dir, rel)
+        files[rel] = {"sha256": sha256sum(path), "size": os.path.getsize(path)}
+
+    return {
+        "format": MANIFEST_FORMAT,
+        "db_version": db_version or now.strftime("%Y.%m.%d-%H%M"),
+        "created": now.isoformat(timespec="seconds"),
+        "files": files,
+    }
+
+
+def check_manifest(manifest: Dict) -> Dict:
+    if manifest.get("format") != MANIFEST_FORMAT:
+        raise ValueError(
+            f"Unsupported manifest format {manifest.get('format')!r}. "
+            "Try upgrading anta_database: pip install -U anta_database"
+        )
+    for rel in manifest["files"]:
+        # Never let a manifest write outside the database directory
+        if rel.startswith("/") or ".." in rel.split("/") or "\\" in rel:
+            raise ValueError(f"Invalid path in manifest: {rel!r}")
+    return manifest
+
+
+def read_manifest(path: str) -> Optional[Dict]:
+    if not os.path.isfile(path):
+        return None
+    with open(path) as f:
+        return json.load(f)
+
+
+def write_manifest(manifest: Dict, path: str) -> None:
+    tmp = f"{path}.part"
+    with open(tmp, "w") as f:
+        json.dump(manifest, f, indent=1, sort_keys=True)
+    os.replace(tmp, path)

anta_database-0.5.0/anta_database/cloud/publish.py

@@ -0,0 +1,165 @@
+"""
+Publish a local AntADatabase to an S3-compatible bucket (maintainers only).
+
+Needs the [publish] extra: pip install "anta_database[publish]"
+
+Credentials are read by boto3 from the environment (AWS_ACCESS_KEY_ID,
+AWS_SECRET_ACCESS_KEY) or from ~/.aws/credentials, never from the package.
+The bucket must be publicly readable for sync() to work. For Infomaniak
+Public Cloud, the S3 endpoint is https://s3.pub1.infomaniak.cloud.
+
+    anta-publish /path/to/AntADatabase --bucket antadatabase --prefix v1
+"""
+
+import argparse
+import json
+import mimetypes
+import os
+from typing import Dict, Optional
+
+from tqdm import tqdm
+
+from anta_database.cloud.manifest import MANIFEST_NAME, build_manifest, check_manifest
+
+try:
+    import boto3
+    from botocore.config import Config
+    from botocore.exceptions import ClientError
+except ImportError as e:
+    raise ImportError(
+        "Publishing the AntADatabase needs optional dependencies. "
+        'Install them with: pip install "anta_database[publish]"'
+    ) from e
+
+DEFAULT_ENDPOINT = "https://s3.pub1.infomaniak.cloud"
+
+
+def _key(prefix: str, rel: str) -> str:
+    return f"{prefix.strip('/')}/{rel}" if prefix.strip("/") else rel
+
+
+def _get_remote_manifest(s3, bucket: str, prefix: str) -> Optional[Dict]:
+    try:
+        obj = s3.get_object(Bucket=bucket, Key=_key(prefix, MANIFEST_NAME))
+    except ClientError as e:
+        if e.response["Error"]["Code"] in ("NoSuchKey", "404"):
+            return None
+        raise
+    return check_manifest(json.load(obj["Body"]))
+
+
+def publish(
+    database_dir: str,
+    bucket: str,
+    prefix: str = "v1",
+    endpoint_url: Optional[str] = None,
+    db_version: Optional[str] = None,
+    dry_run: bool = False,
+    delete: bool = True,
+) -> Dict:
+    """
+    Upload new and changed files of a local database, then the manifest.
+
+    The manifest is uploaded after all data files, so users never sync a
+    manifest that points to missing files. A user syncing during a publish may
+    hit a checksum error on a file being replaced and simply has to sync again.
+    Files that are no longer part of the database are deleted afterwards.
+
+    Args:
+        database_dir: Local database directory (with AntADatabase.db).
+        bucket: Bucket name.
+        prefix: Key prefix inside the bucket, e.g. "v1".
+        endpoint_url: S3 endpoint. Defaults to $AWS_ENDPOINT_URL or Infomaniak.
+        db_version: Version label stored in the manifest. Defaults to the UTC date and time.
+        dry_run: Only report what would be uploaded and deleted.
+        delete: Delete objects that were published before but are no longer in the database.
+
+    Returns:
+        A dict with "db_version" and the lists "uploaded" and "deleted".
+    """
+    s3 = boto3.client(
+        "s3",
+        endpoint_url=endpoint_url or os.environ.get("AWS_ENDPOINT_URL", DEFAULT_ENDPOINT),
+        # Swift-based S3 APIs (Infomaniak) reject the default aws-chunked checksummed uploads
+        config=Config(
+            request_checksum_calculation="when_required",
+            response_checksum_validation="when_required",
+        ),
+    )
+
+    manifest = build_manifest(database_dir, db_version)
+    remote = _get_remote_manifest(s3, bucket, prefix) or {"files": {}}
+
+    to_upload = [
+        rel
+        for rel, entry in manifest["files"].items()
+        if remote["files"].get(rel, {}).get("sha256") != entry["sha256"]
+    ]
+    to_delete = sorted(set(remote["files"]) - set(manifest["files"])) if delete else []
+    result = {"db_version": manifest["db_version"], "uploaded": to_upload, "deleted": to_delete}
+    if dry_run:
+        return result
+
+    total = sum(manifest["files"][rel]["size"] for rel in to_upload)
+    with tqdm(total=total, unit="B", unit_scale=True, desc="Uploading") as progress:
+        for rel in to_upload:
+            content_type = mimetypes.guess_type(rel)[0] or "application/octet-stream"
+            s3.upload_file(
+                os.path.join(database_dir, rel),
+                bucket,
+                _key(prefix, rel),
+                ExtraArgs={"ContentType": content_type},
+                Callback=progress.update,
+            )
+
+    # Clients must never fetch a stale manifest from a cache
+    s3.put_object(
+        Bucket=bucket,
+        Key=_key(prefix, MANIFEST_NAME),
+        Body=json.dumps(manifest, indent=1, sort_keys=True).encode(),
+        ContentType="application/json",
+        CacheControl="no-cache",
+    )
+
+    for rel in to_delete:
+        s3.delete_object(Bucket=bucket, Key=_key(prefix, rel))
+
+    print(
+        f"Published AntADatabase {manifest['db_version']} to {bucket}/{prefix}: "
+        f"{len(to_upload)} file(s) uploaded, {len(to_delete)} deleted."
+    )
+    return result
+
+
+def main(argv=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog="anta-publish", description="Publish a local AntADatabase to an S3 bucket."
+    )
+    parser.add_argument("database_dir", help="Local database directory")
+    parser.add_argument("--bucket", required=True)
+    parser.add_argument("--prefix", default="v1")
+    parser.add_argument("--endpoint-url", help=f"S3 endpoint (default: $AWS_ENDPOINT_URL or {DEFAULT_ENDPOINT})")
+    parser.add_argument("--db-version", help="Version label (default: UTC date and time)")
+    parser.add_argument("--dry-run", action="store_true", help="Only show what would change")
+    parser.add_argument("--no-delete", action="store_true", help="Keep objects that are no longer in the database")
+    args = parser.parse_args(argv)
+
+    result = publish(
+        args.database_dir,
+        args.bucket,
+        prefix=args.prefix,
+        endpoint_url=args.endpoint_url,
+        db_version=args.db_version,
+        dry_run=args.dry_run,
+        delete=not args.no_delete,
+    )
+    if args.dry_run:
+        print(f"Would upload {len(result['uploaded'])} file(s), delete {len(result['deleted'])}.")
+        for rel in result["uploaded"]:
+            print(f"  + {rel}")
+        for rel in result["deleted"]:
+            print(f"  - {rel}")
+
+
+if __name__ == "__main__":
+    main()