mash-core 0.9.0__py3-none-any.whl

mash/__init__.py

@@ -0,0 +1,426 @@
+_SHAPE_NS = "urn:graph:shapes"
+_ONTO_NS = "urn:graph:ontology"
+
+
+class _STATIC:
+    maplib_compatible_mimetypes = [
+        "application/n-triples",
+        "text/turtle",
+        "application/rdf+xml",
+    ]
+
+
+import atexit
+import os
+import shutil
+from enum import StrEnum
+from pathlib import Path
+from tempfile import TemporaryDirectory, mkdtemp
+from typing import Literal
+from warnings import warn
+
+from cachetools import LRUCache
+from maplib import Model
+
+from mash.core.config import Config
+from mash.core.logging import color_logger
+from mash.repositories.mesh.mesh_repository import MeshRepository
+from mash.repositories.mesh.repository_collection import RepositoryCollection
+from mash.repositories.namespace.namespace import (
+    DictBackedNamespaceRepository,
+    NamespaceRepository,
+)
+
+logger = color_logger()
+
+
+class _MashImpl:
+    namespaces: NamespaceRepository
+    config: Config
+
+    _static_repo_collection: RepositoryCollection
+    _static_repo_view: MeshRepository
+    _live_repo_view: MeshRepository
+    _live_repo_collection: RepositoryCollection
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.config = Config()
+        self._static_repo_collection = RepositoryCollection()
+        self._live_repo_collection = RepositoryCollection()
+        self.reset_temp_directory()
+        self.namespaces = DictBackedNamespaceRepository()
+
+    def repositories_ordered(self, default=True, non_default=True) -> list[MeshRepository]:
+        pass
+
+    @property
+    def active_repository(self) -> MeshRepository:
+        return self._live_repo_collection.active_repository()
+
+    def reset_temp_directory(self):
+        if hasattr(self, "_tmpdir_cls") and isinstance(self._tmpdir_cls, TemporaryDirectory):
+            self._tmpdir_cls.cleanup()
+
+        tempdir = self.config.data_storage_directory / "temp"
+        if not tempdir.exists():
+            tempdir.mkdir(parents=True)
+        self._tmpdir_cls = TemporaryDirectory(
+            "mash",
+            delete=False,
+            dir=tempdir,
+        )
+        self._tmpdir = self._tmpdir_cls.name
+
+    def static_repository(self):
+        return self._static_repo_view
+
+    def add_repository(self, key, repository: MeshRepository):
+        self._live_repo_collection[key] = repository
+
+    def remove_repository(self, key):
+        self._live_repo_collection.pop(key)
+
+
+_mash = _MashImpl()
+
+
+def _get_impl() -> _MashImpl:
+    return _mash
+
+
+from mash.core.metadata import models
+from mash.repositories.models import ModelRegistry, find_semantic_models_in_module
+
+_model_registry = ModelRegistry()
+
+
+def clean_up_tempdir():
+    _mash._tmpdir_cls.cleanup()
+
+
+atexit.register(clean_up_tempdir)
+
+
+from mash.core.metadata.models import SemanticModel
+from mash.core.models import IRI, Predicate, ReverseRelation
+from mash.orm import instantiate, serialize
+
+# Repository stuff below, could make into optional package
+
+
+def get_active_repository() -> MeshRepository:
+    return _mash.active_repository
+
+
+def _where_is_it(arg: models.SemanticModel, exclude: list):
+    for repository in _mash._live_repo_collection.values():
+        if repository in exclude:
+            continue
+        res = repository.has_data(arg.identifier)
+        if res:
+            return repository
+
+
+def repository_add(key: str, repository: MeshRepository, ignore_if_exists=False):
+    if ignore_if_exists and key in _mash._live_repo_collection:
+        return False
+
+    _mash.add_repository(key, repository)
+    return True
+
+
+def repository_remove(key: str):
+    _mash.remove_repository(key)
+
+
+def _ActiveRepository() -> None:
+    """Vanguard statement to represent the active repostory"""
+    return None
+
+
+def write_data(data: bytes, resource: models.Distribution | str, repo=_ActiveRepository()):
+    if repo is None:
+        repo = _mash.active_repository
+
+    with repo.open(resource, mode="wb") as file:
+        file.write(data)
+
+
+def copy(
+    instances: list[models.SemanticModel] | models.SemanticModel,
+    dest: MeshRepository,
+    source: MeshRepository | Literal["any"] = _ActiveRepository(),
+    filter_mimetype=None,
+):
+    """Copies all metadata and data-data contained in a set of instances from one repository to another"""
+    if not isinstance(instances, list):
+        instances = [instances]
+
+    if source is None:
+        source = _mash.active_repository
+
+    # 1. Copy metadata
+    dest.update_metadata(*instances)
+
+    # 2. Copy data-data
+    for instance in instances:
+        is_distribution = isinstance(instance, models.Distribution) or issubclass(instance.__class__, models.Distribution)
+
+        if source == "any":
+            if isinstance(instance, models.Dataset):
+                _dist = instance.distributions[0]
+            else:
+                _dist = instance
+            source_repository = _where_is_it(_dist, exclude=[dest])
+
+        else:
+            source_repository = source
+
+        if source_repository is None:
+            # This means the object doesn't exist anywhere yet, so there is no data to copy
+            return
+
+        if is_distribution:
+            _copy_dist_data(instance, source_repository, dest)
+            return
+
+        elif isinstance(instance, models.Dataset):
+            for dist in instance.distributions:
+                if filter_mimetype and dist.mime_type != filter_mimetype:
+                    continue
+                _copy_dist_data(dist, source_repository, dest)
+
+
+def _copy_dist_data(distribution: models.Distribution, source_repo: MeshRepository, dest_repo: MeshRepository):
+    try:
+        with source_repo.open(distribution) as source_fp:
+            with dest_repo.open(distribution, "wb") as dest_fp:
+                shutil.copyfileobj(source_fp, dest_fp)
+    except FileNotFoundError:
+        logger.warning("No data found for distribution")
+
+
+def save(*args: models.SemanticModel, repo: MeshRepository = _ActiveRepository()):
+    """Updates or saves a model in active storage"""
+    if repo is None:
+        repo = _mash.active_repository
+    repo.update_metadata(*args)
+
+
+def fetch(identifier: str, repo: MeshRepository | Literal["any"] = _ActiveRepository()) -> models.SemanticModel | None:
+    """Fetches a resource from any repository"""
+    if repo is None:
+        return _mash._live_repo_view.fetch(identifier, repo="active")
+
+    if repo == "any":
+        return _mash._live_repo_view.fetch(identifier, repo="any")
+
+    return repo.fetch(identifier)
+
+
+def safe_fetch(identifier: str, repo: MeshRepository | Literal["any"] = _ActiveRepository()) -> models.SemanticModel | None:
+    """Fetches a resource from any repository, returning None on error"""
+    try:
+        return fetch(identifier, repo)
+    except Exception as e:
+        logger.warning(f"Failed to fetch {identifier} from repository {repo}: {e}")
+        return None
+
+
+def delete(*args: models.SemanticModel, repo=_ActiveRepository()):
+    """Deletes metadata and data attached to models from temporary and local folders"""
+    if repo is None:
+        repo = _mash.active_repository
+
+    repo.delete(*args)
+
+
+def metadata_graph(format="ntriples") -> Model:
+    """Fetch metadata from all repositories and return a common RDf graph with all of it"""
+    return _mash._live_repo_view.metadata_graph()
+
+
+class data:
+    def repositories_with_data(distribution_id) -> list[MeshRepository]:
+        live_repos = _mash._live_repo_collection.values()
+        return [r for r in live_repos if r.has_data(distribution_id)]
+
+    def exists_locally(distribution_id) -> bool:
+        return known_repository("local").has_data(distribution_id)
+
+
+from mash.core.utilities.dataset import distribution_with_mimetype
+
+
+def read_data(
+    source: models.Distribution | models.Dataset | str, mime_type: str = None, repo: MeshRepository | Literal["any"] = _ActiveRepository()
+) -> bytes:
+    """Fetches data from any repository"""
+    if repo is None:
+        repo = _mash.active_repository
+
+    # Resolve distribution
+    if isinstance(source, str):
+        source = fetch(source, repo)
+        if source is None:
+            raise FileNotFoundError("The distribution was not found in provided repo")
+
+    if isinstance(source, models.Dataset):
+        if mime_type is None:
+            raise ValueError("When trying to fetch data from a dataset, the attribute mime_type is required")
+
+        source = distribution_with_mimetype(source, mime_type)
+        if source is None:
+            raise ValueError("No distribution found with provided mime_type")
+
+    distribution_id = source.identifier
+
+    # Fetch data
+    if repo != "any":
+        with repo.open(distribution_id) as file:
+            return file.read()
+
+    with _mash._live_repo_view.open(distribution_id, mode="rb", repo="any") as file:
+        return file.read()
+
+
+from mash.repositories.mesh.filesystem_mesh_repository import FilesystemMeshRepository
+from mash.repositories.mesh.multi_repository_view import MultiRepositoryView
+
+
+def add_local_repository(key, ignore_if_exists=False):
+    path: Path = _mash.config.data_storage_directory / key
+    path.mkdir(exist_ok=True)
+    if repository_add(
+        key,
+        FilesystemMeshRepository(
+            path,
+        ),
+        ignore_if_exists=ignore_if_exists,
+    ):
+        logger.info(f"Added local repository '{key}' at {path}")
+    else:
+        logger.info(f"Repository with key '{key}' already exists")
+
+
+def _setup_default_repositories():
+    _mash._static_repo_view = MultiRepositoryView(_mash._static_repo_collection, read_only=True)
+    _mash._live_repo_collection["static"] = _mash._static_repo_view
+    _mash._live_repo_view = MultiRepositoryView(_mash._live_repo_collection)
+    _mash.reset_temp_directory()
+    temp_repo = FilesystemMeshRepository(Path(_mash._tmpdir))
+    _mash.add_repository("temp", temp_repo)
+    _mash._live_repo_collection.activate_layer(temp_repo)
+
+    storage_dir = _mash.config.data_storage_directory
+    if not storage_dir.exists():
+        storage_dir.mkdir()
+
+    _mash.add_repository(
+        "local",
+        FilesystemMeshRepository(
+            _mash.config.data_storage_directory,
+        ),
+    )
+
+
+from mash.repositories.mesh.utilities import load_configured_blob_repositories
+
+_setup_default_repositories()
+load_configured_blob_repositories()
+
+
+class _UseRepositoryImpl:
+    """
+    Activates a repository as the default one,
+    works as both a context manager and a function.
+    """
+
+    def __init__(self, repo: MeshRepository | str):
+        if isinstance(repo, str):
+            repo = _mash._live_repo_collection[repo]
+
+        _mash._live_repo_collection.activate_layer(repo)
+
+    def __enter__(self):
+        """Doesn't need to do anything"""
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        _mash._live_repo_collection.pop_layer()
+
+
+use_repository = _UseRepositoryImpl
+
+
+def known_repository(kind: Literal["temp", "local", "static", "default"]) -> MeshRepository:
+    if kind == "default":
+        return _mash.active_repository
+
+    return _mash._live_repo_collection[kind]
+
+
+from mash.core.maplib import model_from_dataset, serialize_model_to_dataset
+from mash.core.validation import validate_dataset
+from mash.orm.query import query, strict_query
+
+
+def ns():
+    return _mash.namespaces
+
+
+from mash.core.profiles import MashProfile, MetadataProfile
+
+
+def globally_use_profile(profile: MashProfile):
+    _mash.namespaces.add_layer(profile.namespaces)
+    _model_registry.add_layer(profile.models)
+    if profile.static_resources is not None:
+        _mash._static_repo_collection[profile.identifier] = profile.static_resources
+
+
+globally_use_profile(MetadataProfile)
+
+
+def get_repository(key: str) -> MeshRepository | None:
+    return _mash._live_repo_collection.get(key)
+
+
+K3_DIRECTORY = os.getenv("K3_DIRECTORY", None)
+
+
+def k3(filename) -> Path:
+    """This is a utility to make it easy to write and read files from K3_DIRECTORY
+    rather than the working directory
+    """
+    try:
+        k3_directory = Path(os.environ["K3_DIRECTORY"])
+        assert k3_directory.exists(), "The K3_DIRECTORY pointed to must exist"
+        work_dir = k3_directory
+        if not work_dir.exists():
+            os.mkdir(work_dir)
+            logger.info("Created directory " + str(work_dir))
+    except KeyError:
+        raise ValueError("Your must have the environment K3_DIRECTORY set pointing to a safe folder for storage of sensitive data")
+
+    return work_dir / filename
+
+
+import importlib
+
+for extension in _mash.config.extensions_enabled:
+    importlib.import_module(extension)
+
+
+def _reset():
+    clean_up_tempdir()
+    _mash.reset()
+    _model_registry.reset()
+    _setup_default_repositories()
+    load_configured_blob_repositories()
+    globally_use_profile(MetadataProfile)
+    for extension in _mash.config.extensions_enabled:
+        _mod = importlib.import_module(extension)
+        _mod.module_enable()

mash/core/catalog/__init__.py

@@ -0,0 +1 @@
+from .api import app

mash/core/catalog/api.py

@@ -0,0 +1,76 @@
+import os
+import webbrowser
+from contextlib import asynccontextmanager
+
+from cachetools import LRUCache, cached
+from fastapi import APIRouter, FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+
+import mash
+from mash import models
+from mash.core.utilities.user import get_current_agent
+from mash.orm import filters
+
+HOST = ""
+DATASET_PREFIX = "/dataset"
+GRID_MODEL_PREFIX = "/models"
+SERVICE_PREFIX = "/service"
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    if os.getenv("DEV_WP") == "true":
+        webbrowser.open_new_tab("http://localhost:8234/catalog")
+    yield
+
+
+app = FastAPI(title="Mash API", lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+catalog = models.Catalog(
+    identifier="urn:mash:local_catalog",
+    title="Mash data catalog",
+    description="An automatically generated catalog of resources found in Mash",
+    publisher=get_current_agent(),
+    contact=models.ContactGroup(identifier="mail@example.com", name="Example", mail="mail@example.com"),
+    datasets=[f"{HOST}{DATASET_PREFIX}/{d.identifier}" for d in mash.query(filters.isKind(models.Dataset))],
+    dataservices=[f"{HOST}{SERVICE_PREFIX}/{d.identifier}" for d in mash.query(filters.isKind(models.DataService))],
+)
+
+
+app.mount(
+    "/catalog",
+    StaticFiles(directory="mash/core/catalog/frontend", html=True),
+    name="Data catalog UI",
+)
+app.mount("/assets", StaticFiles(directory="mash/core/catalog/frontend/assets"))
+
+
+@app.get("/dcat/catalog", tags=["Catalog"])
+def describe_data_catalog() -> models.Catalog:
+    return catalog
+
+
+datasets = APIRouter(prefix=DATASET_PREFIX, tags=["Datasets"])
+
+
+@datasets.get("")
+@cached(cache=LRUCache(8))
+def list_datasets() -> list[models.Dataset]:
+    return mash.query(filters.isKind(models.Dataset))
+
+
+@datasets.get("/{id}")
+def describe_dataset(id: str) -> models.Dataset:
+    return mash.query(filters.isKind(models.Dataset), filters.hasIdentifier(id))[0]
+
+
+app.include_router(datasets)