minedrac 0.0.1__tar.gz

minedrac-0.0.1/LICENSE.md

@@ -0,0 +1,20 @@
+# MIT License
+
+**Copyright (c) 2021 European Synchrotron Radiation Facility**
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

minedrac-0.0.1/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: minedrac
+Version: 0.0.1
+Requires-Python: >=3.11
+License-File: LICENSE.md
+Requires-Dist: icat-plus-client
+Requires-Dist: typer
+Requires-Dist: python-dotenv
+Requires-Dist: pandas
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-spec; extra == "test"
+Requires-Dist: pytest-dotenv; extra == "test"
+Provides-Extra: docs
+Requires-Dist: sphinx<9,>=7; extra == "docs"
+Requires-Dist: myst-parser<5,>=3; extra == "docs"
+Requires-Dist: sphinx-click>=6; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: typer>=0.20; extra == "dev"
+Requires-Dist: pandas>=2.3; extra == "dev"
+Requires-Dist: black>=25; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-spec; extra == "dev"
+Requires-Dist: pytest-dotenv; extra == "dev"
+Dynamic: license-file

minedrac-0.0.1/README.md

@@ -0,0 +1,56 @@
+# MineDRAC
+
+## Configuration
+
+### Logging
+
+You can manage the logging level with an env variable:
+
+```
+export LOG_LEVEL=DEBUG
+```
+
+### ICAT+
+
+MineDRAC is configured to use the official ESRF icat+ server by default: https://icatplus.esrf.fr
+
+It can be configured to use any other server via a configuration `.env` file or export the variable `icat_plus`
+Example:
+
+```
+export icat_plus_server="http://localhost:8000"
+```
+
+You can find out which server is pointing to by using the command:
+
+```
+> minedrac config icat_plus
+http://localhost:8083
+
+```
+
+### Commands
+
+#### login
+
+This command allows to get the token of a DB user (this will not work for user accounts). For getting your token you can access it via the data portal.
+
+```
+> minedrac login authentication --username reader --password ****
+c99d6e5d-096b-45f1-bde3-c4322fe15251
+
+```
+
+#### Retrieve
+
+This command family allows to retrieve data from ICAT via ICAT+.
+
+##### Dataset
+
+This method will return the json serialization of the datasets with the filters applied.
+
+You can list the filters by typing:
+
+```
+minedrac retrieve dataset --help
+```

minedrac-0.0.1/pyproject.toml

@@ -0,0 +1,73 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "minedrac"
+version = "0.0.1"
+requires-python = ">=3.11"
+dependencies = [
+    "icat-plus-client",    
+    "typer",
+    "python-dotenv",
+    "pandas"
+]
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-spec",
+    "pytest-dotenv"
+]
+docs = [
+  "sphinx>=7,<9",
+  "myst-parser>=3,<5",
+  "sphinx-click>=6",
+]
+dev = [
+    "typer >=0.20",
+    "pandas >=2.3",
+    "black >=25",
+    "flake8 >=4",
+    "ruff",
+    "pytest",
+    "pytest-spec",
+    "pytest-dotenv"
+]
+
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.black]
+line-length = 110
+target-version = ["py39", "py310", "py311"]  # adjust to your versions
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.mypy_cache
+  | \.pytest_cache
+  | build
+  | dist
+)/
+'''
+
+[tool.ruff]
+line-length = 110
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+    "E",   # errors
+    "F",   # pyflakes
+    "I",   # import sorting
+    "B",   # flake8-bugbear
+    "UP",  # pyupgrade
+]
+
+
+[project.scripts]
+minedrac = "cli:main"

minedrac-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

minedrac-0.0.1/src/authentication/cli.py

@@ -0,0 +1,30 @@
+import typer
+
+from data import session
+
+authentication_app = typer.Typer(help="Authentication commands")
+
+
+@authentication_app.command("authenticate", help="Gets the ICAT's token based on username and password")
+def authenticate_cli(
+    authenticator: str = typer.Option(
+        "db",
+        "-a",
+        "--authenticator",
+        help="Plugin authentication. it can be db or esrf. Default is db",
+    ),
+    username: str = typer.Option(..., "-u", "--username", help="ICAT username"),
+    password: str = typer.Option(..., "-p", "--password", help="ICAT password"),
+):
+    """Retrieve investigations by session_id."""
+    session_response = session.get_session(authenticator, username, password)
+    typer.echo(session_response)
+
+
+@authentication_app.command("info", help="Gets the ICAT's session information based on a valid ICAT's token")
+def get_session_info(
+    token: str = typer.Option(..., "-t", "--token", help="ICAT token"),
+):
+    """Retrieve investigations by session_id."""
+    session_response = session.get_info(token)
+    typer.echo(session_response)

minedrac-0.0.1/src/data/dataset.py

@@ -0,0 +1,64 @@
+import logging
+
+import icat_plus_client
+from dotenv import load_dotenv
+from icat_plus_client.models.dataset import Dataset
+
+from config import configuration
+
+# Load .env if needed
+load_dotenv()
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_datasets(
+    session_id: str,
+    investigation_ids: str | None = None,
+    instrument_name: str | None = None,
+    dataset_ids: str | None = None,
+    dataset_type: str | None = None,
+    nested: str | None = None,
+    sample_id: str | None = None,
+    limit: int | None = None,
+    skip: int | None = None,
+    search: str | None = None,
+    sort_by: str | None = None,
+    sort_order: str | None = None,
+) -> list[Dataset]:
+    with icat_plus_client.ApiClient(configuration) as api_client:
+        api_instance = icat_plus_client.CatalogueApi(api_client)
+        try:
+            datasets: list[Dataset] = api_instance.catalogue_session_id_dataset_get(
+                session_id,
+                investigation_ids=investigation_ids,
+                dataset_ids=dataset_ids,
+                sample_id=sample_id,
+                instrument_name=instrument_name,
+                limit=limit,
+                skip=skip,
+                search=search,
+                dataset_type=dataset_type,
+                nested=nested,
+                sort_by=sort_by,
+                sort_order=sort_order,
+            )
+
+            if datasets:
+                logger.debug("Fetched %d datasets successfully.", len(datasets))
+                return datasets
+            else:
+                logger.debug("No datasets found for the given parameters.")
+                return []
+
+        except icat_plus_client.rest.ApiException as api_exc:
+            logger.error(
+                "API Exception when CatalogueApi->catalogue_session_id_dataset_get: %s",
+                api_exc,
+            )
+        except Exception as e:
+            logger.exception("Unexpected error while fetching datasets: %s", e)
+
+        return []

minedrac-0.0.1/src/data/instrument.py

@@ -0,0 +1,39 @@
+import logging
+
+import icat_plus_client
+from dotenv import load_dotenv
+from icat_plus_client.models.instrument import Instrument
+
+from config import configuration
+
+# Load .env if needed
+load_dotenv()
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_instruments() -> list[Instrument]:
+    """Fetch all instruments from ICAT Catalogue."""
+    with icat_plus_client.ApiClient(configuration) as api_client:
+        api_instance = icat_plus_client.CatalogueApi(api_client)
+        try:
+            instruments: list[Instrument] = api_instance.catalogue_instruments_get()
+
+            if instruments:
+                logger.debug("Fetched %d instruments successfully.", len(instruments))
+                return instruments
+            else:
+                logger.warning("No instruments found in the catalogue.")
+                return []
+
+        except icat_plus_client.rest.ApiException as api_exc:
+            logger.error(
+                "API Exception when calling CatalogueApi->catalogue_instruments_get: %s",
+                api_exc,
+            )
+        except Exception as e:
+            logger.exception("Unexpected error while fetching instruments: %s", e)
+
+        return []

minedrac-0.0.1/src/data/investigation.py

@@ -0,0 +1,65 @@
+import logging
+
+import icat_plus_client
+from dotenv import load_dotenv
+from icat_plus_client.models.investigation import Investigation
+
+from config import configuration
+
+# Load .env if needed
+load_dotenv()
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_investigation_by_id(
+    session_id: str,
+    investigation_id: str | None = None,
+    instrument_name: str | None = None,
+    investigation_name: str | None = None,
+    filter: str | None = None,
+    start_date: str | None = None,
+    end_date: str | None = None,
+    limit: int | None = None,
+    skip: int | None = None,
+    search: str | None = None,
+    sort_by: str | None = None,
+    sort_order: str | None = None,
+    with_has_access: bool | None = None,
+) -> list[Investigation]:
+    """Fetch investigations by session ID with optional filters."""
+
+    with icat_plus_client.ApiClient(configuration) as api_client:
+        api_instance = icat_plus_client.CatalogueApi(api_client)
+        try:
+            investigations: list[Investigation] = api_instance.catalogue_session_id_investigation_get(
+                session_id=session_id,
+                instrument_name=instrument_name,
+                investigation_name=investigation_name,
+                ids=investigation_id,
+                filter=filter,
+                start_date=start_date,
+                end_date=end_date,
+                limit=limit,
+                skip=skip,
+                search=search,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                with_has_access=with_has_access,
+            )
+
+            if investigations:
+                logger.debug("Fetched %d investigations successfully.", len(investigations))
+                return investigations
+            else:
+                logger.debug("No investigations found for the given parameters.")
+                return []
+
+        except icat_plus_client.rest.ApiException as api_exc:
+            logger.error(f"API Exception CatalogueApi->catalogue_session_id_investigation_get: {api_exc}")
+        except Exception as e:
+            logger.exception("Unexpected error while fetching investigations: %s", e)
+
+        return []

minedrac-0.0.1/src/data/parcel.py

@@ -0,0 +1,59 @@
+import logging
+
+import icat_plus_client
+from dotenv import load_dotenv
+from icat_plus_client.models.item import Item
+from icat_plus_client.models.parcel import Parcel
+
+from config import configuration
+
+# Load .env if needed
+load_dotenv()
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_parcels_by(
+    token: str,
+    investigation_id: str | None = None,
+) -> list[Parcel]:
+    with icat_plus_client.ApiClient(configuration) as api_client:
+        # Create an instance of the API class
+        api_instance = icat_plus_client.TrackingApi(api_client)
+
+        try:
+            # Lists shipments associated to an investigation
+            shipments = api_instance.tracking_session_id_shipment_get(
+                investigation_id=investigation_id, session_id=token
+            )
+            if len(shipments) == 1:
+                parcels = api_instance.tracking_session_id_parcel_get(
+                    shipments[0].id, token, "", investigation_id=investigation_id
+                )
+                logger.debug("Fetched %d parcels successfully.", len(parcels))
+                if parcels:
+                    return parcels
+        except Exception as e:
+            logger.error(f"Exception when TrackingApi->tracking_session_id_parcel_get: {e}")
+        return []
+
+
+def get_declared_samples_by(
+    token: str,
+    investigation_id: str | None = None,
+) -> list[Item]:
+    try:
+        parcels = get_parcels_by(token, investigation_id)
+        samples = []
+        for parcel in parcels:
+            for item in parcel.content:
+                if item.type == "CONTAINER":
+                    samples.extend(item.content)
+                else:
+                    samples.extend(item)
+        return samples
+    except Exception as e:
+        logger.error(f"Exception when get_declared_samples_by: {e}")
+    return []

minedrac-0.0.1/src/data/sample.py

@@ -0,0 +1,37 @@
+import logging
+
+import icat_plus_client
+from dotenv import load_dotenv
+from icat_plus_client.models.sample import Sample
+
+from config import configuration
+
+# Load .env if needed
+load_dotenv()
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_samples_by(
+    token: str,
+    investigation_id: str | None = None,
+) -> list[Sample]:
+    with icat_plus_client.ApiClient(configuration) as api_client:
+        # Create an instance of the API class
+        api_instance = icat_plus_client.CatalogueApi(api_client)
+        investigation_id = investigation_id
+        try:
+            # Gets samples
+            samples = api_instance.catalogue_session_id_samples_get(
+                token, investigation_id=str(investigation_id)
+            )
+            if samples:
+                logger.debug("Fetched %d samples successfully.", len(samples))
+                return samples
+            else:
+                logger.debug("No samples found for the given parameters.")
+                return []
+        except Exception as e:
+            print(f"Exception when CatalogueApi->catalogue_session_id_samples_get: {e}")

minedrac-0.0.1/src/data/session.py

@@ -0,0 +1,53 @@
+import logging
+
+import icat_plus_client
+from dotenv import load_dotenv
+from icat_plus_client.models.credentials import Credentials
+from icat_plus_client.models.session import Session
+
+from config import configuration
+
+# Load .env if needed
+load_dotenv()
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_session(authenticator: str, username: str, password: str):
+    """Get the sessionId"""
+    with icat_plus_client.ApiClient(configuration) as api_client:
+        api_instance = icat_plus_client.SessionApi(api_client)
+        try:
+            cred = Credentials(
+                username=username,
+                password=password,
+                plugin=authenticator,
+            )
+            api_response = api_instance.session_post(cred)
+            return api_response.session_id
+        except icat_plus_client.rest.ApiException as api_exc:
+            logger.error(
+                "API Exception when calling SessionApi->session_post: %s",
+                api_exc,
+            )
+        except Exception as e:
+            logger.exception("Unexpected error while logging in: %s", e)
+
+        return []
+
+
+def get_info(token: str) -> Session:
+    """Get the sessionId"""
+    with icat_plus_client.ApiClient(configuration) as api_client:
+        # Create an instance of the API class
+        api_instance = icat_plus_client.SessionApi(api_client)
+
+        try:
+            # Gets information about the session in ICAT
+            api_response: Session = api_instance.session_session_id_get(token)
+            return api_response
+
+        except Exception as e:
+            print(f"Exception when calling SessionApi->session_session_id_get: {e}\n")

minedrac-0.0.1/src/invoicing/cli.py

@@ -0,0 +1,148 @@
+import csv
+import sys
+from datetime import datetime
+
+import typer
+
+from data import dataset, investigation
+from data.parcel import get_declared_samples_by, get_parcels_by
+from invoicing import duration
+from stats.volume.volume import get_volumes_by_investigation
+
+invoicing_app = typer.Typer(help="Invoicing commands")
+report_invoicing_app = typer.Typer(help="report commands")
+invoicing_app.add_typer(report_invoicing_app, name="report")
+
+
+@report_invoicing_app.command("report", help="Generates a report for invoicong for each investigation")
+def invoicing_report(
+    token: str = typer.Option(..., "-t", "--token", help="ICAT session ID"),
+    investigation_ids: str | None = typer.Option(
+        None,
+        "--investigation-ids",
+        "-i",
+        help="Comma separated list of investigation IDs (optional)",
+    ),
+    instrument_name: str | None = typer.Option(None, help="Filter by instrument name"),
+    investigation_name: str | None = typer.Option(None, help="Filter by instrument name"),
+    start_date: str = typer.Option(
+        None,
+        "-s",
+        "--start-date",
+        help="Start date range. Format is YYYY-MM-DD. Example: 2019-01-01",
+    ),
+    end_date: str = typer.Option(
+        None,
+        "-e",
+        "--end-date",
+        help="End date range. Format is YYYY-MM-DD. Example: 2019-01-01",
+    ),
+    unit: str = typer.Option(
+        "GB",
+        "-u",
+        "--unit",
+        help="It can be MB, GB or TB (otherwise will return in GB)",
+    ),
+    merge_gap_minutes: int = typer.Option(
+        20,
+        "-g",
+        "--gap",
+        help="Gap in minutes to merge datasets. Less than 'gap' minutes the datasets time will be merged",
+    ),
+    output: str = typer.Option(
+        None,
+        "-o",
+        "--output",
+        help="CSV output file path. Example: myfile.csv",
+    ),
+    log: str = typer.Option(
+        None,
+        "-l",
+        "--log",
+        help="CSV log output file path. Example: myfile_log.csv",
+    ),
+    ignore_longer_than: int = typer.Option(
+        None,
+        "-ig",
+        "--ignore",
+        help="The time in hours that will be taken into account to omit a dataset as considered an error.",
+    ),
+):
+    investigations = investigation.get_investigation_by_id(
+        session_id=token,
+        investigation_id=investigation_ids,
+        start_date=start_date,
+        end_date=end_date,
+        instrument_name=instrument_name,
+        investigation_name=investigation_name,
+    )
+
+    fields = [
+        "proposal",
+        "investigation_id",
+        "beamline",
+        "start_date",
+        "end_date",
+        "booked_time",
+        "samples_collected",
+        "samples_declared",
+        "parcels",
+        "volume",
+        "raw_volume",
+        "processed_volume",
+        "effective_time",
+        "total_time",
+    ]
+
+    writers = [csv.DictWriter(sys.stdout, fieldnames=fields, delimiter="\t")]
+    # If file provided, also write to the file
+    if output:
+        f = open(output, "w", newline="")
+        writers.append(csv.DictWriter(f, fieldnames=fields, delimiter="\t"))
+    else:
+        f = None
+
+    # Write headers on all writers
+    for w in writers:
+        w.writeheader()
+
+    for inv in investigations:
+        (total, raw, processed) = get_volumes_by_investigation([inv], unit)
+        samples_declared = get_declared_samples_by(token, str(inv.id))
+        parcels = get_parcels_by(token, str(inv.id))
+        start = datetime.fromisoformat(inv.start_date)
+        end = datetime.fromisoformat(inv.end_date) if inv.end_date is not None else start
+        row = {
+            "proposal": inv.name,
+            "investigation_id": inv.id,
+            "beamline": inv.instrument.name,
+            "start_date": start.strftime("%Y-%m-%d %H:%M:%S"),
+            "end_date": end.strftime("%Y-%m-%d %H:%M:%S"),
+            "booked_time": end - start,
+            "volume": f"{total} {unit}",
+            "raw_volume": f"{raw} {unit}",
+            "processed_volume": f"{processed} {unit}",
+        }
+
+        """Calculate total dataset duration for a session."""
+        datasets = dataset.get_datasets(
+            session_id=token,
+            investigation_ids=str(inv.id),
+            dataset_type="acquisition",
+        )
+
+        samples_collected = sorted({d.sample_name for d in datasets if getattr(d, "sample_name", None)})
+        row["samples_collected"] = len(samples_collected)
+        row["samples_declared"] = len(samples_declared)
+        row["parcels"] = len(parcels)
+        row["total_time"] = duration.sum_dataset_durations(
+            datasets, merge_gap_minutes, log, ignore_longer_than
+        )
+        row["effective_time"] = duration.sum_dataset_durations(datasets, 0, None, ignore_longer_than)
+
+        for w in writers:
+            w.writerow(row)
+
+
+click_invoicing_app = typer.main.get_command(invoicing_app)
+report_click_invoicing_app = typer.main.get_command(report_invoicing_app)