cmsdials 1.4.0__tar.gz → 1.5.0__tar.gz

{cmsdials-1.4.0 → cmsdials-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cmsdials
-Version: 1.4.0
+Version: 1.5.0
 Summary: The Python api client interface to DIALS service
 Home-page: https://github.com/cms-DQM/dials-py
 Author: Gabriel Moreira
@@ -108,11 +108,16 @@ This package interacts with DIALS api endpoints using underlying classes in `Dia
 ### Retrieving a specific object using `get`
 
 ```python
+dials.dataset_index.get(dataset_id=14677060)
 dials.file_index.get(dataset_id=14677060, file_id=3393809397)
 dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=1)
 dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=96)
 dials.lumi.get(dataset_id=14677060, run_number=367112, ls_number=10)
 dials.run.get(dataset_id=14677060, run_number=367112)
+
+# jetmet worskpace
+dials.ml_models_index(model_id=1)
+dials.ml_bad_lumisection(model_id=19, dataset_id=15102369, run_number=386951, ls_number=36)
 ```
 
 ### Retrieving a list of objects per page using `list`
@@ -120,11 +125,14 @@ dials.run.get(dataset_id=14677060, run_number=367112)
 It is possible to get a list of entries from those endpoint using the `list` and `list_all` methods, the `list` method will fetch only one page and the `list_all` will fetch all available pages:
 
 ```python
+dials.dataset_index.list()
 dials.file_index.list()
 dials.h1d.list()
 dials.h2d.list()
 dials.lumi.list()
 dials.run.list()
+dials.ml_models_index.list()
+dials.ml_bad_lumisection.list()
 ```
 
 ### Retrieving all available pages of a list of objects using `list_all`
@@ -132,11 +140,14 @@ dials.run.list()
 Note: Keep in mind that running `list_all` without any filter can take too much time, since you will be retrieving all rows in the database.
 
 ```python
+dials.dataset_index.list_all()
 dials.file_index.list_all()
 dials.h1d.list_all()
 dials.h2d.list_all()
 dials.lumi.list_all()
 dials.run.list_all()
+dials.ml_models_index.list_all()
+dials.ml_bad_lumisection.list_all()
 ```
 
 If you don't need all available pages but just a subset of then, it is possible to specify a `max_pages` integer parameter:
@@ -155,9 +166,12 @@ from cmsdials.filters import (
     LumisectionHistogram1DFilters,
     LumisectionHistogram2DFilters,
     LumisectionFilters,
-    RunFilters
+    RunFilters,
+    MLBadLumisectionFilters
 )
 
+dials.dataset_index.list_all(DatasetIndexFilters(page_size=500))
+
 dials.file_index.list(FileIndexFilters(dataset__regex="2024B"))
 
 dials.h1d.list(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"))
@@ -167,6 +181,19 @@ dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number
 dials.lumi.list_all(LumisectionFilters(run_number=360392), max_pages=5)
 
 dials.run.list_all(RunFilters(run_number__gte=360392, run_number__lte=365000), max_pages=5)
+
+# jetmet workspace
+dials.ml_models_index.list_all(MLModelsIndexFilters(active=True))
+
+# jetmet workspace
+dials.ml_bad_lumis.list_all(
+    MLBadLumisectionFilters(
+        page_size=500,
+        model_id__in=[20,19],
+        dataset_id__in=[15042670],
+        run_number__in=[385801,385799,385764]
+    )
+)
 ```
 
 ### Dials MEs
@@ -248,6 +275,28 @@ data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls
 print(len(data.results))  # 200
 ```
 
+## Fetching ML certification json and ML golden-like json
+
+The following examples are testable in the `jetmet` workspace:
+
+```python
+dials.ml_bad_lumis.cert_json(
+    model_id__in=[20,19],
+    dataset_id__in=[15042670],
+    run_number__in=[385801,385799,385764]
+)
+
+dials.ml_bad_lumis.golden_json(
+    model_id__in=[20,19],
+    dataset_id__in=[15042670],
+    run_number__in=[385801,385799,385764]
+)
+```
+
+You may need to query the `ml_models_index` client to fetch the models ids you are interested and the `dataset-index` client to fetch the datasets ids. Take a look in the live documentation to check all possible filters.
+
+Attention: Those endpoints doesn't return a Pydantic model, instead they are returning a plain json response. Consequently, the method `to_pandas` doesn't work on them.
+
 ## Usage with local DIALS
 
 All classes that interface the DIALS service inherits the class `BaseAPIClient` which propagate the `base_url`, `route` and `version` attributes with production values. In order to use dials-py with a local version of DIALS it is possible to overwrite those attributes when instantiating the `AuthClient` and the `Dials` client, for example:

{cmsdials-1.4.0 → cmsdials-1.5.0}/README.md

@@ -80,11 +80,16 @@ This package interacts with DIALS api endpoints using underlying classes in `Dia
 ### Retrieving a specific object using `get`
 
 ```python
+dials.dataset_index.get(dataset_id=14677060)
 dials.file_index.get(dataset_id=14677060, file_id=3393809397)
 dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=1)
 dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=96)
 dials.lumi.get(dataset_id=14677060, run_number=367112, ls_number=10)
 dials.run.get(dataset_id=14677060, run_number=367112)
+
+# jetmet worskpace
+dials.ml_models_index(model_id=1)
+dials.ml_bad_lumisection(model_id=19, dataset_id=15102369, run_number=386951, ls_number=36)
 ```
 
 ### Retrieving a list of objects per page using `list`
@@ -92,11 +97,14 @@ dials.run.get(dataset_id=14677060, run_number=367112)
 It is possible to get a list of entries from those endpoint using the `list` and `list_all` methods, the `list` method will fetch only one page and the `list_all` will fetch all available pages:
 
 ```python
+dials.dataset_index.list()
 dials.file_index.list()
 dials.h1d.list()
 dials.h2d.list()
 dials.lumi.list()
 dials.run.list()
+dials.ml_models_index.list()
+dials.ml_bad_lumisection.list()
 ```
 
 ### Retrieving all available pages of a list of objects using `list_all`
@@ -104,11 +112,14 @@ dials.run.list()
 Note: Keep in mind that running `list_all` without any filter can take too much time, since you will be retrieving all rows in the database.
 
 ```python
+dials.dataset_index.list_all()
 dials.file_index.list_all()
 dials.h1d.list_all()
 dials.h2d.list_all()
 dials.lumi.list_all()
 dials.run.list_all()
+dials.ml_models_index.list_all()
+dials.ml_bad_lumisection.list_all()
 ```
 
 If you don't need all available pages but just a subset of then, it is possible to specify a `max_pages` integer parameter:
@@ -127,9 +138,12 @@ from cmsdials.filters import (
     LumisectionHistogram1DFilters,
     LumisectionHistogram2DFilters,
     LumisectionFilters,
-    RunFilters
+    RunFilters,
+    MLBadLumisectionFilters
 )
 
+dials.dataset_index.list_all(DatasetIndexFilters(page_size=500))
+
 dials.file_index.list(FileIndexFilters(dataset__regex="2024B"))
 
 dials.h1d.list(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"))
@@ -139,6 +153,19 @@ dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number
 dials.lumi.list_all(LumisectionFilters(run_number=360392), max_pages=5)
 
 dials.run.list_all(RunFilters(run_number__gte=360392, run_number__lte=365000), max_pages=5)
+
+# jetmet workspace
+dials.ml_models_index.list_all(MLModelsIndexFilters(active=True))
+
+# jetmet workspace
+dials.ml_bad_lumis.list_all(
+    MLBadLumisectionFilters(
+        page_size=500,
+        model_id__in=[20,19],
+        dataset_id__in=[15042670],
+        run_number__in=[385801,385799,385764]
+    )
+)
 ```
 
 ### Dials MEs
@@ -220,6 +247,28 @@ data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls
 print(len(data.results))  # 200
 ```
 
+## Fetching ML certification json and ML golden-like json
+
+The following examples are testable in the `jetmet` workspace:
+
+```python
+dials.ml_bad_lumis.cert_json(
+    model_id__in=[20,19],
+    dataset_id__in=[15042670],
+    run_number__in=[385801,385799,385764]
+)
+
+dials.ml_bad_lumis.golden_json(
+    model_id__in=[20,19],
+    dataset_id__in=[15042670],
+    run_number__in=[385801,385799,385764]
+)
+```
+
+You may need to query the `ml_models_index` client to fetch the models ids you are interested and the `dataset-index` client to fetch the datasets ids. Take a look in the live documentation to check all possible filters.
+
+Attention: Those endpoints doesn't return a Pydantic model, instead they are returning a plain json response. Consequently, the method `to_pandas` doesn't work on them.
+
 ## Usage with local DIALS
 
 All classes that interface the DIALS service inherits the class `BaseAPIClient` which propagate the `base_url`, `route` and `version` attributes with production values. In order to use dials-py with a local version of DIALS it is possible to overwrite those attributes when instantiating the `AuthClient` and the `Dials` client, for example:

cmsdials-1.5.0/cmsdials/clients/dataset_index/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import DatasetIndex, DatasetIndexFilters, PaginatedDatasetIndexList
+
+
+class DatasetIndexClient(BaseAuthorizedAPIClient):
+    data_model = DatasetIndex
+    pagination_model = PaginatedDatasetIndexList
+    filter_class = DatasetIndexFilters
+    lookup_url = "dataset-index/"
+
+    def get(self, dataset_id: int, file_id: int, **kwargs):
+        edp = f"{dataset_id}/{file_id}/"
+        return super().get(edp, **kwargs)

cmsdials-1.5.0/cmsdials/clients/dataset_index/models.py

@@ -0,0 +1,30 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel, Field
+
+from ...utils.base_model import OBaseModel, PaginatedBaseModel
+
+
+class DatasetIndex(BaseModel):
+    dataset_id: int
+    dataset: str = Field(..., max_length=255)
+    era: str = Field(..., max_length=255)
+    data_tier: str = Field(..., max_length=255)
+    primary_ds_name: str = Field(..., max_length=255)
+    processed_ds_name: str = Field(..., max_length=255)
+    processing_version: int
+    last_modification_date: datetime
+
+
+class PaginatedDatasetIndexList(PaginatedBaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[DatasetIndex]
+
+
+class DatasetIndexFilters(OBaseModel):
+    next_token: Optional[str] = None
+    page_size: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/clients/file_index/models.py

@@ -26,6 +26,7 @@ class PaginatedFileIndexList(PaginatedBaseModel):
 
 class FileIndexFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     logical_file_name: Optional[str] = None
     logical_file_name__regex: Optional[str] = None

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/clients/h1d/models.py

@@ -28,6 +28,7 @@ class PaginatedLumisectionHistogram1DList(PaginatedBaseModel):
 
 class LumisectionHistogram1DFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     file_id: Optional[int] = None
     run_number: Optional[int] = None

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/clients/h2d/models.py

@@ -31,6 +31,7 @@ class PaginatedLumisectionHistogram2DList(PaginatedBaseModel):
 
 class LumisectionHistogram2DFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     file_id: Optional[int] = None
     run_number: Optional[int] = None

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/clients/lumisection/models.py

@@ -22,6 +22,7 @@ class PaginatedLumisectionList(PaginatedBaseModel):
 
 class LumisectionFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     run_number: Optional[int] = None
     run_number__lte: Optional[int] = None

cmsdials-1.5.0/cmsdials/clients/ml_bad_lumisection/client.py

@@ -0,0 +1,29 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import MLBadLumisection, MLBadLumisectionFilters, PaginatedMLBadLumisectionList
+
+
+class MLBadLumisectionClient(BaseAuthorizedAPIClient):
+    data_model = MLBadLumisection
+    pagination_model = PaginatedMLBadLumisectionList
+    filter_class = MLBadLumisectionFilters
+    lookup_url = "ml-bad-lumisection/"
+
+    def get(self, model_id: int, dataset_id: int, run_number: int, ls_number: int, me_id: int, **kwargs):
+        edp = f"{model_id}/{dataset_id}/{run_number}/{ls_number}/{me_id}/"
+        return super().get(edp, **kwargs)
+
+    def cert_json(self, model_id__in: list[int], dataset_id__in: list[int], run_number__in: list[int], **kwargs):
+        edp = "cert-json/"
+        midin = ",".join(str(v) for v in model_id__in)
+        didin = ",".join(str(v) for v in dataset_id__in)
+        ridin = ",".join(str(v) for v in run_number__in)
+        params = {"model_id__in": midin, "dataset_id__in": didin, "run_number__in": ridin}
+        return super().get(edp, params=params, return_raw_json=True, **kwargs)
+
+    def golden_json(self, model_id__in: list[int], dataset_id__in: list[int], run_number__in: list[int], **kwargs):
+        edp = "golden-json/"
+        midin = ",".join(str(v) for v in model_id__in)
+        didin = ",".join(str(v) for v in dataset_id__in)
+        ridin = ",".join(str(v) for v in run_number__in)
+        params = {"model_id__in": midin, "dataset_id__in": didin, "run_number__in": ridin}
+        return super().get(edp, params=params, return_raw_json=True, **kwargs)

cmsdials-1.5.0/cmsdials/clients/ml_bad_lumisection/models.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel
+
+from ...utils.base_model import OBaseModel, PaginatedBaseModel
+
+
+class MLBadLumisection(BaseModel):
+    model_id: int
+    dataset_id: int
+    file_id: int
+    run_number: int
+    ls_number: int
+    me_id: int
+    mse: float
+
+
+class PaginatedMLBadLumisectionList(PaginatedBaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[MLBadLumisection]
+
+
+class MLBadLumisectionFilters(OBaseModel):
+    next_token: Optional[str] = None
+    page_size: Optional[int] = None
+    model_id: Optional[int] = None
+    model_id__in: Optional[list[int]] = None
+    dataset_id: Optional[int] = None
+    dataset_id__in: Optional[list[int]] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None
+    me_id: Optional[int] = None
+    me: Optional[str] = None
+    me__regex: Optional[str] = None
+    run_number: Optional[int] = None
+    run_number__in: Optional[list[int]] = None
+    ls_number: Optional[int] = None

cmsdials-1.5.0/cmsdials/clients/ml_models_index/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import MLModelsIndex, MLModelsIndexFilters, PaginatedMLModelsIndexList
+
+
+class MLModelsIndexClient(BaseAuthorizedAPIClient):
+    data_model = MLModelsIndex
+    pagination_model = PaginatedMLModelsIndexList
+    filter_class = MLModelsIndexFilters
+    lookup_url = "ml-models-index/"
+
+    def get(self, model_id: int, **kwargs):
+        edp = f"{model_id}/"
+        return super().get(edp, **kwargs)

cmsdials-1.5.0/cmsdials/clients/ml_models_index/models.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel
+
+from ...utils.base_model import OBaseModel, PaginatedBaseModel
+
+
+class MLModelsIndex(BaseModel):
+    model_id: int
+    filename: str
+    target_me: str
+    active: bool
+
+
+class PaginatedMLModelsIndexList(PaginatedBaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[MLModelsIndex]
+
+
+class MLModelsIndexFilters(OBaseModel):
+    next_token: Optional[str] = None
+    page_size: Optional[int] = None
+    model_id: Optional[int] = None
+    model_id__in: Optional[list[int]] = None
+    filename: Optional[str] = None
+    filename__regex: Optional[str] = None
+    target_me: Optional[str] = None
+    target_me__regex: Optional[str] = None
+    active: Optional[bool] = None

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/clients/run/models.py

@@ -20,6 +20,7 @@ class PaginatedRunList(PaginatedBaseModel):
 
 class RunFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     run_number: Optional[int] = None
     run_number__lte: Optional[int] = None

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/cmsdials.py

@@ -1,17 +1,21 @@
 from typing import Optional
 
 from .auth._base import BaseCredentials
+from .clients.dataset_index.client import DatasetIndexClient
 from .clients.file_index.client import FileIndexClient
 from .clients.h1d.client import LumisectionHistogram1DClient
 from .clients.h2d.client import LumisectionHistogram2DClient
 from .clients.lumisection.client import LumisectionClient
 from .clients.mes.client import MonitoringElementClient
+from .clients.ml_bad_lumisection.client import MLBadLumisectionClient
+from .clients.ml_models_index.client import MLModelsIndexClient
 from .clients.oms_proxy.client import OMSProxyClient
 from .clients.run.client import RunClient
 
 
 class Dials:
     def __init__(self, creds: BaseCredentials, workspace: Optional[str] = None, *args, **kwargs) -> None:
+        self.dataset_index = DatasetIndexClient(creds, workspace, *args, **kwargs)
         self.file_index = FileIndexClient(creds, workspace, *args, **kwargs)
         self.h1d = LumisectionHistogram1DClient(creds, workspace, *args, **kwargs)
         self.h2d = LumisectionHistogram2DClient(creds, workspace, *args, **kwargs)
@@ -19,3 +23,5 @@ class Dials:
         self.run = RunClient(creds, workspace, *args, **kwargs)
         self.mes = MonitoringElementClient(creds, workspace, *args, **kwargs)
         self.oms = OMSProxyClient(creds, *args, **kwargs)
+        self.ml_bad_lumis = MLBadLumisectionClient(creds, workspace, *args, **kwargs)
+        self.ml_models_index = MLModelsIndexClient(creds, workspace, *args, **kwargs)

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/filters.py

@@ -1,19 +1,25 @@
+from .clients.dataset_index.models import DatasetIndexFilters
 from .clients.file_index.models import FileIndexFilters
 from .clients.h1d.models import LumisectionHistogram1DFilters
 from .clients.h2d.models import LumisectionHistogram2DFilters
 from .clients.lumisection.models import LumisectionFilters
 from .clients.mes.models import MEFilters
+from .clients.ml_bad_lumisection.models import MLBadLumisectionFilters
+from .clients.ml_models_index.models import MLModelsIndexFilters
 from .clients.oms_proxy.models import OMSFilter, OMSPage
 from .clients.run.models import RunFilters
 
 
 __all__ = [
+    "DatasetIndexFilters",
     "FileIndexFilters",
     "LumisectionHistogram1DFilters",
     "LumisectionHistogram2DFilters",
     "LumisectionFilters",
     "RunFilters",
     "MEFilters",
+    "MLBadLumisectionFilters",
+    "MLModelsIndexFilters",
     "OMSFilter",
     "OMSPage",
 ]

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/utils/api_client.py

@@ -81,12 +81,13 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
         self.creds.before_request(base)
         return base
 
-    def get(self, edp: str, retries=DEFAULT_RETRIES):  # noqa: A002
+    def get(self, edp: str, retries=DEFAULT_RETRIES, params: Optional[dict] = None, return_raw_json: bool = False):
         endpoint_url = self.api_url + self.lookup_url + edp
         headers = self._build_headers()
         response = self._requests_get_retriable(
             endpoint_url,
             headers=headers,
+            params=params,
             timeout=self.default_timeout,
             retries=retries,
         )
@@ -98,6 +99,8 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
             raise err
 
         response = response.json()
+        if return_raw_json:
+            return response
         return self.data_model(**response)
 
     def list(self, filters=None, retries=DEFAULT_RETRIES):

{cmsdials-1.4.0 → cmsdials-1.5.0}/cmsdials/utils/base_model.py

@@ -14,7 +14,11 @@ else:
 
 class OBaseModel(BaseModel):
     def cleandict(self):
-        return {key: value for key, value in self.dict().items() if value is not None}
+        return {
+            key: ",".join(str(v) for v in value) if isinstance(value, (list, tuple)) else value
+            for key, value in self.dict().items()
+            if value is not None
+        }
 
 
 class PaginatedBaseModel(BaseModel):

{cmsdials-1.4.0 → cmsdials-1.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cmsdials"
-version = "1.4.0"
+version = "1.5.0"
 description = "The Python api client interface to DIALS service"
 authors = ["Gabriel Moreira <gabrielmscampos@gmail.com>"]
 readme = "README.md"