cmsdials 1.3.0__tar.gz → 1.5.0__tar.gz

{cmsdials-1.3.0 → cmsdials-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cmsdials
-Version: 1.3.0
+Version: 1.5.0
 Summary: The Python api client interface to DIALS service
 Home-page: https://github.com/cms-DQM/dials-py
 Author: Gabriel Moreira
@@ -22,6 +22,7 @@ Requires-Dist: pydantic (>=1,<2)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: tqdm (>=4.66.1) ; extra == "tqdm"
 Requires-Dist: typing-extensions (>=3.6.6,<4.6.0)
+Requires-Dist: urllib3 (>=2.2.3,<3.0.0)
 Project-URL: Repository, https://github.com/cms-DQM/dials-py
 Description-Content-Type: text/markdown
 
@@ -107,11 +108,16 @@ This package interacts with DIALS api endpoints using underlying classes in `Dia
 ### Retrieving a specific object using `get`
 
 ```python
+dials.dataset_index.get(dataset_id=14677060)
 dials.file_index.get(dataset_id=14677060, file_id=3393809397)
 dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=1)
 dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=96)
 dials.lumi.get(dataset_id=14677060, run_number=367112, ls_number=10)
 dials.run.get(dataset_id=14677060, run_number=367112)
+
+# jetmet worskpace
+dials.ml_models_index(model_id=1)
+dials.ml_bad_lumisection(model_id=19, dataset_id=15102369, run_number=386951, ls_number=36)
 ```
 
 ### Retrieving a list of objects per page using `list`
@@ -119,11 +125,14 @@ dials.run.get(dataset_id=14677060, run_number=367112)
 It is possible to get a list of entries from those endpoint using the `list` and `list_all` methods, the `list` method will fetch only one page and the `list_all` will fetch all available pages:
 
 ```python
+dials.dataset_index.list()
 dials.file_index.list()
 dials.h1d.list()
 dials.h2d.list()
 dials.lumi.list()
 dials.run.list()
+dials.ml_models_index.list()
+dials.ml_bad_lumisection.list()
 ```
 
 ### Retrieving all available pages of a list of objects using `list_all`
@@ -131,11 +140,14 @@ dials.run.list()
 Note: Keep in mind that running `list_all` without any filter can take too much time, since you will be retrieving all rows in the database.
 
 ```python
+dials.dataset_index.list_all()
 dials.file_index.list_all()
 dials.h1d.list_all()
 dials.h2d.list_all()
 dials.lumi.list_all()
 dials.run.list_all()
+dials.ml_models_index.list_all()
+dials.ml_bad_lumisection.list_all()
 ```
 
 If you don't need all available pages but just a subset of then, it is possible to specify a `max_pages` integer parameter:
@@ -154,9 +166,12 @@ from cmsdials.filters import (
     LumisectionHistogram1DFilters,
     LumisectionHistogram2DFilters,
     LumisectionFilters,
-    RunFilters
+    RunFilters,
+    MLBadLumisectionFilters
 )
 
+dials.dataset_index.list_all(DatasetIndexFilters(page_size=500))
+
 dials.file_index.list(FileIndexFilters(dataset__regex="2024B"))
 
 dials.h1d.list(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"))
@@ -166,6 +181,19 @@ dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number
 dials.lumi.list_all(LumisectionFilters(run_number=360392), max_pages=5)
 
 dials.run.list_all(RunFilters(run_number__gte=360392, run_number__lte=365000), max_pages=5)
+
+# jetmet workspace
+dials.ml_models_index.list_all(MLModelsIndexFilters(active=True))
+
+# jetmet workspace
+dials.ml_bad_lumis.list_all(
+    MLBadLumisectionFilters(
+        page_size=500,
+        model_id__in=[20,19],
+        dataset_id__in=[15042670],
+        run_number__in=[385801,385799,385764]
+    )
+)
 ```
 
 ### Dials MEs
@@ -200,6 +228,75 @@ Progress: 100%|█████████████████████
 
 The total attribute of the bar is dynamically updated while fetching the pages.
 
+## Retrying
+
+In case of an unstable connection, DNS failures or service overload it is possible to configure any `get`, `list` and  `list_all` to retry the underlying requests using native `urllib3` retry class, for example:
+
+```python
+from urllib3.util import Retry
+
+data = dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=1, retries=Retry(total=3, backoff_factor=0.1))
+data = dials.h1d.list(retries=Retry(total=3, backoff_factor=0.1))
+data = dials.h1d.list_all(LumisectionHistogram1DFilters(), max_pages=5, retries=Retry(total=5, backoff_factor=0.1))
+```
+
+## Resuming
+
+When listing and endpoint for a long time, you may loose connection or the server can potentially return an error. By specifying `keep_failed` and using `resume_from` you can resume from an older response object, for example:
+
+```python
+from cmsdials.auth.client import AuthClient
+from cmsdials.auth.bearer import Credentials
+from cmsdials import Dials
+from cmsdials.filters import LumisectionHistogram2DFilters
+
+auth = AuthClient()
+creds = Credentials.from_creds_file()
+dials = Dials(creds, workspace="tracker")
+
+data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), keep_failed=True)  # Code may broke inside this routine
+print(len(data.results))  # 100
+print(data.exc_type)
+print(data.exc_formatted)
+
+# After failing, run this again
+data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), keep_failed=True, resume_from=data)  # Resume from failed object
+print(len(data.results))  # 200
+```
+
+You may find it useful to resume from a partial response, i.e,
+
+```python
+data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), max_pages=10, keep_failed=True)  # Code may not break, however will fetch only 100 elements
+print(len(data.results))  # 100
+
+# After fetching first 100, fetch next 100
+data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), max_pages=10, keep_failed=True, resume_from=data)  # Resume from partial object
+print(len(data.results))  # 200
+```
+
+## Fetching ML certification json and ML golden-like json
+
+The following examples are testable in the `jetmet` workspace:
+
+```python
+dials.ml_bad_lumis.cert_json(
+    model_id__in=[20,19],
+    dataset_id__in=[15042670],
+    run_number__in=[385801,385799,385764]
+)
+
+dials.ml_bad_lumis.golden_json(
+    model_id__in=[20,19],
+    dataset_id__in=[15042670],
+    run_number__in=[385801,385799,385764]
+)
+```
+
+You may need to query the `ml_models_index` client to fetch the models ids you are interested and the `dataset-index` client to fetch the datasets ids. Take a look in the live documentation to check all possible filters.
+
+Attention: Those endpoints doesn't return a Pydantic model, instead they are returning a plain json response. Consequently, the method `to_pandas` doesn't work on them.
+
 ## Usage with local DIALS
 
 All classes that interface the DIALS service inherits the class `BaseAPIClient` which propagate the `base_url`, `route` and `version` attributes with production values. In order to use dials-py with a local version of DIALS it is possible to overwrite those attributes when instantiating the `AuthClient` and the `Dials` client, for example:

{cmsdials-1.3.0 → cmsdials-1.5.0}/README.md

@@ -80,11 +80,16 @@ This package interacts with DIALS api endpoints using underlying classes in `Dia
 ### Retrieving a specific object using `get`
 
 ```python
+dials.dataset_index.get(dataset_id=14677060)
 dials.file_index.get(dataset_id=14677060, file_id=3393809397)
 dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=1)
 dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=96)
 dials.lumi.get(dataset_id=14677060, run_number=367112, ls_number=10)
 dials.run.get(dataset_id=14677060, run_number=367112)
+
+# jetmet worskpace
+dials.ml_models_index(model_id=1)
+dials.ml_bad_lumisection(model_id=19, dataset_id=15102369, run_number=386951, ls_number=36)
 ```
 
 ### Retrieving a list of objects per page using `list`
@@ -92,11 +97,14 @@ dials.run.get(dataset_id=14677060, run_number=367112)
 It is possible to get a list of entries from those endpoint using the `list` and `list_all` methods, the `list` method will fetch only one page and the `list_all` will fetch all available pages:
 
 ```python
+dials.dataset_index.list()
 dials.file_index.list()
 dials.h1d.list()
 dials.h2d.list()
 dials.lumi.list()
 dials.run.list()
+dials.ml_models_index.list()
+dials.ml_bad_lumisection.list()
 ```
 
 ### Retrieving all available pages of a list of objects using `list_all`
@@ -104,11 +112,14 @@ dials.run.list()
 Note: Keep in mind that running `list_all` without any filter can take too much time, since you will be retrieving all rows in the database.
 
 ```python
+dials.dataset_index.list_all()
 dials.file_index.list_all()
 dials.h1d.list_all()
 dials.h2d.list_all()
 dials.lumi.list_all()
 dials.run.list_all()
+dials.ml_models_index.list_all()
+dials.ml_bad_lumisection.list_all()
 ```
 
 If you don't need all available pages but just a subset of then, it is possible to specify a `max_pages` integer parameter:
@@ -127,9 +138,12 @@ from cmsdials.filters import (
     LumisectionHistogram1DFilters,
     LumisectionHistogram2DFilters,
     LumisectionFilters,
-    RunFilters
+    RunFilters,
+    MLBadLumisectionFilters
 )
 
+dials.dataset_index.list_all(DatasetIndexFilters(page_size=500))
+
 dials.file_index.list(FileIndexFilters(dataset__regex="2024B"))
 
 dials.h1d.list(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"))
@@ -139,6 +153,19 @@ dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number
 dials.lumi.list_all(LumisectionFilters(run_number=360392), max_pages=5)
 
 dials.run.list_all(RunFilters(run_number__gte=360392, run_number__lte=365000), max_pages=5)
+
+# jetmet workspace
+dials.ml_models_index.list_all(MLModelsIndexFilters(active=True))
+
+# jetmet workspace
+dials.ml_bad_lumis.list_all(
+    MLBadLumisectionFilters(
+        page_size=500,
+        model_id__in=[20,19],
+        dataset_id__in=[15042670],
+        run_number__in=[385801,385799,385764]
+    )
+)
 ```
 
 ### Dials MEs
@@ -173,6 +200,75 @@ Progress: 100%|█████████████████████
 
 The total attribute of the bar is dynamically updated while fetching the pages.
 
+## Retrying
+
+In case of an unstable connection, DNS failures or service overload it is possible to configure any `get`, `list` and  `list_all` to retry the underlying requests using native `urllib3` retry class, for example:
+
+```python
+from urllib3.util import Retry
+
+data = dials.h1d.get(dataset_id=14677060, run_number=367112, ls_number=10, me_id=1, retries=Retry(total=3, backoff_factor=0.1))
+data = dials.h1d.list(retries=Retry(total=3, backoff_factor=0.1))
+data = dials.h1d.list_all(LumisectionHistogram1DFilters(), max_pages=5, retries=Retry(total=5, backoff_factor=0.1))
+```
+
+## Resuming
+
+When listing and endpoint for a long time, you may loose connection or the server can potentially return an error. By specifying `keep_failed` and using `resume_from` you can resume from an older response object, for example:
+
+```python
+from cmsdials.auth.client import AuthClient
+from cmsdials.auth.bearer import Credentials
+from cmsdials import Dials
+from cmsdials.filters import LumisectionHistogram2DFilters
+
+auth = AuthClient()
+creds = Credentials.from_creds_file()
+dials = Dials(creds, workspace="tracker")
+
+data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), keep_failed=True)  # Code may broke inside this routine
+print(len(data.results))  # 100
+print(data.exc_type)
+print(data.exc_formatted)
+
+# After failing, run this again
+data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), keep_failed=True, resume_from=data)  # Resume from failed object
+print(len(data.results))  # 200
+```
+
+You may find it useful to resume from a partial response, i.e,
+
+```python
+data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), max_pages=10, keep_failed=True)  # Code may not break, however will fetch only 100 elements
+print(len(data.results))  # 100
+
+# After fetching first 100, fetch next 100
+data = dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), max_pages=10, keep_failed=True, resume_from=data)  # Resume from partial object
+print(len(data.results))  # 200
+```
+
+## Fetching ML certification json and ML golden-like json
+
+The following examples are testable in the `jetmet` workspace:
+
+```python
+dials.ml_bad_lumis.cert_json(
+    model_id__in=[20,19],
+    dataset_id__in=[15042670],
+    run_number__in=[385801,385799,385764]
+)
+
+dials.ml_bad_lumis.golden_json(
+    model_id__in=[20,19],
+    dataset_id__in=[15042670],
+    run_number__in=[385801,385799,385764]
+)
+```
+
+You may need to query the `ml_models_index` client to fetch the models ids you are interested and the `dataset-index` client to fetch the datasets ids. Take a look in the live documentation to check all possible filters.
+
+Attention: Those endpoints doesn't return a Pydantic model, instead they are returning a plain json response. Consequently, the method `to_pandas` doesn't work on them.
+
 ## Usage with local DIALS
 
 All classes that interface the DIALS service inherits the class `BaseAPIClient` which propagate the `base_url`, `route` and `version` attributes with production values. In order to use dials-py with a local version of DIALS it is possible to overwrite those attributes when instantiating the `AuthClient` and the `Dials` client, for example:

cmsdials-1.5.0/cmsdials/clients/dataset_index/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import DatasetIndex, DatasetIndexFilters, PaginatedDatasetIndexList
+
+
+class DatasetIndexClient(BaseAuthorizedAPIClient):
+    data_model = DatasetIndex
+    pagination_model = PaginatedDatasetIndexList
+    filter_class = DatasetIndexFilters
+    lookup_url = "dataset-index/"
+
+    def get(self, dataset_id: int, file_id: int, **kwargs):
+        edp = f"{dataset_id}/{file_id}/"
+        return super().get(edp, **kwargs)

cmsdials-1.5.0/cmsdials/clients/dataset_index/models.py

@@ -0,0 +1,30 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel, Field
+
+from ...utils.base_model import OBaseModel, PaginatedBaseModel
+
+
+class DatasetIndex(BaseModel):
+    dataset_id: int
+    dataset: str = Field(..., max_length=255)
+    era: str = Field(..., max_length=255)
+    data_tier: str = Field(..., max_length=255)
+    primary_ds_name: str = Field(..., max_length=255)
+    processed_ds_name: str = Field(..., max_length=255)
+    processing_version: int
+    last_modification_date: datetime
+
+
+class PaginatedDatasetIndexList(PaginatedBaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[DatasetIndex]
+
+
+class DatasetIndexFilters(OBaseModel):
+    next_token: Optional[str] = None
+    page_size: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None

cmsdials-1.5.0/cmsdials/clients/file_index/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import FileIndex, FileIndexFilters, PaginatedFileIndexList
+
+
+class FileIndexClient(BaseAuthorizedAPIClient):
+    data_model = FileIndex
+    pagination_model = PaginatedFileIndexList
+    filter_class = FileIndexFilters
+    lookup_url = "file-index/"
+
+    def get(self, dataset_id: int, file_id: int, **kwargs):
+        edp = f"{dataset_id}/{file_id}/"
+        return super().get(edp, **kwargs)

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/clients/file_index/models.py

@@ -26,6 +26,7 @@ class PaginatedFileIndexList(PaginatedBaseModel):
 
 class FileIndexFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     logical_file_name: Optional[str] = None
     logical_file_name__regex: Optional[str] = None

cmsdials-1.5.0/cmsdials/clients/h1d/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import LumisectionHistogram1D, LumisectionHistogram1DFilters, PaginatedLumisectionHistogram1DList
+
+
+class LumisectionHistogram1DClient(BaseAuthorizedAPIClient):
+    data_model = LumisectionHistogram1D
+    pagination_model = PaginatedLumisectionHistogram1DList
+    filter_class = LumisectionHistogram1DFilters
+    lookup_url = "th1/"
+
+    def get(self, dataset_id: int, run_number: int, ls_number: int, me_id: int, **kwargs):
+        edp = f"{dataset_id}/{run_number}/{ls_number}/{me_id}/"
+        return super().get(edp, **kwargs)

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/clients/h1d/models.py

@@ -28,6 +28,7 @@ class PaginatedLumisectionHistogram1DList(PaginatedBaseModel):
 
 class LumisectionHistogram1DFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     file_id: Optional[int] = None
     run_number: Optional[int] = None

cmsdials-1.5.0/cmsdials/clients/h2d/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import LumisectionHistogram2D, LumisectionHistogram2DFilters, PaginatedLumisectionHistogram2DList
+
+
+class LumisectionHistogram2DClient(BaseAuthorizedAPIClient):
+    data_model = LumisectionHistogram2D
+    pagination_model = PaginatedLumisectionHistogram2DList
+    filter_class = LumisectionHistogram2DFilters
+    lookup_url = "th2/"
+
+    def get(self, dataset_id: int, run_number: int, ls_number: int, me_id: int, **kwargs):
+        edp = f"{dataset_id}/{run_number}/{ls_number}/{me_id}/"
+        return super().get(edp, **kwargs)

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/clients/h2d/models.py

@@ -31,6 +31,7 @@ class PaginatedLumisectionHistogram2DList(PaginatedBaseModel):
 
 class LumisectionHistogram2DFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     file_id: Optional[int] = None
     run_number: Optional[int] = None

cmsdials-1.5.0/cmsdials/clients/lumisection/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import Lumisection, LumisectionFilters, PaginatedLumisectionList
+
+
+class LumisectionClient(BaseAuthorizedAPIClient):
+    data_model = Lumisection
+    pagination_model = PaginatedLumisectionList
+    filter_class = LumisectionFilters
+    lookup_url = "lumisection/"
+
+    def get(self, dataset_id: int, run_number: int, ls_number: int, **kwargs):
+        edp = f"{dataset_id}/{run_number}/{ls_number}/"
+        return super().get(edp, **kwargs)

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/clients/lumisection/models.py

@@ -22,6 +22,7 @@ class PaginatedLumisectionList(PaginatedBaseModel):
 
 class LumisectionFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     run_number: Optional[int] = None
     run_number__lte: Optional[int] = None

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/clients/mes/client.py

@@ -6,3 +6,7 @@ class MonitoringElementClient(BaseAuthorizedAPIClient):
     data_model = MonitoringElement
     filter_class = MEFilters
     lookup_url = "mes/"
+
+    def get(self, id: int, **kwargs):  # noqa: A002
+        edp = f"{id}/"
+        return super().get(edp, **kwargs)

cmsdials-1.5.0/cmsdials/clients/ml_bad_lumisection/client.py

@@ -0,0 +1,29 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import MLBadLumisection, MLBadLumisectionFilters, PaginatedMLBadLumisectionList
+
+
+class MLBadLumisectionClient(BaseAuthorizedAPIClient):
+    data_model = MLBadLumisection
+    pagination_model = PaginatedMLBadLumisectionList
+    filter_class = MLBadLumisectionFilters
+    lookup_url = "ml-bad-lumisection/"
+
+    def get(self, model_id: int, dataset_id: int, run_number: int, ls_number: int, me_id: int, **kwargs):
+        edp = f"{model_id}/{dataset_id}/{run_number}/{ls_number}/{me_id}/"
+        return super().get(edp, **kwargs)
+
+    def cert_json(self, model_id__in: list[int], dataset_id__in: list[int], run_number__in: list[int], **kwargs):
+        edp = "cert-json/"
+        midin = ",".join(str(v) for v in model_id__in)
+        didin = ",".join(str(v) for v in dataset_id__in)
+        ridin = ",".join(str(v) for v in run_number__in)
+        params = {"model_id__in": midin, "dataset_id__in": didin, "run_number__in": ridin}
+        return super().get(edp, params=params, return_raw_json=True, **kwargs)
+
+    def golden_json(self, model_id__in: list[int], dataset_id__in: list[int], run_number__in: list[int], **kwargs):
+        edp = "golden-json/"
+        midin = ",".join(str(v) for v in model_id__in)
+        didin = ",".join(str(v) for v in dataset_id__in)
+        ridin = ",".join(str(v) for v in run_number__in)
+        params = {"model_id__in": midin, "dataset_id__in": didin, "run_number__in": ridin}
+        return super().get(edp, params=params, return_raw_json=True, **kwargs)

cmsdials-1.5.0/cmsdials/clients/ml_bad_lumisection/models.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel
+
+from ...utils.base_model import OBaseModel, PaginatedBaseModel
+
+
+class MLBadLumisection(BaseModel):
+    model_id: int
+    dataset_id: int
+    file_id: int
+    run_number: int
+    ls_number: int
+    me_id: int
+    mse: float
+
+
+class PaginatedMLBadLumisectionList(PaginatedBaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[MLBadLumisection]
+
+
+class MLBadLumisectionFilters(OBaseModel):
+    next_token: Optional[str] = None
+    page_size: Optional[int] = None
+    model_id: Optional[int] = None
+    model_id__in: Optional[list[int]] = None
+    dataset_id: Optional[int] = None
+    dataset_id__in: Optional[list[int]] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None
+    me_id: Optional[int] = None
+    me: Optional[str] = None
+    me__regex: Optional[str] = None
+    run_number: Optional[int] = None
+    run_number__in: Optional[list[int]] = None
+    ls_number: Optional[int] = None

cmsdials-1.5.0/cmsdials/clients/ml_models_index/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import MLModelsIndex, MLModelsIndexFilters, PaginatedMLModelsIndexList
+
+
+class MLModelsIndexClient(BaseAuthorizedAPIClient):
+    data_model = MLModelsIndex
+    pagination_model = PaginatedMLModelsIndexList
+    filter_class = MLModelsIndexFilters
+    lookup_url = "ml-models-index/"
+
+    def get(self, model_id: int, **kwargs):
+        edp = f"{model_id}/"
+        return super().get(edp, **kwargs)

cmsdials-1.5.0/cmsdials/clients/ml_models_index/models.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel
+
+from ...utils.base_model import OBaseModel, PaginatedBaseModel
+
+
+class MLModelsIndex(BaseModel):
+    model_id: int
+    filename: str
+    target_me: str
+    active: bool
+
+
+class PaginatedMLModelsIndexList(PaginatedBaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[MLModelsIndex]
+
+
+class MLModelsIndexFilters(OBaseModel):
+    next_token: Optional[str] = None
+    page_size: Optional[int] = None
+    model_id: Optional[int] = None
+    model_id__in: Optional[list[int]] = None
+    filename: Optional[str] = None
+    filename__regex: Optional[str] = None
+    target_me: Optional[str] = None
+    target_me__regex: Optional[str] = None
+    active: Optional[bool] = None

cmsdials-1.5.0/cmsdials/clients/run/client.py

@@ -0,0 +1,13 @@
+from ...utils.api_client import BaseAuthorizedAPIClient
+from .models import PaginatedRunList, Run, RunFilters
+
+
+class RunClient(BaseAuthorizedAPIClient):
+    data_model = Run
+    pagination_model = PaginatedRunList
+    filter_class = RunFilters
+    lookup_url = "run/"
+
+    def get(self, dataset_id: int, run_number: int, **kwargs):
+        edp = f"{dataset_id}/{run_number}/"
+        return super().get(edp, **kwargs)

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/clients/run/models.py

@@ -20,6 +20,7 @@ class PaginatedRunList(PaginatedBaseModel):
 
 class RunFilters(OBaseModel):
     next_token: Optional[str] = None
+    page_size: Optional[int] = None
     dataset_id: Optional[int] = None
     run_number: Optional[int] = None
     run_number__lte: Optional[int] = None

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/cmsdials.py

@@ -1,17 +1,21 @@
 from typing import Optional
 
 from .auth._base import BaseCredentials
+from .clients.dataset_index.client import DatasetIndexClient
 from .clients.file_index.client import FileIndexClient
 from .clients.h1d.client import LumisectionHistogram1DClient
 from .clients.h2d.client import LumisectionHistogram2DClient
 from .clients.lumisection.client import LumisectionClient
 from .clients.mes.client import MonitoringElementClient
+from .clients.ml_bad_lumisection.client import MLBadLumisectionClient
+from .clients.ml_models_index.client import MLModelsIndexClient
 from .clients.oms_proxy.client import OMSProxyClient
 from .clients.run.client import RunClient
 
 
 class Dials:
     def __init__(self, creds: BaseCredentials, workspace: Optional[str] = None, *args, **kwargs) -> None:
+        self.dataset_index = DatasetIndexClient(creds, workspace, *args, **kwargs)
         self.file_index = FileIndexClient(creds, workspace, *args, **kwargs)
         self.h1d = LumisectionHistogram1DClient(creds, workspace, *args, **kwargs)
         self.h2d = LumisectionHistogram2DClient(creds, workspace, *args, **kwargs)
@@ -19,3 +23,5 @@ class Dials:
         self.run = RunClient(creds, workspace, *args, **kwargs)
         self.mes = MonitoringElementClient(creds, workspace, *args, **kwargs)
         self.oms = OMSProxyClient(creds, *args, **kwargs)
+        self.ml_bad_lumis = MLBadLumisectionClient(creds, workspace, *args, **kwargs)
+        self.ml_models_index = MLModelsIndexClient(creds, workspace, *args, **kwargs)

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/filters.py

@@ -1,19 +1,25 @@
+from .clients.dataset_index.models import DatasetIndexFilters
 from .clients.file_index.models import FileIndexFilters
 from .clients.h1d.models import LumisectionHistogram1DFilters
 from .clients.h2d.models import LumisectionHistogram2DFilters
 from .clients.lumisection.models import LumisectionFilters
 from .clients.mes.models import MEFilters
+from .clients.ml_bad_lumisection.models import MLBadLumisectionFilters
+from .clients.ml_models_index.models import MLModelsIndexFilters
 from .clients.oms_proxy.models import OMSFilter, OMSPage
 from .clients.run.models import RunFilters
 
 
 __all__ = [
+    "DatasetIndexFilters",
     "FileIndexFilters",
     "LumisectionHistogram1DFilters",
     "LumisectionHistogram2DFilters",
     "LumisectionFilters",
     "RunFilters",
     "MEFilters",
+    "MLBadLumisectionFilters",
+    "MLModelsIndexFilters",
     "OMSFilter",
     "OMSPage",
 ]

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/utils/api_client.py

@@ -1,8 +1,11 @@
 from importlib import util as importlib_util
+from traceback import format_exc
 from typing import Optional
 from urllib.parse import parse_qs, urlparse
+from warnings import warn
 
-import requests
+from requests import Response, Session
+from requests.adapters import DEFAULT_RETRIES, HTTPAdapter
 from requests.exceptions import HTTPError
 
 from ..auth._base import BaseCredentials
@@ -35,6 +38,19 @@ class BaseAPIClient:
             return value + "/"
         return value
 
+    @classmethod
+    def _requests_get_retriable(cls, *args, retries=DEFAULT_RETRIES, **kwargs) -> Response:
+        """
+        requests.get() with an additional `retries` parameter.
+
+        Specify retries=<number of attempts - 1> for simple use cases.
+        For advanced usage, see https://docs.python-requests.org/en/latest/user/advanced/
+        """
+        with Session() as s:
+            s.mount(cls.PRODUCTION_BASE_URL, HTTPAdapter(max_retries=retries))
+            ret = s.get(*args, **kwargs)
+        return ret
+
     @property
     def api_url(self):
         return self.base_url + self.route + self.version
@@ -65,10 +81,16 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
         self.creds.before_request(base)
         return base
 
-    def get(self, id: int):  # noqa: A002
-        endpoint_url = self.api_url + self.lookup_url + str(id) + "/"
+    def get(self, edp: str, retries=DEFAULT_RETRIES, params: Optional[dict] = None, return_raw_json: bool = False):
+        endpoint_url = self.api_url + self.lookup_url + edp
         headers = self._build_headers()
-        response = requests.get(endpoint_url, headers=headers, timeout=self.default_timeout)
+        response = self._requests_get_retriable(
+            endpoint_url,
+            headers=headers,
+            params=params,
+            timeout=self.default_timeout,
+            retries=retries,
+        )
 
         try:
             response.raise_for_status()
@@ -77,13 +99,21 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
             raise err
 
         response = response.json()
+        if return_raw_json:
+            return response
         return self.data_model(**response)
 
-    def list(self, filters=None):
+    def list(self, filters=None, retries=DEFAULT_RETRIES):
         filters = filters or self.filter_class()
         endpoint_url = self.api_url + self.lookup_url
         headers = self._build_headers()
-        response = requests.get(endpoint_url, headers=headers, params=filters.cleandict(), timeout=self.default_timeout)
+        response = self._requests_get_retriable(
+            endpoint_url,
+            headers=headers,
+            params=filters.cleandict(),
+            timeout=self.default_timeout,
+            retries=retries,
+        )
 
         try:
             response.raise_for_status()
@@ -99,10 +129,30 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
 
         raise ValueError("pagination model is None and response is not a list.")
 
-    def __list_sync(self, filters, max_pages: Optional[int] = None, enable_progress: bool = False):
-        next_token = None
+    def __list_sync(
+        self,
+        filters,
+        max_pages: Optional[int] = None,
+        enable_progress: bool = False,
+        retries=DEFAULT_RETRIES,
+        keep_failed: bool = False,
+        resume_from=None,
+    ):
+        next_string: Optional[str] = None
         results = []
         is_last_page = False
+
+        if resume_from is not None:
+            results = resume_from.results
+            next_string = resume_from.next
+            if next_string is None and len(results):
+                warn(
+                    "resume_from.next is None while resume_from.result is not empty, doing nothing.",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
+                is_last_page = True
+
         total_pages = 0
         use_tqdm = TQDM_INSTALLED and enable_progress
 
@@ -110,12 +160,34 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
             progress = tqdm(desc="Progress", total=1)
 
         while is_last_page is False:
+            next_token = parse_qs(urlparse(next_string).query).get("next_token") if next_string else None
             curr_filters = self.filter_class(**filters.dict())
             curr_filters.next_token = next_token
-            response = self.list(curr_filters)
+            try:
+                response = self.list(curr_filters, retries=retries)
+            except Exception as e:  # noqa: BLE001
+                if use_tqdm:
+                    progress.close()
+
+                if not keep_failed:
+                    raise e
+
+                exc_formatted = format_exc()
+                warn(
+                    "HTTP request failed, returning partial results. Exception: " + exc_formatted,
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
+                return self.pagination_model(
+                    next=next_string,
+                    previous=None,
+                    results=results,
+                    exc_type=e.__class__.__name__,
+                    exc_formatted=exc_formatted,
+                )
             results.extend(response.results)
-            is_last_page = response.next is None
-            next_token = parse_qs(urlparse(response.next).query).get("next_token") if response.next else None
+            next_string = response.next
+            is_last_page = next_string is None
             total_pages += 1
             max_pages_reached = max_pages and total_pages >= max_pages
             if use_tqdm:
@@ -131,12 +203,12 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
             progress.close()
 
         return self.pagination_model(
-            next=None,
+            next=next_string,
             previous=None,
             results=results,  # No problem re-using last response count
         )
 
     def list_all(self, filters, max_pages: Optional[int] = None, **kwargs):
         if self.pagination_model is None:
-            return self.list(filters)
+            return self.list(filters, retries=kwargs.get("retries", DEFAULT_RETRIES))
         return self.__list_sync(filters, max_pages, **kwargs)

{cmsdials-1.3.0 → cmsdials-1.5.0}/cmsdials/utils/base_model.py

@@ -1,4 +1,5 @@
 from importlib import util as importlib_util
+from typing import Optional
 
 from pydantic import BaseModel
 
@@ -13,10 +14,17 @@ else:
 
 class OBaseModel(BaseModel):
     def cleandict(self):
-        return {key: value for key, value in self.dict().items() if value is not None}
+        return {
+            key: ",".join(str(v) for v in value) if isinstance(value, (list, tuple)) else value
+            for key, value in self.dict().items()
+            if value is not None
+        }
 
 
 class PaginatedBaseModel(BaseModel):
+    exc_type: Optional[str] = None
+    exc_formatted: Optional[str] = None
+
     def to_pandas(self):
         if PANDAS_NOT_INSTALLED:
             raise RuntimeError(

{cmsdials-1.3.0 → cmsdials-1.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cmsdials"
-version = "1.3.0"
+version = "1.5.0"
 description = "The Python api client interface to DIALS service"
 authors = ["Gabriel Moreira <gabrielmscampos@gmail.com>"]
 readme = "README.md"
@@ -23,6 +23,7 @@ pydantic = "<2, >=1"
 typing-extensions = "<4.6.0, >=3.6.6"
 pandas = {version = ">=1.5.3", optional = true}
 tqdm = {version = ">=4.66.1", optional = true}
+urllib3 = "^2.2.3"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^3.6.2"

cmsdials-1.3.0/cmsdials/clients/file_index/client.py

@@ -1,27 +0,0 @@
-import requests
-from requests.exceptions import HTTPError
-
-from ...utils.api_client import BaseAuthorizedAPIClient
-from ...utils.logger import logger
-from .models import FileIndex, FileIndexFilters, PaginatedFileIndexList
-
-
-class FileIndexClient(BaseAuthorizedAPIClient):
-    data_model = FileIndex
-    pagination_model = PaginatedFileIndexList
-    filter_class = FileIndexFilters
-    lookup_url = "file-index/"
-
-    def get(self, dataset_id: int, file_id: int):
-        endpoint_url = f"{self.api_url}{self.lookup_url}{dataset_id}/{file_id}/"
-        headers = self._build_headers()
-        response = requests.get(endpoint_url, headers=headers, timeout=self.default_timeout)
-
-        try:
-            response.raise_for_status()
-        except HTTPError as err:
-            logger.info(f"Api raw response: {response.text}")
-            raise err
-
-        response = response.json()
-        return self.data_model(**response)

cmsdials-1.3.0/cmsdials/clients/h1d/client.py

@@ -1,27 +0,0 @@
-import requests
-from requests.exceptions import HTTPError
-
-from ...utils.api_client import BaseAuthorizedAPIClient
-from ...utils.logger import logger
-from .models import LumisectionHistogram1D, LumisectionHistogram1DFilters, PaginatedLumisectionHistogram1DList
-
-
-class LumisectionHistogram1DClient(BaseAuthorizedAPIClient):
-    data_model = LumisectionHistogram1D
-    pagination_model = PaginatedLumisectionHistogram1DList
-    filter_class = LumisectionHistogram1DFilters
-    lookup_url = "th1/"
-
-    def get(self, dataset_id: int, run_number: int, ls_number: int, me_id: int):
-        endpoint_url = f"{self.api_url}{self.lookup_url}{dataset_id}/{run_number}/{ls_number}/{me_id}/"
-        headers = self._build_headers()
-        response = requests.get(endpoint_url, headers=headers, timeout=self.default_timeout)
-
-        try:
-            response.raise_for_status()
-        except HTTPError as err:
-            logger.info(f"Api raw response: {response.text}")
-            raise err
-
-        response = response.json()
-        return self.data_model(**response)

cmsdials-1.3.0/cmsdials/clients/h2d/client.py

@@ -1,27 +0,0 @@
-import requests
-from requests.exceptions import HTTPError
-
-from ...utils.api_client import BaseAuthorizedAPIClient
-from ...utils.logger import logger
-from .models import LumisectionHistogram2D, LumisectionHistogram2DFilters, PaginatedLumisectionHistogram2DList
-
-
-class LumisectionHistogram2DClient(BaseAuthorizedAPIClient):
-    data_model = LumisectionHistogram2D
-    pagination_model = PaginatedLumisectionHistogram2DList
-    filter_class = LumisectionHistogram2DFilters
-    lookup_url = "th2/"
-
-    def get(self, dataset_id: int, run_number: int, ls_number: int, me_id: int):
-        endpoint_url = f"{self.api_url}{self.lookup_url}{dataset_id}/{run_number}/{ls_number}/{me_id}/"
-        headers = self._build_headers()
-        response = requests.get(endpoint_url, headers=headers, timeout=self.default_timeout)
-
-        try:
-            response.raise_for_status()
-        except HTTPError as err:
-            logger.info(f"Api raw response: {response.text}")
-            raise err
-
-        response = response.json()
-        return self.data_model(**response)

cmsdials-1.3.0/cmsdials/clients/lumisection/client.py

@@ -1,27 +0,0 @@
-import requests
-from requests.exceptions import HTTPError
-
-from ...utils.api_client import BaseAuthorizedAPIClient
-from ...utils.logger import logger
-from .models import Lumisection, LumisectionFilters, PaginatedLumisectionList
-
-
-class LumisectionClient(BaseAuthorizedAPIClient):
-    data_model = Lumisection
-    pagination_model = PaginatedLumisectionList
-    filter_class = LumisectionFilters
-    lookup_url = "lumisection/"
-
-    def get(self, dataset_id: int, run_number: int, ls_number: int):
-        endpoint_url = f"{self.api_url}{self.lookup_url}{dataset_id}/{run_number}/{ls_number}/"
-        headers = self._build_headers()
-        response = requests.get(endpoint_url, headers=headers, timeout=self.default_timeout)
-
-        try:
-            response.raise_for_status()
-        except HTTPError as err:
-            logger.info(f"Api raw response: {response.text}")
-            raise err
-
-        response = response.json()
-        return self.data_model(**response)

cmsdials-1.3.0/cmsdials/clients/run/client.py

@@ -1,27 +0,0 @@
-import requests
-from requests.exceptions import HTTPError
-
-from ...utils.api_client import BaseAuthorizedAPIClient
-from ...utils.logger import logger
-from .models import PaginatedRunList, Run, RunFilters
-
-
-class RunClient(BaseAuthorizedAPIClient):
-    data_model = Run
-    pagination_model = PaginatedRunList
-    filter_class = RunFilters
-    lookup_url = "run/"
-
-    def get(self, dataset_id: int, run_number: int):
-        endpoint_url = f"{self.api_url}{self.lookup_url}{dataset_id}/{run_number}/"
-        headers = self._build_headers()
-        response = requests.get(endpoint_url, headers=headers, timeout=self.default_timeout)
-
-        try:
-            response.raise_for_status()
-        except HTTPError as err:
-            logger.info(f"Api raw response: {response.text}")
-            raise err
-
-        response = response.json()
-        return self.data_model(**response)