cmsdials 0.3.0__tar.gz → 1.0.0__tar.gz

{cmsdials-0.3.0 → cmsdials-1.0.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: cmsdials
-Version: 0.3.0
+Version: 1.0.0
 Summary: The Python api client interface to DIALS service
 Home-page: https://github.com/cms-DQM/dials-py
 Author: Gabriel Moreira
 Author-email: gabrielmscampos@gmail.com
 Requires-Python: >=3.9,<4.0
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
@@ -76,10 +76,18 @@ from cmsdials import Dials
 from cmsdials.filters import LumisectionHistogram1DFilters
 
 creds = Credentials.from_creds_file()
-dials = Dials(creds, nthreads=2)
+dials = Dials(creds)
 
 # Getting h1d data
-data = dials.h1d.list_all(LumisectionHistogram1DFilters(title="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"), max_pages=5)
+data = dials.h1d.list_all(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"), max_pages=5)
+```
+
+### Workspace
+
+Users are automatically routed to a workspace based on e-groups, but it is possible to overwrite this configuration and inspect data from others workspaces:
+
+```python
+dials = Dials(creds, workspace="jetmet")
 ```
 
 ## Available endpoints
@@ -89,12 +97,11 @@ This package interacts with DIALS api endpoints using underlying classes in `Dia
 ### Retrieving a specific object using `get`
 
 ```python
-dials.bad_file_index.get(id=1)
-dials.file_index.get(id=1)
+dials.file_index.get(id=3386119397)
 dials.h1d.get(id=1)
 dials.h2d.get(id=1)
-dials.lumi.get(id=1)
-dials.run.get(id=1)
+dials.lumi.get(dataset_id=14677060, run_number=367094, ls_number=1)
+dials.run.get(dataset_id=14677060, run_number=367094)
 ```
 
 ### Retrieving a list of objects per page using `list`
@@ -102,7 +109,6 @@ dials.run.get(id=1)
 It is possible to get a list of entries from those endpoint using the `list` and `list_all` methods, the `list` method will fetch only one page and the `list_all` will fetch all available pages:
 
 ```python
-dials.bad_file_index.list()
 dials.file_index.list()
 dials.h1d.list()
 dials.h2d.list()
@@ -112,8 +118,9 @@ dials.run.list()
 
 ### Retrieving all available pages of a list of objects using `list_all`
 
+Note: Keep in mind that running `list_all` without any filter can take too much time, since you will be retrieving all rows in the database.
+
 ```python
-dials.bad_file_index.list_all()
 dials.file_index.list_all()
 dials.h1d.list_all()
 dials.h2d.list_all()
@@ -133,7 +140,6 @@ Keep in mind that calling those methods without any filter can take a lot of tim
 
 ```python
 from cmsdials.filters import (
-    BadFileIndexFilters,
     FileIndexFilters,
     LumisectionHistogram1DFilters,
     LumisectionHistogram2DFilters,
@@ -141,17 +147,15 @@ from cmsdials.filters import (
     RunFilters
 )
 
-dials.bad_file_index.list(BadFileIndexFilters(path_contains="ZeroBias"))
-
-dials.file_index.list(FileIndexFilters(page="10"))
+dials.file_index.list(FileIndexFilters(dataset__regex="2024B"))
 
-dials.h1d.list(LumisectionHistogram1DFilters(title="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2", page="15"))
+dials.h1d.list(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"))
 
-dials.h2d.list_all(LumisectionHistogram2DFilters(title_contains="EEOT digi occupancy EE +", min_entries=100, min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), max_pages=5)
 
 dials.lumi.list_all(LumisectionFilters(run_number=360392), max_pages=5)
 
-dials.run.list_all(RunFilters(min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.run.list_all(RunFilters(run_number__gte=360392, run_number__lte=365000), max_pages=5)
 ```
 
 ### Dials MEs
@@ -173,12 +177,12 @@ from cmsdials import Dials
 from cmsdials.filters import LumisectionHistogram2DFilters
 
 DEV_URL = "http://localhost:8000/"
-DEV_CACHE_DIR = ".cache-dev
+DEV_CACHE_DIR = ".cache-dev"
 
 auth = AuthClient(base_url=DEV_URL)
 creds = Credentials.from_creds_file(cache_dir=DEV_CACHE_DIR, client=auth)  # Make sure to specify the auth client with overwritten values, using another cache_dir is recommended
-dials = Dials(creds, nthreads=2, base_url=DEV_URL)
+dials = Dials(creds, base_url=DEV_URL)
 
-dials.h2d.list_all(LumisectionHistogram2DFilters(title_contains="EEOT digi occupancy EE +", min_entries=100, min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="EEOT digi occupancy EE +", entries__gte=100, run_number__gte=360392, run_number__lte=365000), max_pages=5)
 ```
 

{cmsdials-0.3.0 → cmsdials-1.0.0}/README.md

@@ -53,10 +53,18 @@ from cmsdials import Dials
 from cmsdials.filters import LumisectionHistogram1DFilters
 
 creds = Credentials.from_creds_file()
-dials = Dials(creds, nthreads=2)
+dials = Dials(creds)
 
 # Getting h1d data
-data = dials.h1d.list_all(LumisectionHistogram1DFilters(title="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"), max_pages=5)
+data = dials.h1d.list_all(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"), max_pages=5)
+```
+
+### Workspace
+
+Users are automatically routed to a workspace based on e-groups, but it is possible to overwrite this configuration and inspect data from others workspaces:
+
+```python
+dials = Dials(creds, workspace="jetmet")
 ```
 
 ## Available endpoints
@@ -66,12 +74,11 @@ This package interacts with DIALS api endpoints using underlying classes in `Dia
 ### Retrieving a specific object using `get`
 
 ```python
-dials.bad_file_index.get(id=1)
-dials.file_index.get(id=1)
+dials.file_index.get(id=3386119397)
 dials.h1d.get(id=1)
 dials.h2d.get(id=1)
-dials.lumi.get(id=1)
-dials.run.get(id=1)
+dials.lumi.get(dataset_id=14677060, run_number=367094, ls_number=1)
+dials.run.get(dataset_id=14677060, run_number=367094)
 ```
 
 ### Retrieving a list of objects per page using `list`
@@ -79,7 +86,6 @@ dials.run.get(id=1)
 It is possible to get a list of entries from those endpoint using the `list` and `list_all` methods, the `list` method will fetch only one page and the `list_all` will fetch all available pages:
 
 ```python
-dials.bad_file_index.list()
 dials.file_index.list()
 dials.h1d.list()
 dials.h2d.list()
@@ -89,8 +95,9 @@ dials.run.list()
 
 ### Retrieving all available pages of a list of objects using `list_all`
 
+Note: Keep in mind that running `list_all` without any filter can take too much time, since you will be retrieving all rows in the database.
+
 ```python
-dials.bad_file_index.list_all()
 dials.file_index.list_all()
 dials.h1d.list_all()
 dials.h2d.list_all()
@@ -110,7 +117,6 @@ Keep in mind that calling those methods without any filter can take a lot of tim
 
 ```python
 from cmsdials.filters import (
-    BadFileIndexFilters,
     FileIndexFilters,
     LumisectionHistogram1DFilters,
     LumisectionHistogram2DFilters,
@@ -118,17 +124,15 @@ from cmsdials.filters import (
     RunFilters
 )
 
-dials.bad_file_index.list(BadFileIndexFilters(path_contains="ZeroBias"))
-
-dials.file_index.list(FileIndexFilters(page="10"))
+dials.file_index.list(FileIndexFilters(dataset__regex="2024B"))
 
-dials.h1d.list(LumisectionHistogram1DFilters(title="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2", page="15"))
+dials.h1d.list(LumisectionHistogram1DFilters(me="PixelPhase1/Tracks/PXBarrel/charge_PXLayer_2"))
 
-dials.h2d.list_all(LumisectionHistogram2DFilters(title_contains="EEOT digi occupancy EE +", min_entries=100, min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="PXBarrel", ls_number=78, entries__gte=100), max_pages=5)
 
 dials.lumi.list_all(LumisectionFilters(run_number=360392), max_pages=5)
 
-dials.run.list_all(RunFilters(min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.run.list_all(RunFilters(run_number__gte=360392, run_number__lte=365000), max_pages=5)
 ```
 
 ### Dials MEs
@@ -150,11 +154,11 @@ from cmsdials import Dials
 from cmsdials.filters import LumisectionHistogram2DFilters
 
 DEV_URL = "http://localhost:8000/"
-DEV_CACHE_DIR = ".cache-dev
+DEV_CACHE_DIR = ".cache-dev"
 
 auth = AuthClient(base_url=DEV_URL)
 creds = Credentials.from_creds_file(cache_dir=DEV_CACHE_DIR, client=auth)  # Make sure to specify the auth client with overwritten values, using another cache_dir is recommended
-dials = Dials(creds, nthreads=2, base_url=DEV_URL)
+dials = Dials(creds, base_url=DEV_URL)
 
-dials.h2d.list_all(LumisectionHistogram2DFilters(title_contains="EEOT digi occupancy EE +", min_entries=100, min_run_number=360392, max_run_number=365000), max_pages=5)
+dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="EEOT digi occupancy EE +", entries__gte=100, run_number__gte=360392, run_number__lte=365000), max_pages=5)
 ```

cmsdials-1.0.0/cmsdials/clients/file_index/models.py

@@ -0,0 +1,35 @@
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import AnyUrl, BaseModel, Field
+
+from ...utils.base_model import OBaseModel
+
+
+class FileIndex(BaseModel):
+    dataset_id: int
+    dataset: str = Field(..., max_length=255)
+    file_id: int
+    file_size: int
+    creation_date: datetime
+    last_modification_date: datetime
+    logical_file_name: str
+    status: str = Field(..., max_length=15)
+    err_trace: Optional[str] = Field(..., max_length=5000)
+
+
+class PaginatedFileIndexList(BaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: List[FileIndex]
+
+
+class FileIndexFilters(OBaseModel):
+    next_token: Optional[str] = None
+    dataset_id: Optional[int] = None
+    logical_file_name: Optional[str] = None
+    logical_file_name__regex: Optional[str] = None
+    status: Optional[str] = None
+    min_size: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None

{cmsdials-0.3.0 → cmsdials-1.0.0}/cmsdials/clients/h1d/client.py

@@ -6,4 +6,4 @@ class LumisectionHistogram1DClient(BaseAuthorizedAPIClient):
     data_model = LumisectionHistogram1D
     pagination_model = PaginatedLumisectionHistogram1DList
     filter_class = LumisectionHistogram1DFilters
-    lookup_url = "lumisection-h1d/"
+    lookup_url = "th1/"

cmsdials-1.0.0/cmsdials/clients/h1d/models.py

@@ -0,0 +1,49 @@
+from typing import List, Optional
+
+from pydantic import AnyUrl, BaseModel, Field
+
+from ...utils.base_model import OBaseModel
+
+
+class LumisectionHistogram1D(BaseModel):
+    hist_id: int
+    dataset: str = Field(..., max_length=255)
+    me: str = Field(..., max_length=255)
+    dataset_id: int
+    file_id: int
+    run_number: int
+    ls_number: int
+    me_id: int
+    ls_id: int
+    x_min: float
+    x_max: float
+    x_bin: float
+    entries: int
+    data: list[float]
+
+
+class PaginatedLumisectionHistogram1DList(BaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: List[LumisectionHistogram1D]
+
+
+class LumisectionHistogram1DFilters(OBaseModel):
+    next_token: Optional[str] = None
+    dataset_id: Optional[int] = None
+    file_id: Optional[int] = None
+    run_number: Optional[int] = None
+    run_number__lte: Optional[int] = None
+    run_number__gte: Optional[int] = None
+    ls_number: Optional[int] = None
+    ls_numbet__lte: Optional[int] = None
+    ls_number__gte: Optional[int] = None
+    me_id: Optional[int] = None
+    ls_id: Optional[int] = None
+    entries__gte: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None
+    logical_file_name: Optional[str] = None
+    logical_file_name__regex: Optional[str] = None
+    me: Optional[str] = None
+    me__regex: Optional[str] = None

{cmsdials-0.3.0 → cmsdials-1.0.0}/cmsdials/clients/h2d/client.py

@@ -6,4 +6,4 @@ class LumisectionHistogram2DClient(BaseAuthorizedAPIClient):
     data_model = LumisectionHistogram2D
     pagination_model = PaginatedLumisectionHistogram2DList
     filter_class = LumisectionHistogram2DFilters
-    lookup_url = "lumisection-h2d/"
+    lookup_url = "th2/"

cmsdials-1.0.0/cmsdials/clients/h2d/models.py

@@ -0,0 +1,52 @@
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel, Field
+
+from ...utils.base_model import OBaseModel
+
+
+class LumisectionHistogram2D(BaseModel):
+    hist_id: int
+    dataset: str = Field(..., max_length=255)
+    me: str = Field(..., max_length=255)
+    dataset_id: int
+    file_id: int
+    run_number: int
+    ls_number: int
+    me_id: int
+    ls_id: int
+    x_min: float
+    x_max: float
+    x_bin: float
+    y_min: float
+    y_max: float
+    y_bin: float
+    entries: int
+    data: list[list[float]]
+
+
+class PaginatedLumisectionHistogram2DList(BaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[LumisectionHistogram2D]
+
+
+class LumisectionHistogram2DFilters(OBaseModel):
+    next_token: Optional[str] = None
+    dataset_id: Optional[int] = None
+    file_id: Optional[int] = None
+    run_number: Optional[int] = None
+    run_number__lte: Optional[int] = None
+    run_number__gte: Optional[int] = None
+    ls_number: Optional[int] = None
+    ls_numbet__lte: Optional[int] = None
+    ls_number__gte: Optional[int] = None
+    me_id: Optional[int] = None
+    ls_id: Optional[int] = None
+    entries__gte: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None
+    logical_file_name: Optional[str] = None
+    logical_file_name__regex: Optional[str] = None
+    me: Optional[str] = None
+    me__regex: Optional[str] = None

cmsdials-1.0.0/cmsdials/clients/lumisection/client.py

@@ -0,0 +1,27 @@
+import requests
+from requests.exceptions import HTTPError
+
+from ...utils.api_client import BaseAuthorizedAPIClient
+from ...utils.logger import logger
+from .models import Lumisection, LumisectionFilters, PaginatedLumisectionList
+
+
+class LumisectionClient(BaseAuthorizedAPIClient):
+    data_model = Lumisection
+    pagination_model = PaginatedLumisectionList
+    filter_class = LumisectionFilters
+    lookup_url = "lumisection/"
+
+    def get(self, dataset_id: int, run_number: int, ls_number: int):
+        endpoint_url = f"{self.api_url}{self.lookup_url}{dataset_id}/{run_number}/{ls_number}/"
+        headers = self._build_headers()
+        response = requests.get(endpoint_url, headers=headers)
+
+        try:
+            response.raise_for_status()
+        except HTTPError as err:
+            logger.info(f"Api raw response: {response.text}")
+            raise err
+
+        response = response.json()
+        return self.data_model(**response)

cmsdials-1.0.0/cmsdials/clients/lumisection/models.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel, Field
+
+from ...utils.base_model import OBaseModel
+
+
+class Lumisection(BaseModel):
+    dataset_id: int
+    dataset: str = Field(..., max_length=255)
+    run_number: int
+    ls_number: int
+    ls_id: int
+    th1_count: int
+    th2_count: int
+
+
+class PaginatedLumisectionList(BaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[Lumisection]
+
+
+class LumisectionFilters(OBaseModel):
+    next_token: Optional[str] = None
+    dataset_id: Optional[int] = None
+    run_number: Optional[int] = None
+    run_number__lte: Optional[int] = None
+    run_number__gte: Optional[int] = None
+    ls_number: Optional[int] = None
+    ls_number__lte: Optional[int] = None
+    ls_number__gte: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None

cmsdials-1.0.0/cmsdials/clients/run/client.py

@@ -0,0 +1,27 @@
+import requests
+from requests.exceptions import HTTPError
+
+from ...utils.api_client import BaseAuthorizedAPIClient
+from ...utils.logger import logger
+from .models import PaginatedRunList, Run, RunFilters
+
+
+class RunClient(BaseAuthorizedAPIClient):
+    data_model = Run
+    pagination_model = PaginatedRunList
+    filter_class = RunFilters
+    lookup_url = "run/"
+
+    def get(self, dataset_id: int, run_number: int):
+        endpoint_url = f"{self.api_url}{self.lookup_url}{dataset_id}/{run_number}/"
+        headers = self._build_headers()
+        response = requests.get(endpoint_url, headers=headers)
+
+        try:
+            response.raise_for_status()
+        except HTTPError as err:
+            logger.info(f"Api raw response: {response.text}")
+            raise err
+
+        response = response.json()
+        return self.data_model(**response)

cmsdials-1.0.0/cmsdials/clients/run/models.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+from pydantic import AnyUrl, BaseModel, Field
+
+from ...utils.base_model import OBaseModel
+
+
+class Run(BaseModel):
+    dataset_id: int
+    dataset: str = Field(..., max_length=255)
+    run_number: int
+    ls_count: int
+
+
+class PaginatedRunList(BaseModel):
+    next: Optional[AnyUrl]
+    previous: Optional[AnyUrl]
+    results: list[Run]
+
+
+class RunFilters(OBaseModel):
+    next_token: Optional[str] = None
+    dataset_id: Optional[int] = None
+    run_number: Optional[int] = None
+    run_number__lte: Optional[int] = None
+    run_number__gte: Optional[int] = None
+    dataset: Optional[str] = None
+    dataset__regex: Optional[str] = None

cmsdials-1.0.0/cmsdials/cmsdials.py

@@ -0,0 +1,17 @@
+from typing import Optional
+
+from .auth._base import BaseCredentials
+from .clients.file_index.client import FileIndexClient
+from .clients.h1d.client import LumisectionHistogram1DClient
+from .clients.h2d.client import LumisectionHistogram2DClient
+from .clients.lumisection.client import LumisectionClient
+from .clients.run.client import RunClient
+
+
+class Dials:
+    def __init__(self, creds: BaseCredentials, workspace: Optional[str] = None, *args, **kwargs) -> None:
+        self.file_index = FileIndexClient(creds, workspace, *args, **kwargs)
+        self.h1d = LumisectionHistogram1DClient(creds, workspace, *args, **kwargs)
+        self.h2d = LumisectionHistogram2DClient(creds, workspace, *args, **kwargs)
+        self.lumi = LumisectionClient(creds, workspace, *args, **kwargs)
+        self.run = RunClient(creds, workspace, *args, **kwargs)

{cmsdials-0.3.0 → cmsdials-1.0.0}/cmsdials/filters.py

@@ -1,4 +1,3 @@
-from .clients.bad_file_index.models import BadFileIndexFilters
 from .clients.file_index.models import FileIndexFilters
 from .clients.h1d.models import LumisectionHistogram1DFilters
 from .clients.h2d.models import LumisectionHistogram2DFilters
@@ -6,7 +5,6 @@ from .clients.lumisection.models import LumisectionFilters
 from .clients.run.models import RunFilters
 
 __all__ = [
-    "BadFileIndexFilters",
     "FileIndexFilters",
     "LumisectionHistogram1DFilters",
     "LumisectionHistogram2DFilters",

{cmsdials-0.3.0 → cmsdials-1.0.0}/cmsdials/utils/api_client.py

@@ -1,6 +1,5 @@
-from concurrent.futures import ThreadPoolExecutor
-from math import ceil
 from typing import Optional
+from urllib.parse import parse_qs, urlparse
 
 import requests
 from requests.exceptions import HTTPError
@@ -38,15 +37,27 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
     filter_class = None
     lookup_url = None
 
-    def __init__(self, creds: BaseCredentials, nthreads: Optional[int] = None, *args: str, **kwargs: str) -> None:
+    def __init__(
+        self,
+        creds: BaseCredentials,
+        workspace: Optional[str] = None,
+        *args: str,
+        **kwargs: str,
+    ) -> None:
         super().__init__(*args, **kwargs)
         self.creds = creds
-        self.nthreads = nthreads or 1
+        self.workspace = workspace
 
-    def get(self, id: str):
+    def _build_headers(self) -> dict:
+        base = {"accept": "application/json"}
+        if self.workspace is not None:
+            base["Workspace"] = self.workspace
+        self.creds.before_request(base)
+        return base
+
+    def get(self, id: int):
         endpoint_url = self.api_url + self.lookup_url + str(id) + "/"
-        headers = {"accept": "application/json"}
-        self.creds.before_request(headers)
+        headers = self._build_headers()
         response = requests.get(endpoint_url, headers=headers)
 
         try:
@@ -61,8 +72,7 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
     def list(self, filters=None):
         filters = filters or self.filter_class()
         endpoint_url = self.api_url + self.lookup_url
-        headers = {"accept": "application/json"}
-        self.creds.before_request(headers)
+        headers = self._build_headers()
         response = requests.get(endpoint_url, headers=headers, params=filters.cleandict())
 
         try:
@@ -75,54 +85,25 @@ class BaseAuthorizedAPIClient(BaseAPIClient):
         return self.pagination_model(**response)
 
     def __list_sync(self, filters, max_pages: Optional[int] = None):
-        page = 1
+        next_token = None
         results = []
         is_last_page = False
+        total_pages = 0
 
         while is_last_page is False:
             curr_filters = self.filter_class(**filters.dict())
-            curr_filters.page = str(page)
+            curr_filters.next_token = next_token
             response = self.list(curr_filters)
             results.extend(response.results)
             is_last_page = response.next is None
-            page += 1
-            if max_pages and page > max_pages:
+            next_token = parse_qs(urlparse(response.next).query).get("next_token") if response.next else None
+            total_pages += 1
+            if max_pages and total_pages > max_pages:
                 break
 
         return self.pagination_model(
-            count=response.count, next=None, previous=None, results=results  # No problem re-using last response count
+            next=None, previous=None, results=results  # No problem re-using last response count
         )
 
-    def __list_multithreaded(self, filters, max_pages: Optional[int] = None):
-        results = []
-
-        # Request the first page to compute number of pages
-        curr_filters = self.filter_class(**filters.dict())
-        curr_filters.page = "1"
-        response = self.list(curr_filters)
-        results.extend(response.results)
-        count = response.count
-        n_pages = ceil(response.count / 10)
-        n_pages = max_pages if max_pages is not None and n_pages > max_pages else n_pages
-
-        # If there isn't another page, just return current results
-        if response.next is None:
-            return response
-
-        # Request other pages
-        all_filters = [{**filters.dict(), "page": str(page)} for page in range(2, n_pages + 1)]
-        all_filters = [self.filter_class(**filter) for filter in all_filters]
-        with ThreadPoolExecutor(max_workers=self.nthreads) as executor:
-            jobs = executor.map(self.list, all_filters)
-            del all_filters
-            [results.extend(job_result.results) for job_result in jobs]
-            del jobs
-
-        return self.pagination_model(count=count, next=None, previous=None, results=results)
-
     def list_all(self, filters, max_pages: Optional[int] = None):
-        return (
-            self.__list_sync(filters, max_pages)
-            if self.nthreads is None
-            else self.__list_multithreaded(filters, max_pages)
-        )
+        return self.__list_sync(filters, max_pages)

cmsdials-1.0.0/pyproject.toml

@@ -0,0 +1,126 @@
+[tool.poetry]
+name = "cmsdials"
+version = "1.0.0"
+description = "The Python api client interface to DIALS service"
+authors = ["Gabriel Moreira <gabrielmscampos@gmail.com>"]
+readme = "README.md"
+include = ["LICENSE"]
+repository = "https://github.com/cms-DQM/dials-py"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Programming Language :: Python",
+    "Operating System :: OS Independent",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+]
+packages = [
+    { include = "cmsdials" }
+]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+requests = "^2.31.0"
+pydantic = "<2, >=1"
+typing-extensions = "<4.6.0, >=3.6.6"
+
+[tool.poetry.group.dev.dependencies]
+pre-commit = "^3.6.2"
+ruff = "^0.4.2"
+
+[tool.ruff]
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "venv",
+    "virtualenvs",
+]
+line-length = 120
+indent-width = 4
+target-version = "py310"
+
+[tool.ruff.lint]
+select = [
+    # pycodestyle
+    "E",
+    # Pyflakes
+    "F",
+    # pep8-naming
+    "N",
+    # pyupgrade
+    "UP",
+    # flake8-bugbear
+    "B",
+    # flake8-bandit
+    "S",
+    # flake8-blind-except
+    "BLE",
+    # flake8-builtins
+    "A",
+    # isort
+    "I",
+    # flake8-logging-format
+    "G",
+    # flake8-no-pep420
+    "INP",
+    # Ruff-specific rules
+    "RUF"
+]
+ignore = [
+  # Disable eradicate (commented code removal)
+  "ERA001",
+  # Disable Conflicting lint rules,
+  # see https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
+  "W191",
+  "E501",
+  "E111",
+  "E117",
+  "D206",
+  "D300",
+  "Q000",
+  "Q001",
+  "Q002",
+  "Q003",
+  "COM812",
+  "COM819",
+  "ISC001",
+  "ISC002",
+  # Disable unused `noqa` directive
+  "RUF100",
+]
+# Allow unused variables when underscore-prefixed:
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.ruff.lint.pycodestyle]
+ignore-overlong-task-comments = true
+
+[tool.ruff.lint.isort]
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
+lines-after-imports = 2
+
+[tool.ruff.lint.per-file-ignores]
+# Ignore "E402", "F403", "F405" (import violations) in __init__.py files.
+# Ignore "S" (flake8-bandit) and "N802" (function name should be lowercase) in tests and docs.
+# Ignore "RUF" (Ruff-specific rules) and "I" (isort) in migrations.
+"__init__.py" = ["E402", "F403", "F405"]
+"**/{tests,docs}/*" = ["E402", "F403", "F405", "S", "N802"]
+"**/*test*.py" = ["E402", "F403", "F405", "S", "N802"]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

cmsdials-0.3.0/cmsdials/clients/bad_file_index/client.py

@@ -1,9 +0,0 @@
-from ...utils.api_client import BaseAuthorizedAPIClient
-from .models import BadFileIndex, BadFileIndexFilters, PaginatedBadFileIndexList
-
-
-class BadFileIndexClient(BaseAuthorizedAPIClient):
-    data_model = BadFileIndex
-    pagination_model = PaginatedBadFileIndexList
-    filter_class = BadFileIndexFilters
-    lookup_url = "bad-file-index/"

cmsdials-0.3.0/cmsdials/clients/bad_file_index/models.py

@@ -1,30 +0,0 @@
-from datetime import datetime
-from typing import List, Optional
-
-from pydantic import AnyUrl, BaseModel, Field
-
-from ...utils.base_model import OBaseModel
-
-
-class BadFileIndex(BaseModel):
-    id: int
-    file_path: str = Field(..., description="Path where the file is stored", max_length=255)
-    data_era: str = Field(..., description="The era that the data refers to (e.g. 2018A)", max_length=7)
-    st_size: float = Field(..., description="The data file's size in bytes")
-    st_ctime: datetime = Field(..., description="Time of files's last status change in filesystem")
-    st_itime: datetime = Field(..., description="Time when file was indexed in database")
-    err: str = Field(..., description="Error message", max_length=255)
-
-
-class PaginatedBadFileIndexList(BaseModel):
-    count: int
-    next: Optional[AnyUrl]
-    previous: Optional[AnyUrl]
-    results: List[BadFileIndex]
-
-
-class BadFileIndexFilters(OBaseModel):
-    era: Optional[str] = None
-    min_size: Optional[float] = None
-    page: Optional[str] = None
-    path_contains: Optional[str] = None

cmsdials-0.3.0/cmsdials/clients/file_index/models.py

@@ -1,44 +0,0 @@
-from datetime import datetime
-from typing import List, Optional
-
-from pydantic import AnyUrl, BaseModel, Field
-
-from ...utils.base_model import OBaseModel
-
-
-class FileIndex(BaseModel):
-    id: int
-    file_uuid: str = Field(..., description="ROOT TFile UUID", max_length=36)
-    file_path: str = Field(..., description="Path where the file is stored", max_length=255)
-    data_era: str = Field(..., description="The era that the data refers to (e.g. 2018A)", max_length=7)
-    n_entries: int = Field(
-        ...,
-        description="Total number of entries contained in this histogram file",
-    )
-    n_entries_ingested: int = Field(
-        ...,
-        description="Number of histogram entries that have been extracted from the file",
-    )
-    st_size: float = Field(..., description="The data file's size in bytes")
-    st_ctime: datetime = Field(..., description="Time of files's last status change in filesystem")
-    st_itime: datetime = Field(..., description="Time when file was indexed in database")
-    status: str = Field(
-        ...,
-        description="Indicate the processing status of run-histogram within the file",
-        max_length=9,
-    )
-
-
-class PaginatedFileIndexList(BaseModel):
-    count: int
-    next: Optional[AnyUrl]
-    previous: Optional[AnyUrl]
-    results: List[FileIndex]
-
-
-class FileIndexFilters(OBaseModel):
-    era: Optional[str] = None
-    min_size: Optional[float] = None
-    page: Optional[str] = None
-    path_contains: Optional[str] = None
-    status: Optional[str] = None

cmsdials-0.3.0/cmsdials/clients/h1d/models.py

@@ -1,44 +0,0 @@
-from datetime import datetime
-from typing import List, Optional
-
-from pydantic import AnyUrl, BaseModel, Field
-
-from ...utils.base_model import OBaseModel
-
-
-class LumisectionHistogram1D(BaseModel):
-    id: int
-    ls_number: int
-    run_number: int
-    date: datetime
-    title: str = Field(..., max_length=220)
-    entries: int
-    data: list[float]
-    x_min: float
-    x_max: float
-    x_bin: int
-    source_data_file: int
-    lumisection: int
-
-
-class PaginatedLumisectionHistogram1DList(BaseModel):
-    count: int
-    next: Optional[AnyUrl]
-    previous: Optional[AnyUrl]
-    results: List[LumisectionHistogram1D]
-
-
-class LumisectionHistogram1DFilters(OBaseModel):
-    ls_number: Optional[int] = None
-    lumisection_id: Optional[int] = None
-    max_ls_number: Optional[int] = None
-    max_run_number: Optional[int] = None
-    min_entries: Optional[int] = None
-    min_ls_number: Optional[int] = None
-    min_run_number: Optional[int] = None
-    page: Optional[str] = None
-    run_number: Optional[int] = None
-    title: Optional[str] = None
-    title_contains: Optional[str] = None
-    era: Optional[str] = None
-    dqmio_filepath_contains: Optional[str] = None

cmsdials-0.3.0/cmsdials/clients/h2d/models.py

@@ -1,50 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from pydantic import AnyUrl, BaseModel, Field
-
-from ...utils.base_model import OBaseModel
-
-
-class LumisectionHistogram2D(BaseModel):
-    id: int
-    ls_number: int
-    run_number: int
-    date: datetime
-    title: str = Field(..., max_length=220)
-    entries: int
-    data: list[list[float]]
-    x_min: float
-    x_max: float
-    x_bin: int
-    y_max: float
-    y_min: float
-    y_bin: int
-    source_data_file: int = Field(
-        ...,
-        description="Source data file that the specific Histogram was read from, if any",
-    )
-    lumisection: int
-
-
-class PaginatedLumisectionHistogram2DList(BaseModel):
-    count: int
-    next: Optional[AnyUrl]
-    previous: Optional[AnyUrl]
-    results: list[LumisectionHistogram2D]
-
-
-class LumisectionHistogram2DFilters(OBaseModel):
-    ls_number: Optional[int] = None
-    lumisection_id: Optional[int] = None
-    max_ls_number: Optional[int] = None
-    max_run_number: Optional[int] = None
-    min_entries: Optional[int] = None
-    min_ls_number: Optional[int] = None
-    min_run_number: Optional[int] = None
-    page: Optional[str] = None
-    run_number: Optional[int] = None
-    title: Optional[str] = None
-    title_contains: Optional[str] = None
-    era: Optional[str] = None
-    dqmio_filepath_contains: Optional[str] = None

cmsdials-0.3.0/cmsdials/clients/lumisection/client.py

@@ -1,20 +0,0 @@
-import requests
-
-from ...utils.api_client import BaseAuthorizedAPIClient
-from .models import ConfiguredMEs, Lumisection, LumisectionFilters, PaginatedLumisectionList
-
-
-class LumisectionClient(BaseAuthorizedAPIClient):
-    data_model = Lumisection
-    pagination_model = PaginatedLumisectionList
-    filter_class = LumisectionFilters
-    lookup_url = "lumisection/"
-
-    def configured_mes(self):
-        endpoint_url = self.api_url + self.lookup_url + "configured-mes/"
-        headers = {"accept": "application/json"}
-        self.creds.before_request(headers)
-        response = requests.get(endpoint_url, headers=headers)
-        response.raise_for_status()
-        response = response.json()
-        return ConfiguredMEs(**response)

cmsdials-0.3.0/cmsdials/clients/lumisection/models.py

@@ -1,35 +0,0 @@
-from datetime import datetime
-from typing import List, Optional
-
-from pydantic import AnyUrl, BaseModel
-
-from ...utils.base_model import OBaseModel
-
-
-class Lumisection(BaseModel):
-    id: int
-    ls_number: int
-    date: datetime
-    oms_zerobias_rate: Optional[float]
-    run: int
-
-
-class PaginatedLumisectionList(BaseModel):
-    count: int
-    next: Optional[AnyUrl]
-    previous: Optional[AnyUrl]
-    results: list[Lumisection]
-
-
-class LumisectionFilters(OBaseModel):
-    ls_number: Optional[int] = None
-    max_ls_number: Optional[int] = None
-    max_run_number: Optional[int] = None
-    min_ls_number: Optional[int] = None
-    min_run_number: Optional[int] = None
-    page: Optional[str] = None
-    run_number: Optional[int] = None
-
-
-class ConfiguredMEs(BaseModel):
-    mes: List[str]

cmsdials-0.3.0/cmsdials/clients/run/client.py

@@ -1,9 +0,0 @@
-from ...utils.api_client import BaseAuthorizedAPIClient
-from .models import PaginatedRunList, Run, RunFilters
-
-
-class RunClient(BaseAuthorizedAPIClient):
-    data_model = Run
-    pagination_model = PaginatedRunList
-    filter_class = RunFilters
-    lookup_url = "run/"

cmsdials-0.3.0/cmsdials/clients/run/models.py

@@ -1,31 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from pydantic import AnyUrl, BaseModel, Field
-
-from ...utils.base_model import OBaseModel
-
-
-class Run(BaseModel):
-    run_number: int
-    run_date: Optional[datetime]
-    year: Optional[int]
-    period: Optional[str] = Field(..., max_length=1)
-    date: datetime
-    oms_fill: Optional[int]
-    oms_lumisections: Optional[int]
-    oms_initial_lumi: Optional[float]
-    oms_end_lumi: Optional[float]
-
-
-class PaginatedRunList(BaseModel):
-    count: int
-    next: Optional[AnyUrl]
-    previous: Optional[AnyUrl]
-    results: list[Run]
-
-
-class RunFilters(OBaseModel):
-    max_run_number: Optional[int] = None
-    min_run_number: Optional[int] = None
-    page: Optional[str] = None

cmsdials-0.3.0/cmsdials/cmsdials.py

@@ -1,19 +0,0 @@
-from typing import Optional
-
-from .auth._base import BaseCredentials
-from .clients.bad_file_index.client import BadFileIndexClient
-from .clients.file_index.client import FileIndexClient
-from .clients.h1d.client import LumisectionHistogram1DClient
-from .clients.h2d.client import LumisectionHistogram2DClient
-from .clients.lumisection.client import LumisectionClient
-from .clients.run.client import RunClient
-
-
-class Dials:
-    def __init__(self, creds: BaseCredentials, nthreads: Optional[int] = None, *args, **kwargs) -> None:
-        self.bad_file_index = BadFileIndexClient(creds, nthreads, *args, **kwargs)
-        self.file_index = FileIndexClient(creds, nthreads, *args, **kwargs)
-        self.h1d = LumisectionHistogram1DClient(creds, nthreads, *args, **kwargs)
-        self.h2d = LumisectionHistogram2DClient(creds, nthreads, *args, **kwargs)
-        self.lumi = LumisectionClient(creds, nthreads, *args, **kwargs)
-        self.run = RunClient(creds, nthreads, *args, **kwargs)

cmsdials-0.3.0/pyproject.toml

@@ -1,31 +0,0 @@
-[tool.poetry]
-name = "cmsdials"
-version = "0.3.0"
-description = "The Python api client interface to DIALS service"
-authors = ["Gabriel Moreira <gabrielmscampos@gmail.com>"]
-readme = "README.md"
-include = ["LICENSE"]
-repository = "https://github.com/cms-DQM/dials-py"
-classifiers = [
-    "Development Status :: 4 - Beta",
-    "Programming Language :: Python",
-    "Operating System :: OS Independent",
-    "Topic :: Software Development :: Libraries :: Python Modules"
-]
-packages = [
-    { include = "cmsdials" }
-]
-
-[tool.poetry.dependencies]
-python = "^3.9"
-requests = "^2.31.0"
-pydantic = "<2, >=1"
-typing-extensions = "<4.6.0, >=3.6.6"
-
-
-[tool.poetry.group.dev.dependencies]
-pre-commit = "^3.6.2"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"