cmsdials 2.0.0rc4__tar.gz → 2.2.0__tar.gz

{cmsdials-2.0.0rc4 → cmsdials-2.2.0}/.gitignore

@@ -163,8 +163,5 @@ cython_debug/
 # ruff
 .ruff_cache/
 
-# uv
-uv.lock
-
 # setuptools_scm
 cmsdials/_version.py

cmsdials-2.2.0/.gitlab-ci.yml

@@ -0,0 +1,125 @@
+include:
+  - template: Jobs/Dependency-Scanning.v2.gitlab-ci.yml
+  - template: Security/Secret-Detection.gitlab-ci.yml
+  - template: Security/SAST.gitlab-ci.yml
+  - project: "cms-ppd/technical-support/ci-templates"
+    ref: release/v2
+    file:
+      - "templates/python/uv_build.yml"
+      - "templates/python/uv_publish.yml"
+      - "templates/gitlab/create_release.yml"
+      - "templates/python/uv_ruff.yml"
+      - "templates/python/uv_mypy.yml"
+      - "templates/gitlab/mr_quality_comment.yml"
+
+stages:
+  - quality
+  - test
+  - build
+  - publish
+  - create_release
+
+#################################################
+# Run lint tools and export patch in case of failures
+#################################################
+
+secret_detection:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+
+semgrep-sast:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+
+lint:
+  extends: .uv_ruff_lint
+  variables:
+    PYTHON_VERSION: '3.9'
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+
+static_check:
+  extends: .uv_mypy_check
+  variables:
+    PYTHON_VERSION: '3.9'
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+
+quality_comment:
+  extends: .mr_quality_comment
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+  needs:
+    - job: lint
+      artifacts: true
+      optional: true
+    - job: static_check
+      artifacts: true
+      optional: true
+
+###########
+# Run tests
+###########
+
+test_matrix:
+  stage: test
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"]
+  image: registry.cern.ch/docker.io/library/python:$PYTHON_VERSION
+  before_script:
+    - python -m pip install --upgrade pip
+    - pip install uv
+    - uv sync --all-groups --extra pandas --extra tqdm
+  script:
+    - uv run pytest tests
+  coverage: "/TOTAL.+ ([0-9]{1,3}%)/"
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+  variables:
+    SECRET_KEY: "$SECRET_KEY"
+    # Application owning SECRET_KEY, enables the api-access token tests
+    CLIENT_ID: "cms-dials-dev-api-client-test"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+
+###############
+# Build package
+###############
+
+build:
+  extends: .uv_build
+  variables:
+    UV_SYNC_FLAGS: "--all-groups --extra pandas --extra tqdm"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '$CI_COMMIT_TAG'
+  needs:
+    - job: test_matrix
+      optional: true
+
+#################
+# Publish to PyPI
+#################
+
+publish:
+  extends: .uv_publish
+  rules:
+    - if: $CI_COMMIT_TAG
+  needs:
+    - build
+
+##########################
+# Create release in GitLab
+##########################
+
+create_release:
+  extends: .create_gitlab_release
+  rules:
+    - if: $CI_COMMIT_TAG
+  needs:
+    - publish

{cmsdials-2.0.0rc4 → cmsdials-2.2.0}/.pre-commit-config.yaml

@@ -31,3 +31,4 @@ repos:
           - pandas-stubs>=2.2.2.240807
           - types-requests>=2.31.0.6
           - types-tqdm>=4.67.0.20250516
+          - numpy<2.3

{cmsdials-2.0.0rc4 → cmsdials-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cmsdials
-Version: 2.0.0rc4
+Version: 2.2.0
 Summary: The Python api client interface to DIALS service
 Author-email: Gabriel Moreira <gabriel.moreira@cern.ch>
 Project-URL: Repository, https://gitlab.cern.ch/cms-dqmdc/libraries/dials-py
@@ -21,9 +21,9 @@ Provides-Extra: tqdm
 Requires-Dist: tqdm>=4.66.1; extra == "tqdm"
 Dynamic: license-file
 
-![Build Status](https://gitlab.cern.ch/cms-dqmdc/libraries/dials-py/badges/develop/pipeline.svg)
-![Coverage](https://gitlab.cern.ch/cms-dqmdc/libraries/dials-py/badges/develop/coverage.svg)
-![Latest Release](https://gitlab.cern.ch/cms-dqmdc/libraries/dials-py/-/badges/release.svg)
+![Build Status](https://gitlab.cern.ch/cms-ppd/technical-support/libraries/dials-py/badges/develop/pipeline.svg)
+![Coverage](https://gitlab.cern.ch/cms-ppd/technical-support/libraries/dials-py/badges/develop/coverage.svg)
+![Latest Release](https://gitlab.cern.ch/cms-ppd/technical-support/libraries/dials-py/-/badges/release.svg)
 [![PyPI version](https://badge.fury.io/py/cmsdials.png)](https://badge.fury.io/py/cmsdials)
 
 # dials-py
@@ -46,16 +46,29 @@ pandas, tqdm
 
 ## Usage
 
-Before interfacing with any route you need to generate valid credentials, it is possible to authenticate trough the `device authorization flow` or using the `client secret key` of any application registered in DIALS. Note that, the device flow is an interactively authentication procedure that is possible to distinguish users in DIALS backend and the client secret flow is not interactive and is not possible to distinguish users so it should only be used for automation scripts.
+Before interfacing with any route you need to generate valid credentials, it is possible to authenticate trough the `device authorization flow` or using the `client id and secret` of an application granted API access in DIALS. Note that, the device flow is an interactively authentication procedure that is possible to distinguish users in DIALS backend and the client secret flow is not interactive and is not possible to distinguish users so it should only be used for automation scripts.
+
+Credentials are issued by CERN's Keycloak directly, dials-py never sends your password or client secret to DIALS.
 
 ### Generating credentials with client secret
 
 ```python
 from cmsdials.auth.secret_key import Credentials
 
-creds = Credentials(token=".....")
+creds = Credentials(client_id="my-application", token=".....")
 ```
 
+The token is renewed automatically before it expires.
+
+To get API access:
+
+1. Register an application in the [CERN Application Portal](https://application-portal.web.cern.ch) and enable "My application will need to get tokens using its own client ID and secret".
+2. Ask the DIALS team to grant API access to the application (it is added to the DIALS API access group).
+3. Use the application's client id and secret as shown above.
+
+> [!WARNING]
+> `Credentials(token=".....")` without `client_id` authenticates with the `X-CLIENT-SECRET` header, which is deprecated and emits a `DeprecationWarning`. DIALS will stop accepting it, so pass the `client_id` of your application.
+
 ### Generating credentials with device
 
 #### Loading from AuthClient
@@ -298,9 +311,11 @@ You may need to query the `ml_models_index` client to fetch the models ids you a
 
 Attention: Those endpoints doesn't return a Pydantic model, instead they are returning a plain json response. Consequently, the method `to_pandas` doesn't work on them.
 
-## Usage with local DIALS
+## Usage with other DIALS instances
+
+All classes that interface the DIALS service inherit the class `BaseAPIClient`, which propagates the `base_url`, `route` and `version` attributes with production values. In order to use dials-py with another DIALS instance it is possible to overwrite those attributes when instantiating the `Dials` client.
 
-All classes that interface the DIALS service inherits the class `BaseAPIClient` which propagate the `base_url`, `route` and `version` attributes with production values. In order to use dials-py with a local version of DIALS it is possible to overwrite those attributes when instantiating the `AuthClient` and the `Dials` client, for example:
+The `AuthClient` issues tokens for the production DIALS applications by default. For another instance, overwrite the Keycloak public application (`client_id`) and the DIALS application the tokens are issued for (`audience`). The audience must be the `DJANGO_KEYCLOAK_CLIENT_ID` configured in that DIALS backend, and that application must allow token exchange from the public application. For example, with a local instance:
 
 ```python
 from cmsdials.auth.client import AuthClient
@@ -308,16 +323,36 @@ from cmsdials.auth.bearer import Credentials
 from cmsdials import Dials
 from cmsdials.filters import LumisectionHistogram2DFilters
 
-DEV_URL = "http://localhost:8000/"
-DEV_CACHE_DIR = ".cache-dev"
+CLIENT_ID = "<your_public_application>"
+AUDIENCE = "<your_confidential_application>"
+CACHE_DIR = ".cache-local"
+DIALS_URL = "http://localhost:8000/"
 
-auth = AuthClient(base_url=DEV_URL)
-creds = Credentials.from_creds_file(cache_dir=DEV_CACHE_DIR, client=auth)  # Make sure to specify the auth client with overwritten values, using another cache_dir is recommended
-dials = Dials(creds, base_url=DEV_URL)
+auth = AuthClient(client_id=CLIENT_ID, audience=AUDIENCE)
+creds = Credentials.from_creds_file(cache_dir=CACHE_DIR, client=auth)  # Make sure to specify the auth client with overwritten values, using another cache_dir is recommended
+dials = Dials(creds, base_url=DIALS_URL)
 
 dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="EEOT digi occupancy EE +", entries__gte=100, run_number__gte=360392, run_number__lte=365000), max_pages=5)
 ```
 
+When authenticating with a client secret, pass the same `AuthClient` so the token is issued for that instance:
+
+```python
+from cmsdials.auth.secret_key import Credentials as SecretKeyCredentials
+
+creds = SecretKeyCredentials(client_id="my-application", token=".....", client=auth)
+dials = Dials(creds, base_url=DIALS_URL)
+```
+
+To connect to the development instance of DIALS, use these values in the examples above:
+
+```python
+CLIENT_ID = "cms-dials-dev-public-app"
+AUDIENCE = "cms-dials-dev-confidential-app"
+CACHE_DIR = ".cache-dev"
+DIALS_URL = "https://dev-cmsdials-api.web.cern.ch/"
+```
+
 ## Development
 
 Install the dependencies and the package using `uv`:
@@ -336,7 +371,7 @@ The repository has some tests written to make sure DIALS responses are compatibl
 SECRET_KEY=... uv run pytest tests
 ```
 
-The secret key is an api client enabled secret key and can be obtained from the applications portal, any api client secret key whitelisted in DIALS can be used. The interactive authentication flow should be tested manually, an example for this can be found in [this line](/tests/integration/test_auth_client.py#L33).
+The secret key is an api client enabled secret key and can be obtained from the applications portal, any api client secret key whitelisted in DIALS can be used. Set `CLIENT_ID` to the client id of the same application to test the api-access token flow (`SECRET_KEY=... CLIENT_ID=... uv run pytest tests`); otherwise those tests are skipped and the deprecated `X-CLIENT-SECRET` header is used. Tests target the dev Keycloak applications by default, overwrite them with `AUTH_CLIENT_ID` and `AUTH_AUDIENCE`. The interactive authentication flow should be tested manually, an example for this can be found in [test_auth_client.py](/tests/integration/test_auth_client.py).
 
 If testing against a local version of DIALS you need to specify the BASE_URL:
 

{cmsdials-2.0.0rc4 → cmsdials-2.2.0}/README.md

@@ -1,6 +1,6 @@
-![Build Status](https://gitlab.cern.ch/cms-dqmdc/libraries/dials-py/badges/develop/pipeline.svg)
-![Coverage](https://gitlab.cern.ch/cms-dqmdc/libraries/dials-py/badges/develop/coverage.svg)
-![Latest Release](https://gitlab.cern.ch/cms-dqmdc/libraries/dials-py/-/badges/release.svg)
+![Build Status](https://gitlab.cern.ch/cms-ppd/technical-support/libraries/dials-py/badges/develop/pipeline.svg)
+![Coverage](https://gitlab.cern.ch/cms-ppd/technical-support/libraries/dials-py/badges/develop/coverage.svg)
+![Latest Release](https://gitlab.cern.ch/cms-ppd/technical-support/libraries/dials-py/-/badges/release.svg)
 [![PyPI version](https://badge.fury.io/py/cmsdials.png)](https://badge.fury.io/py/cmsdials)
 
 # dials-py
@@ -23,16 +23,29 @@ pandas, tqdm
 
 ## Usage
 
-Before interfacing with any route you need to generate valid credentials, it is possible to authenticate trough the `device authorization flow` or using the `client secret key` of any application registered in DIALS. Note that, the device flow is an interactively authentication procedure that is possible to distinguish users in DIALS backend and the client secret flow is not interactive and is not possible to distinguish users so it should only be used for automation scripts.
+Before interfacing with any route you need to generate valid credentials, it is possible to authenticate trough the `device authorization flow` or using the `client id and secret` of an application granted API access in DIALS. Note that, the device flow is an interactively authentication procedure that is possible to distinguish users in DIALS backend and the client secret flow is not interactive and is not possible to distinguish users so it should only be used for automation scripts.
+
+Credentials are issued by CERN's Keycloak directly, dials-py never sends your password or client secret to DIALS.
 
 ### Generating credentials with client secret
 
 ```python
 from cmsdials.auth.secret_key import Credentials
 
-creds = Credentials(token=".....")
+creds = Credentials(client_id="my-application", token=".....")
 ```
 
+The token is renewed automatically before it expires.
+
+To get API access:
+
+1. Register an application in the [CERN Application Portal](https://application-portal.web.cern.ch) and enable "My application will need to get tokens using its own client ID and secret".
+2. Ask the DIALS team to grant API access to the application (it is added to the DIALS API access group).
+3. Use the application's client id and secret as shown above.
+
+> [!WARNING]
+> `Credentials(token=".....")` without `client_id` authenticates with the `X-CLIENT-SECRET` header, which is deprecated and emits a `DeprecationWarning`. DIALS will stop accepting it, so pass the `client_id` of your application.
+
 ### Generating credentials with device
 
 #### Loading from AuthClient
@@ -275,9 +288,11 @@ You may need to query the `ml_models_index` client to fetch the models ids you a
 
 Attention: Those endpoints doesn't return a Pydantic model, instead they are returning a plain json response. Consequently, the method `to_pandas` doesn't work on them.
 
-## Usage with local DIALS
+## Usage with other DIALS instances
+
+All classes that interface the DIALS service inherit the class `BaseAPIClient`, which propagates the `base_url`, `route` and `version` attributes with production values. In order to use dials-py with another DIALS instance it is possible to overwrite those attributes when instantiating the `Dials` client.
 
-All classes that interface the DIALS service inherits the class `BaseAPIClient` which propagate the `base_url`, `route` and `version` attributes with production values. In order to use dials-py with a local version of DIALS it is possible to overwrite those attributes when instantiating the `AuthClient` and the `Dials` client, for example:
+The `AuthClient` issues tokens for the production DIALS applications by default. For another instance, overwrite the Keycloak public application (`client_id`) and the DIALS application the tokens are issued for (`audience`). The audience must be the `DJANGO_KEYCLOAK_CLIENT_ID` configured in that DIALS backend, and that application must allow token exchange from the public application. For example, with a local instance:
 
 ```python
 from cmsdials.auth.client import AuthClient
@@ -285,16 +300,36 @@ from cmsdials.auth.bearer import Credentials
 from cmsdials import Dials
 from cmsdials.filters import LumisectionHistogram2DFilters
 
-DEV_URL = "http://localhost:8000/"
-DEV_CACHE_DIR = ".cache-dev"
+CLIENT_ID = "<your_public_application>"
+AUDIENCE = "<your_confidential_application>"
+CACHE_DIR = ".cache-local"
+DIALS_URL = "http://localhost:8000/"
 
-auth = AuthClient(base_url=DEV_URL)
-creds = Credentials.from_creds_file(cache_dir=DEV_CACHE_DIR, client=auth)  # Make sure to specify the auth client with overwritten values, using another cache_dir is recommended
-dials = Dials(creds, base_url=DEV_URL)
+auth = AuthClient(client_id=CLIENT_ID, audience=AUDIENCE)
+creds = Credentials.from_creds_file(cache_dir=CACHE_DIR, client=auth)  # Make sure to specify the auth client with overwritten values, using another cache_dir is recommended
+dials = Dials(creds, base_url=DIALS_URL)
 
 dials.h2d.list_all(LumisectionHistogram2DFilters(me__regex="EEOT digi occupancy EE +", entries__gte=100, run_number__gte=360392, run_number__lte=365000), max_pages=5)
 ```
 
+When authenticating with a client secret, pass the same `AuthClient` so the token is issued for that instance:
+
+```python
+from cmsdials.auth.secret_key import Credentials as SecretKeyCredentials
+
+creds = SecretKeyCredentials(client_id="my-application", token=".....", client=auth)
+dials = Dials(creds, base_url=DIALS_URL)
+```
+
+To connect to the development instance of DIALS, use these values in the examples above:
+
+```python
+CLIENT_ID = "cms-dials-dev-public-app"
+AUDIENCE = "cms-dials-dev-confidential-app"
+CACHE_DIR = ".cache-dev"
+DIALS_URL = "https://dev-cmsdials-api.web.cern.ch/"
+```
+
 ## Development
 
 Install the dependencies and the package using `uv`:
@@ -313,7 +348,7 @@ The repository has some tests written to make sure DIALS responses are compatibl
 SECRET_KEY=... uv run pytest tests
 ```
 
-The secret key is an api client enabled secret key and can be obtained from the applications portal, any api client secret key whitelisted in DIALS can be used. The interactive authentication flow should be tested manually, an example for this can be found in [this line](/tests/integration/test_auth_client.py#L33).
+The secret key is an api client enabled secret key and can be obtained from the applications portal, any api client secret key whitelisted in DIALS can be used. Set `CLIENT_ID` to the client id of the same application to test the api-access token flow (`SECRET_KEY=... CLIENT_ID=... uv run pytest tests`); otherwise those tests are skipped and the deprecated `X-CLIENT-SECRET` header is used. Tests target the dev Keycloak applications by default, overwrite them with `AUTH_CLIENT_ID` and `AUTH_AUDIENCE`. The interactive authentication flow should be tested manually, an example for this can be found in [test_auth_client.py](/tests/integration/test_auth_client.py).
 
 If testing against a local version of DIALS you need to specify the BASE_URL:
 

cmsdials-2.2.0/cmsdials/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '2.2.0'
+__version_tuple__ = version_tuple = (2, 2, 0)
+
+__commit_id__ = commit_id = 'g395406611'

{cmsdials-2.0.0rc4 → cmsdials-2.2.0}/cmsdials/auth/_base.py

@@ -45,7 +45,11 @@ class BaseCredentials(ABC):
         """
         if not self.expires_at:
             return False
-        return self.expires_at > datetime.now() > (self.expires_at - timedelta(seconds=0.1 * self.expires_in))
+        return (
+            self.expires_at
+            > datetime.now()
+            > (self.expires_at - timedelta(seconds=0.1 * self.expires_in))
+        )
 
     @property
     def expired(self) -> bool:
@@ -119,7 +123,10 @@ class BaseCredentials(ABC):
 
         # If token is stale (token is close to expire)
         # Re-use the token but launch a new thread to refresh the token
-        if self.token_state == TokenState.STALE and self._stale_lock.locked() is False:
+        if (
+            self.token_state == TokenState.STALE
+            and self._stale_lock.locked() is False
+        ):
             worker = threading.Thread(target=self._nonblocking_refresh)
             worker.start()
 

{cmsdials-2.0.0rc4 → cmsdials-2.2.0}/cmsdials/auth/bearer.py

@@ -4,6 +4,8 @@ import os.path
 from datetime import datetime
 from typing import Optional
 
+from requests.exceptions import HTTPError
+
 from ..utils._json import TokenDecoder, TokenEncoder
 from ..utils.logger import logger
 from ._base import BaseCredentials, TokenState
@@ -53,7 +55,9 @@ class Credentials(BaseCredentials):
 
         # If loaded token is invalid, trigger authentication
         if self.token_state == TokenState.INVALID:
-            logger.info("Access token and refresh token are expired, triggering device authentication flow...")
+            logger.info(
+                "Access token and refresh token are expired, triggering device authentication flow..."
+            )
             _token = self.client.device_auth_flow()
             self._set_token(_token)
 
@@ -84,7 +88,9 @@ class Credentials(BaseCredentials):
 
     @staticmethod
     def from_authclient_token(
-        token: Token, cache_dir: Optional[str] = None, client: Optional[AuthClient] = None
+        token: Token,
+        cache_dir: Optional[str] = None,
+        client: Optional[AuthClient] = None,
     ) -> "Credentials":
         client = client or AuthClient()
         cache_dir = cache_dir or DEFAULT_CACHE_DIR
@@ -101,7 +107,9 @@ class Credentials(BaseCredentials):
         )
 
     @staticmethod
-    def from_creds_file(cache_dir: Optional[str] = None, client: Optional[AuthClient] = None) -> "Credentials":
+    def from_creds_file(
+        cache_dir: Optional[str] = None, client: Optional[AuthClient] = None
+    ) -> "Credentials":
         client = client or AuthClient()
         cache_dir = cache_dir or DEFAULT_CACHE_DIR
         fpath = Credentials._handle_creds_filepath(cache_dir)
@@ -110,7 +118,9 @@ class Credentials(BaseCredentials):
                 fcontents = json.load(filebuffer, cls=TokenDecoder)
             token = Token(**fcontents)
         except FileNotFoundError:
-            logger.info("Credentials file not found, triggering device authentication flow...")
+            logger.info(
+                "Credentials file not found, triggering device authentication flow..."
+            )
             token = client.device_auth_flow()
 
         return Credentials.from_authclient_token(token, cache_dir, client)
@@ -129,8 +139,32 @@ class Credentials(BaseCredentials):
         with open(fpath, "w") as filebuffer:
             json.dump(self._token.model_dump(), filebuffer, cls=TokenEncoder)
 
+    @staticmethod
+    def _is_invalid_grant(err: HTTPError) -> bool:
+        if err.response is None or err.response.status_code != 400:
+            return False
+        try:
+            return err.response.json().get("error") == "invalid_grant"
+        except ValueError:
+            return False
+
     def refresh(self) -> None:
-        token: Token = self.client.refresh_token(self.token_type, self.token, self.refresh_token)
+        try:
+            token: Token = self.client.refresh_token(
+                self.token_type, self.token, self.refresh_token
+            )
+        except HTTPError as err:
+            # Keycloak only accepts a refresh token from the client it was
+            # issued to. Credentials cached by releases that authenticated
+            # through DIALS hold refresh tokens of the DIALS confidential
+            # application, not the public one. Expired or revoked refresh
+            # tokens are rejected with the same error.
+            if not self._is_invalid_grant(err):
+                raise
+            logger.info(
+                "Refresh token rejected, triggering device authentication flow..."
+            )
+            token = self.client.device_auth_flow()
         self._set_token(token)
         self.cache_credentials()