cmsdials 2.0.0rc3__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/.gitignore +0 -3
- cmsdials-2.1.0/.gitlab-ci.yml +123 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/.pre-commit-config.yaml +1 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/PKG-INFO +1 -1
- cmsdials-2.1.0/cmsdials/_version.py +24 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/auth/_base.py +9 -2
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/auth/bearer.py +15 -5
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/auth/client.py +24 -8
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/auth/secret_key.py +3 -1
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/dataset_index/client.py +5 -1
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/h1d/client.py +13 -2
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/h1d/models.py +3 -1
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/h2d/client.py +13 -2
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/h2d/models.py +3 -1
- cmsdials-2.1.0/cmsdials/clients/mes/client.py +17 -0
- cmsdials-2.1.0/cmsdials/clients/ml_bad_lumisection/client.py +74 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/ml_models_index/client.py +5 -1
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/oms_proxy/client.py +18 -4
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/run/models.py +1 -0
- cmsdials-2.1.0/cmsdials/clients/workspace/client.py +52 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/cmsdials.py +24 -6
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/utils/_json.py +3 -1
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/utils/api_client.py +107 -13
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/utils/base_model.py +9 -3
- cmsdials-2.1.0/cmsdials/utils/logger.py +8 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials.egg-info/PKG-INFO +1 -1
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials.egg-info/SOURCES.txt +6 -0
- cmsdials-2.1.0/cmsdials.egg-info/scm_file_list.json +81 -0
- cmsdials-2.1.0/cmsdials.egg-info/scm_version.json +8 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/mypy.ini +1 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/pyproject.toml +3 -3
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/ruff.toml +1 -1
- cmsdials-2.1.0/tests/integration/env.py +20 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/tests/integration/test_auth_client.py +15 -12
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/tests/integration/test_file_index_client.py +25 -11
- cmsdials-2.1.0/tests/integration/test_h1d_client.py +86 -0
- cmsdials-2.1.0/tests/integration/test_h2d_client.py +86 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/tests/integration/test_lumisection_client.py +26 -11
- cmsdials-2.1.0/tests/integration/test_mes_client.py +31 -0
- cmsdials-2.1.0/tests/integration/test_oms_proxy_client.py +60 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/tests/integration/test_resume.py +16 -7
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/tests/integration/test_run_client.py +19 -10
- cmsdials-2.1.0/tests/integration/utils.py +9 -0
- cmsdials-2.1.0/tests/unit/__init__.py +0 -0
- cmsdials-2.1.0/uv.lock +2571 -0
- cmsdials-2.0.0rc3/.gitlab-ci.yml +0 -129
- cmsdials-2.0.0rc3/cmsdials/_version.py +0 -34
- cmsdials-2.0.0rc3/cmsdials/clients/mes/client.py +0 -36
- cmsdials-2.0.0rc3/cmsdials/clients/ml_bad_lumisection/client.py +0 -41
- cmsdials-2.0.0rc3/cmsdials/utils/logger.py +0 -6
- cmsdials-2.0.0rc3/tests/integration/test_h1d_client.py +0 -49
- cmsdials-2.0.0rc3/tests/integration/test_h2d_client.py +0 -49
- cmsdials-2.0.0rc3/tests/integration/test_mes_client.py +0 -49
- cmsdials-2.0.0rc3/tests/integration/test_oms_proxy_client.py +0 -37
- cmsdials-2.0.0rc3/tests/integration/utils.py +0 -15
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/.coveragerc +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/.editorconfig +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/.mailmap +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/.tool-versions +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/LICENSE +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/README.md +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/auth/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/auth/exceptions.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/auth/models.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/dataset_index/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/dataset_index/models.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/file_index/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/file_index/client.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/file_index/models.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/h1d/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/h2d/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/lumisection/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/lumisection/client.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/lumisection/models.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/mes/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/mes/models.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/ml_bad_lumisection/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/ml_bad_lumisection/models.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/ml_models_index/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/ml_models_index/models.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/oms_proxy/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/oms_proxy/models.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/run/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/clients/run/client.py +0 -0
- {cmsdials-2.0.0rc3/cmsdials/utils → cmsdials-2.1.0/cmsdials/clients/workspace}/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials/filters.py +0 -0
- {cmsdials-2.0.0rc3/tests/integration → cmsdials-2.1.0/cmsdials/utils}/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials.egg-info/dependency_links.txt +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials.egg-info/requires.txt +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/cmsdials.egg-info/top_level.txt +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/pytest.ini +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/setup.cfg +0 -0
- {cmsdials-2.0.0rc3/tests/unit → cmsdials-2.1.0/tests/integration}/__init__.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/tests/unit/test_base_api_client.py +0 -0
- {cmsdials-2.0.0rc3 → cmsdials-2.1.0}/tox.ini +0 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
include:
|
|
2
|
+
- template: Jobs/Dependency-Scanning.v2.gitlab-ci.yml
|
|
3
|
+
- template: Security/Secret-Detection.gitlab-ci.yml
|
|
4
|
+
- template: Security/SAST.gitlab-ci.yml
|
|
5
|
+
- project: "cms-ppd/technical-support/ci-templates"
|
|
6
|
+
ref: release/v2
|
|
7
|
+
file:
|
|
8
|
+
- "templates/python/uv_build.yml"
|
|
9
|
+
- "templates/python/uv_publish.yml"
|
|
10
|
+
- "templates/gitlab/create_release.yml"
|
|
11
|
+
- "templates/python/uv_ruff.yml"
|
|
12
|
+
- "templates/python/uv_mypy.yml"
|
|
13
|
+
- "templates/gitlab/mr_quality_comment.yml"
|
|
14
|
+
|
|
15
|
+
stages:
|
|
16
|
+
- quality
|
|
17
|
+
- test
|
|
18
|
+
- build
|
|
19
|
+
- publish
|
|
20
|
+
- create_release
|
|
21
|
+
|
|
22
|
+
#################################################
|
|
23
|
+
# Run lint tools and export patch in case of failures
|
|
24
|
+
#################################################
|
|
25
|
+
|
|
26
|
+
secret_detection:
|
|
27
|
+
rules:
|
|
28
|
+
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
|
|
29
|
+
|
|
30
|
+
semgrep-sast:
|
|
31
|
+
rules:
|
|
32
|
+
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
|
|
33
|
+
|
|
34
|
+
lint:
|
|
35
|
+
extends: .uv_ruff_lint
|
|
36
|
+
variables:
|
|
37
|
+
PYTHON_VERSION: '3.9'
|
|
38
|
+
rules:
|
|
39
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
40
|
+
|
|
41
|
+
static_check:
|
|
42
|
+
extends: .uv_mypy_check
|
|
43
|
+
variables:
|
|
44
|
+
PYTHON_VERSION: '3.9'
|
|
45
|
+
rules:
|
|
46
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
47
|
+
|
|
48
|
+
quality_comment:
|
|
49
|
+
extends: .mr_quality_comment
|
|
50
|
+
rules:
|
|
51
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
52
|
+
when: always
|
|
53
|
+
needs:
|
|
54
|
+
- job: lint
|
|
55
|
+
artifacts: true
|
|
56
|
+
optional: true
|
|
57
|
+
- job: static_check
|
|
58
|
+
artifacts: true
|
|
59
|
+
optional: true
|
|
60
|
+
|
|
61
|
+
###########
|
|
62
|
+
# Run tests
|
|
63
|
+
###########
|
|
64
|
+
|
|
65
|
+
test_matrix:
|
|
66
|
+
stage: test
|
|
67
|
+
parallel:
|
|
68
|
+
matrix:
|
|
69
|
+
- PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12"]
|
|
70
|
+
image: registry.cern.ch/docker.io/library/python:$PYTHON_VERSION
|
|
71
|
+
before_script:
|
|
72
|
+
- python -m pip install --upgrade pip
|
|
73
|
+
- pip install uv
|
|
74
|
+
- uv sync --all-groups --extra pandas --extra tqdm
|
|
75
|
+
script:
|
|
76
|
+
- uv run pytest tests
|
|
77
|
+
coverage: "/TOTAL.+ ([0-9]{1,3}%)/"
|
|
78
|
+
artifacts:
|
|
79
|
+
reports:
|
|
80
|
+
coverage_report:
|
|
81
|
+
coverage_format: cobertura
|
|
82
|
+
path: coverage.xml
|
|
83
|
+
variables:
|
|
84
|
+
SECRET_KEY: "$SECRET_KEY"
|
|
85
|
+
rules:
|
|
86
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
87
|
+
|
|
88
|
+
###############
|
|
89
|
+
# Build package
|
|
90
|
+
###############
|
|
91
|
+
|
|
92
|
+
build:
|
|
93
|
+
extends: .uv_build
|
|
94
|
+
variables:
|
|
95
|
+
UV_SYNC_FLAGS: "--all-groups --extra pandas --extra tqdm"
|
|
96
|
+
rules:
|
|
97
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
98
|
+
- if: '$CI_COMMIT_TAG'
|
|
99
|
+
needs:
|
|
100
|
+
- job: test_matrix
|
|
101
|
+
optional: true
|
|
102
|
+
|
|
103
|
+
#################
|
|
104
|
+
# Publish to PyPI
|
|
105
|
+
#################
|
|
106
|
+
|
|
107
|
+
publish:
|
|
108
|
+
extends: .uv_publish
|
|
109
|
+
rules:
|
|
110
|
+
- if: $CI_COMMIT_TAG
|
|
111
|
+
needs:
|
|
112
|
+
- build
|
|
113
|
+
|
|
114
|
+
##########################
|
|
115
|
+
# Create release in GitLab
|
|
116
|
+
##########################
|
|
117
|
+
|
|
118
|
+
create_release:
|
|
119
|
+
extends: .create_gitlab_release
|
|
120
|
+
rules:
|
|
121
|
+
- if: $CI_COMMIT_TAG
|
|
122
|
+
needs:
|
|
123
|
+
- publish
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cmsdials
|
|
3
|
-
Version: 2.0
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: The Python api client interface to DIALS service
|
|
5
5
|
Author-email: Gabriel Moreira <gabriel.moreira@cern.ch>
|
|
6
6
|
Project-URL: Repository, https://gitlab.cern.ch/cms-dqmdc/libraries/dials-py
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '2.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'g4b3c689eb'
|
|
@@ -45,7 +45,11 @@ class BaseCredentials(ABC):
|
|
|
45
45
|
"""
|
|
46
46
|
if not self.expires_at:
|
|
47
47
|
return False
|
|
48
|
-
return
|
|
48
|
+
return (
|
|
49
|
+
self.expires_at
|
|
50
|
+
> datetime.now()
|
|
51
|
+
> (self.expires_at - timedelta(seconds=0.1 * self.expires_in))
|
|
52
|
+
)
|
|
49
53
|
|
|
50
54
|
@property
|
|
51
55
|
def expired(self) -> bool:
|
|
@@ -119,7 +123,10 @@ class BaseCredentials(ABC):
|
|
|
119
123
|
|
|
120
124
|
# If token is stale (token is close to expire)
|
|
121
125
|
# Re-use the token but launch a new thread to refresh the token
|
|
122
|
-
if
|
|
126
|
+
if (
|
|
127
|
+
self.token_state == TokenState.STALE
|
|
128
|
+
and self._stale_lock.locked() is False
|
|
129
|
+
):
|
|
123
130
|
worker = threading.Thread(target=self._nonblocking_refresh)
|
|
124
131
|
worker.start()
|
|
125
132
|
|
|
@@ -53,7 +53,9 @@ class Credentials(BaseCredentials):
|
|
|
53
53
|
|
|
54
54
|
# If loaded token is invalid, trigger authentication
|
|
55
55
|
if self.token_state == TokenState.INVALID:
|
|
56
|
-
logger.info(
|
|
56
|
+
logger.info(
|
|
57
|
+
"Access token and refresh token are expired, triggering device authentication flow..."
|
|
58
|
+
)
|
|
57
59
|
_token = self.client.device_auth_flow()
|
|
58
60
|
self._set_token(_token)
|
|
59
61
|
|
|
@@ -84,7 +86,9 @@ class Credentials(BaseCredentials):
|
|
|
84
86
|
|
|
85
87
|
@staticmethod
|
|
86
88
|
def from_authclient_token(
|
|
87
|
-
token: Token,
|
|
89
|
+
token: Token,
|
|
90
|
+
cache_dir: Optional[str] = None,
|
|
91
|
+
client: Optional[AuthClient] = None,
|
|
88
92
|
) -> "Credentials":
|
|
89
93
|
client = client or AuthClient()
|
|
90
94
|
cache_dir = cache_dir or DEFAULT_CACHE_DIR
|
|
@@ -101,7 +105,9 @@ class Credentials(BaseCredentials):
|
|
|
101
105
|
)
|
|
102
106
|
|
|
103
107
|
@staticmethod
|
|
104
|
-
def from_creds_file(
|
|
108
|
+
def from_creds_file(
|
|
109
|
+
cache_dir: Optional[str] = None, client: Optional[AuthClient] = None
|
|
110
|
+
) -> "Credentials":
|
|
105
111
|
client = client or AuthClient()
|
|
106
112
|
cache_dir = cache_dir or DEFAULT_CACHE_DIR
|
|
107
113
|
fpath = Credentials._handle_creds_filepath(cache_dir)
|
|
@@ -110,7 +116,9 @@ class Credentials(BaseCredentials):
|
|
|
110
116
|
fcontents = json.load(filebuffer, cls=TokenDecoder)
|
|
111
117
|
token = Token(**fcontents)
|
|
112
118
|
except FileNotFoundError:
|
|
113
|
-
logger.info(
|
|
119
|
+
logger.info(
|
|
120
|
+
"Credentials file not found, triggering device authentication flow..."
|
|
121
|
+
)
|
|
114
122
|
token = client.device_auth_flow()
|
|
115
123
|
|
|
116
124
|
return Credentials.from_authclient_token(token, cache_dir, client)
|
|
@@ -130,7 +138,9 @@ class Credentials(BaseCredentials):
|
|
|
130
138
|
json.dump(self._token.model_dump(), filebuffer, cls=TokenEncoder)
|
|
131
139
|
|
|
132
140
|
def refresh(self) -> None:
|
|
133
|
-
token: Token = self.client.refresh_token(
|
|
141
|
+
token: Token = self.client.refresh_token(
|
|
142
|
+
self.token_type, self.token, self.refresh_token
|
|
143
|
+
)
|
|
134
144
|
self._set_token(token)
|
|
135
145
|
self.cache_credentials()
|
|
136
146
|
|
|
@@ -23,7 +23,10 @@ class AuthClient(BaseAPIClient):
|
|
|
23
23
|
endpoint_url = self.api_url + "auth/new-device/"
|
|
24
24
|
response = requests.get(
|
|
25
25
|
url=endpoint_url,
|
|
26
|
-
headers={
|
|
26
|
+
headers={
|
|
27
|
+
"accept": "application/json",
|
|
28
|
+
"User-Agent": self._build_user_agent(),
|
|
29
|
+
},
|
|
27
30
|
timeout=self.default_timeout,
|
|
28
31
|
)
|
|
29
32
|
response.raise_for_status()
|
|
@@ -43,7 +46,10 @@ class AuthClient(BaseAPIClient):
|
|
|
43
46
|
timeout=self.default_timeout,
|
|
44
47
|
)
|
|
45
48
|
|
|
46
|
-
if
|
|
49
|
+
if (
|
|
50
|
+
response.status_code == 400
|
|
51
|
+
and "authorization_pending" in response.text
|
|
52
|
+
):
|
|
47
53
|
raise PendingAuthorizationError
|
|
48
54
|
|
|
49
55
|
response.raise_for_status()
|
|
@@ -54,10 +60,13 @@ class AuthClient(BaseAPIClient):
|
|
|
54
60
|
return Token(
|
|
55
61
|
**token_dict,
|
|
56
62
|
expires_at=curr_datetime + timedelta(seconds=token.expires_in),
|
|
57
|
-
refresh_expires_at=curr_datetime
|
|
63
|
+
refresh_expires_at=curr_datetime
|
|
64
|
+
+ timedelta(seconds=token.refresh_expires_in),
|
|
58
65
|
)
|
|
59
66
|
|
|
60
|
-
def refresh_token(
|
|
67
|
+
def refresh_token(
|
|
68
|
+
self, token_type: str, access_token: str, refresh_token: str
|
|
69
|
+
) -> Token:
|
|
61
70
|
endpoint_url = self.api_url + "auth/refresh-token/"
|
|
62
71
|
response = requests.post(
|
|
63
72
|
url=endpoint_url,
|
|
@@ -78,7 +87,8 @@ class AuthClient(BaseAPIClient):
|
|
|
78
87
|
return Token(
|
|
79
88
|
**token_dict,
|
|
80
89
|
expires_at=curr_datetime + timedelta(seconds=token.expires_in),
|
|
81
|
-
refresh_expires_at=curr_datetime
|
|
90
|
+
refresh_expires_at=curr_datetime
|
|
91
|
+
+ timedelta(seconds=token.refresh_expires_in),
|
|
82
92
|
)
|
|
83
93
|
|
|
84
94
|
def device_auth_flow(self):
|
|
@@ -87,8 +97,12 @@ class AuthClient(BaseAPIClient):
|
|
|
87
97
|
end_date = datetime.now() + timedelta(seconds=device.expires_in)
|
|
88
98
|
|
|
89
99
|
logger.info(f"This device will expire in {device.expires_in} seconds.")
|
|
90
|
-
logger.info(
|
|
91
|
-
|
|
100
|
+
logger.info(
|
|
101
|
+
f"Go to the following url and authenticate: {device.verification_uri_complete}"
|
|
102
|
+
)
|
|
103
|
+
logger.info(
|
|
104
|
+
f"Checking authorization status every {interval} seconds..."
|
|
105
|
+
)
|
|
92
106
|
|
|
93
107
|
while True:
|
|
94
108
|
time.sleep(interval)
|
|
@@ -98,7 +112,9 @@ class AuthClient(BaseAPIClient):
|
|
|
98
112
|
|
|
99
113
|
try:
|
|
100
114
|
token = self.device_token(device_code=device.device_code)
|
|
101
|
-
logger.info(
|
|
115
|
+
logger.info(
|
|
116
|
+
"Device authorized, authentication finished successfully!"
|
|
117
|
+
)
|
|
102
118
|
return token
|
|
103
119
|
except PendingAuthorizationError:
|
|
104
120
|
logger.info("Device not authorized yet.")
|
|
@@ -11,7 +11,9 @@ class Credentials(BaseCredentials):
|
|
|
11
11
|
|
|
12
12
|
def __init__(self, token: Optional[str]) -> None:
|
|
13
13
|
if not token:
|
|
14
|
-
raise ValueError(
|
|
14
|
+
raise ValueError(
|
|
15
|
+
"token must be a non-empty Client secret key string"
|
|
16
|
+
)
|
|
15
17
|
self.token = token
|
|
16
18
|
|
|
17
19
|
@property
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
from ...utils.api_client import BaseAuthorizedPaginatedAPIClient
|
|
2
|
-
from .models import
|
|
2
|
+
from .models import (
|
|
3
|
+
DatasetIndex,
|
|
4
|
+
DatasetIndexFilters,
|
|
5
|
+
PaginatedDatasetIndexList,
|
|
6
|
+
)
|
|
3
7
|
|
|
4
8
|
|
|
5
9
|
class DatasetIndexClient(
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
from ...utils.api_client import BaseAuthorizedPaginatedAPIClient
|
|
2
|
-
from .models import
|
|
2
|
+
from .models import (
|
|
3
|
+
LumisectionHistogram1D,
|
|
4
|
+
LumisectionHistogram1DFilters,
|
|
5
|
+
PaginatedLumisectionHistogram1DList,
|
|
6
|
+
)
|
|
3
7
|
|
|
4
8
|
|
|
5
9
|
class LumisectionHistogram1DClient(
|
|
@@ -14,6 +18,13 @@ class LumisectionHistogram1DClient(
|
|
|
14
18
|
filter_class = LumisectionHistogram1DFilters
|
|
15
19
|
lookup_url = "th1/"
|
|
16
20
|
|
|
17
|
-
def get(
|
|
21
|
+
def get(
|
|
22
|
+
self,
|
|
23
|
+
dataset_id: int,
|
|
24
|
+
run_number: int,
|
|
25
|
+
ls_number: int,
|
|
26
|
+
me_id: int,
|
|
27
|
+
**kwargs,
|
|
28
|
+
):
|
|
18
29
|
edp = f"{dataset_id}/{run_number}/{ls_number}/{me_id}/"
|
|
19
30
|
return super()._get(edp, **kwargs)
|
|
@@ -20,7 +20,9 @@ class LumisectionHistogram1D(BaseModel):
|
|
|
20
20
|
data: list[float]
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
class PaginatedLumisectionHistogram1DList(
|
|
23
|
+
class PaginatedLumisectionHistogram1DList(
|
|
24
|
+
PaginatedBaseModel[LumisectionHistogram1D]
|
|
25
|
+
):
|
|
24
26
|
pass
|
|
25
27
|
|
|
26
28
|
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
from ...utils.api_client import BaseAuthorizedPaginatedAPIClient
|
|
2
|
-
from .models import
|
|
2
|
+
from .models import (
|
|
3
|
+
LumisectionHistogram2D,
|
|
4
|
+
LumisectionHistogram2DFilters,
|
|
5
|
+
PaginatedLumisectionHistogram2DList,
|
|
6
|
+
)
|
|
3
7
|
|
|
4
8
|
|
|
5
9
|
class LumisectionHistogram2DClient(
|
|
@@ -14,6 +18,13 @@ class LumisectionHistogram2DClient(
|
|
|
14
18
|
filter_class = LumisectionHistogram2DFilters
|
|
15
19
|
lookup_url = "th2/"
|
|
16
20
|
|
|
17
|
-
def get(
|
|
21
|
+
def get(
|
|
22
|
+
self,
|
|
23
|
+
dataset_id: int,
|
|
24
|
+
run_number: int,
|
|
25
|
+
ls_number: int,
|
|
26
|
+
me_id: int,
|
|
27
|
+
**kwargs,
|
|
28
|
+
):
|
|
18
29
|
edp = f"{dataset_id}/{run_number}/{ls_number}/{me_id}/"
|
|
19
30
|
return super()._get(edp, **kwargs)
|
|
@@ -23,7 +23,9 @@ class LumisectionHistogram2D(BaseModel):
|
|
|
23
23
|
data: list[list[float]]
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
class PaginatedLumisectionHistogram2DList(
|
|
26
|
+
class PaginatedLumisectionHistogram2DList(
|
|
27
|
+
PaginatedBaseModel[LumisectionHistogram2D]
|
|
28
|
+
):
|
|
27
29
|
pass
|
|
28
30
|
|
|
29
31
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from ...utils.api_client import BaseAuthorizedNonPaginatedAPIClient
|
|
2
|
+
from .models import MEFilters, MonitoringElement
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class MonitoringElementClient(
|
|
6
|
+
BaseAuthorizedNonPaginatedAPIClient[
|
|
7
|
+
MonitoringElement,
|
|
8
|
+
MEFilters,
|
|
9
|
+
]
|
|
10
|
+
):
|
|
11
|
+
data_model = MonitoringElement
|
|
12
|
+
filter_class = MEFilters
|
|
13
|
+
lookup_url = "mes/"
|
|
14
|
+
|
|
15
|
+
def get(self, id: int, **kwargs): # noqa: A002
|
|
16
|
+
edp = f"{id}/"
|
|
17
|
+
return super()._get(edp, **kwargs)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from ...utils.api_client import BaseAuthorizedPaginatedAPIClient
|
|
2
|
+
from .models import (
|
|
3
|
+
MLBadLumisection,
|
|
4
|
+
MLBadLumisectionFilters,
|
|
5
|
+
PaginatedMLBadLumisectionList,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MLBadLumisectionClient(
|
|
10
|
+
BaseAuthorizedPaginatedAPIClient[
|
|
11
|
+
MLBadLumisection,
|
|
12
|
+
PaginatedMLBadLumisectionList,
|
|
13
|
+
MLBadLumisectionFilters,
|
|
14
|
+
]
|
|
15
|
+
):
|
|
16
|
+
data_model = MLBadLumisection
|
|
17
|
+
pagination_model = PaginatedMLBadLumisectionList
|
|
18
|
+
filter_class = MLBadLumisectionFilters
|
|
19
|
+
lookup_url = "ml-bad-lumisection/"
|
|
20
|
+
|
|
21
|
+
def get(
|
|
22
|
+
self,
|
|
23
|
+
model_id: int,
|
|
24
|
+
dataset_id: int,
|
|
25
|
+
run_number: int,
|
|
26
|
+
ls_number: int,
|
|
27
|
+
me_id: int,
|
|
28
|
+
**kwargs,
|
|
29
|
+
):
|
|
30
|
+
edp = f"{model_id}/{dataset_id}/{run_number}/{ls_number}/{me_id}/"
|
|
31
|
+
return super()._get(edp, **kwargs)
|
|
32
|
+
|
|
33
|
+
def __json_body_builder(
|
|
34
|
+
self,
|
|
35
|
+
model_id__in: list[int],
|
|
36
|
+
dataset_id__in: list[int],
|
|
37
|
+
run_number__in: list[int],
|
|
38
|
+
**kwargs,
|
|
39
|
+
):
|
|
40
|
+
return {
|
|
41
|
+
"model_id__in": model_id__in,
|
|
42
|
+
"dataset_id__in": dataset_id__in,
|
|
43
|
+
"run_number__in": run_number__in,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
def cert_json(
|
|
47
|
+
self,
|
|
48
|
+
model_id__in: list[int],
|
|
49
|
+
dataset_id__in: list[int],
|
|
50
|
+
run_number__in: list[int],
|
|
51
|
+
**kwargs,
|
|
52
|
+
):
|
|
53
|
+
edp = "cert-json/"
|
|
54
|
+
body = self.__json_body_builder(
|
|
55
|
+
model_id__in=model_id__in,
|
|
56
|
+
dataset_id__in=dataset_id__in,
|
|
57
|
+
run_number__in=run_number__in,
|
|
58
|
+
)
|
|
59
|
+
return super()._post(edp, json=body, return_raw_json=True, **kwargs)
|
|
60
|
+
|
|
61
|
+
def golden_json(
|
|
62
|
+
self,
|
|
63
|
+
model_id__in: list[int],
|
|
64
|
+
dataset_id__in: list[int],
|
|
65
|
+
run_number__in: list[int],
|
|
66
|
+
**kwargs,
|
|
67
|
+
):
|
|
68
|
+
edp = "golden-json/"
|
|
69
|
+
body = self.__json_body_builder(
|
|
70
|
+
model_id__in=model_id__in,
|
|
71
|
+
dataset_id__in=dataset_id__in,
|
|
72
|
+
run_number__in=run_number__in,
|
|
73
|
+
)
|
|
74
|
+
return super()._post(edp, json=body, return_raw_json=True, **kwargs)
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
from ...utils.api_client import BaseAuthorizedPaginatedAPIClient
|
|
2
|
-
from .models import
|
|
2
|
+
from .models import (
|
|
3
|
+
MLModelsIndex,
|
|
4
|
+
MLModelsIndexFilters,
|
|
5
|
+
PaginatedMLModelsIndexList,
|
|
6
|
+
)
|
|
3
7
|
|
|
4
8
|
|
|
5
9
|
class MLModelsIndexClient(
|
|
@@ -23,19 +23,33 @@ class OMSProxyClient(BaseAPIClient):
|
|
|
23
23
|
self.creds = creds
|
|
24
24
|
|
|
25
25
|
def _build_headers(self) -> dict:
|
|
26
|
-
base = {
|
|
26
|
+
base = {
|
|
27
|
+
"Accept": "application/json",
|
|
28
|
+
"User-Agent": self._build_user_agent(),
|
|
29
|
+
}
|
|
27
30
|
self.creds.before_request(base)
|
|
28
31
|
return base
|
|
29
32
|
|
|
30
33
|
def query(
|
|
31
|
-
self,
|
|
34
|
+
self,
|
|
35
|
+
endpoint: str,
|
|
36
|
+
filters: list[OMSFilter],
|
|
37
|
+
pages: Optional[list[OMSPage]] = None,
|
|
38
|
+
retries=DEFAULT_RETRIES,
|
|
32
39
|
):
|
|
33
40
|
headers = self._build_headers()
|
|
34
41
|
endpoint_url = self.api_url + self.lookup_url
|
|
35
42
|
|
|
36
43
|
# Format filters
|
|
37
|
-
_filters = {
|
|
38
|
-
|
|
44
|
+
_filters = {
|
|
45
|
+
f"filter[{_filter.attribute_name}][{_filter.operator}]": _filter.value
|
|
46
|
+
for _filter in filters
|
|
47
|
+
}
|
|
48
|
+
_pages = (
|
|
49
|
+
{f"page[{page.attribute_name}]": page.value for page in pages}
|
|
50
|
+
if pages and len(pages) > 0
|
|
51
|
+
else {}
|
|
52
|
+
)
|
|
39
53
|
params = {"endpoint": endpoint, **_filters, **_pages}
|
|
40
54
|
|
|
41
55
|
response = self._requests_get_retriable(
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from requests.adapters import DEFAULT_RETRIES
|
|
2
|
+
from requests.exceptions import HTTPError
|
|
3
|
+
|
|
4
|
+
from ...auth._base import BaseCredentials
|
|
5
|
+
from ...utils.api_client import BaseAPIClient
|
|
6
|
+
from ...utils.logger import logger
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class WorkspaceClient(BaseAPIClient):
|
|
10
|
+
default_timeout = 30
|
|
11
|
+
lookup_url = "auth/"
|
|
12
|
+
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
creds: BaseCredentials,
|
|
16
|
+
*args: str,
|
|
17
|
+
**kwargs: str,
|
|
18
|
+
) -> None:
|
|
19
|
+
super().__init__(*args, **kwargs)
|
|
20
|
+
self.creds = creds
|
|
21
|
+
|
|
22
|
+
def _build_headers(self) -> dict:
|
|
23
|
+
base = {
|
|
24
|
+
"Accept": "application/json",
|
|
25
|
+
"User-Agent": self._build_user_agent(),
|
|
26
|
+
}
|
|
27
|
+
self.creds.before_request(base)
|
|
28
|
+
return base
|
|
29
|
+
|
|
30
|
+
def __get(self, endpoint: str, retries=DEFAULT_RETRIES):
|
|
31
|
+
headers = self._build_headers()
|
|
32
|
+
endpoint_url = self.api_url + self.lookup_url + endpoint
|
|
33
|
+
|
|
34
|
+
response = self._requests_get_retriable(
|
|
35
|
+
endpoint_url,
|
|
36
|
+
headers=headers,
|
|
37
|
+
timeout=self.default_timeout,
|
|
38
|
+
retries=retries,
|
|
39
|
+
)
|
|
40
|
+
try:
|
|
41
|
+
response.raise_for_status()
|
|
42
|
+
except HTTPError as err:
|
|
43
|
+
logger.info(f"Api raw response: {response.text}")
|
|
44
|
+
raise err
|
|
45
|
+
|
|
46
|
+
return response.json()
|
|
47
|
+
|
|
48
|
+
def list(self, retries=DEFAULT_RETRIES):
|
|
49
|
+
return self.__get("workspaces", retries=retries)
|
|
50
|
+
|
|
51
|
+
def my_workspace(self, retries=DEFAULT_RETRIES):
|
|
52
|
+
return self.__get("user-default-workspace", retries=retries)
|
|
@@ -11,17 +11,35 @@ from .clients.ml_bad_lumisection.client import MLBadLumisectionClient
|
|
|
11
11
|
from .clients.ml_models_index.client import MLModelsIndexClient
|
|
12
12
|
from .clients.oms_proxy.client import OMSProxyClient
|
|
13
13
|
from .clients.run.client import RunClient
|
|
14
|
+
from .clients.workspace.client import WorkspaceClient
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
class Dials:
|
|
17
|
-
def __init__(
|
|
18
|
-
self
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
creds: BaseCredentials,
|
|
21
|
+
workspace: Optional[str] = None,
|
|
22
|
+
*args,
|
|
23
|
+
**kwargs,
|
|
24
|
+
) -> None:
|
|
25
|
+
self.dataset_index = DatasetIndexClient(
|
|
26
|
+
creds, workspace, *args, **kwargs
|
|
27
|
+
)
|
|
19
28
|
self.file_index = FileIndexClient(creds, workspace, *args, **kwargs)
|
|
20
|
-
self.h1d = LumisectionHistogram1DClient(
|
|
21
|
-
|
|
29
|
+
self.h1d = LumisectionHistogram1DClient(
|
|
30
|
+
creds, workspace, *args, **kwargs
|
|
31
|
+
)
|
|
32
|
+
self.h2d = LumisectionHistogram2DClient(
|
|
33
|
+
creds, workspace, *args, **kwargs
|
|
34
|
+
)
|
|
22
35
|
self.lumi = LumisectionClient(creds, workspace, *args, **kwargs)
|
|
23
36
|
self.run = RunClient(creds, workspace, *args, **kwargs)
|
|
24
37
|
self.mes = MonitoringElementClient(creds, workspace, *args, **kwargs)
|
|
25
38
|
self.oms = OMSProxyClient(creds, *args, **kwargs)
|
|
26
|
-
self.ml_bad_lumis = MLBadLumisectionClient(
|
|
27
|
-
|
|
39
|
+
self.ml_bad_lumis = MLBadLumisectionClient(
|
|
40
|
+
creds, workspace, *args, **kwargs
|
|
41
|
+
)
|
|
42
|
+
self.ml_models_index = MLModelsIndexClient(
|
|
43
|
+
creds, workspace, *args, **kwargs
|
|
44
|
+
)
|
|
45
|
+
self.workspace = WorkspaceClient(creds, *args, **kwargs)
|