diracx-testing 0.0.1a23__py3-none-any.whl → 0.0.1a25__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- diracx/testing/__init__.py +42 -746
- diracx/testing/entrypoints.py +67 -0
- diracx/testing/mock_osdb.py +19 -14
- diracx/testing/utils.py +704 -0
- {diracx_testing-0.0.1a23.dist-info → diracx_testing-0.0.1a25.dist-info}/METADATA +2 -2
- diracx_testing-0.0.1a25.dist-info/RECORD +11 -0
- {diracx_testing-0.0.1a23.dist-info → diracx_testing-0.0.1a25.dist-info}/WHEEL +1 -1
- diracx_testing-0.0.1a23.dist-info/RECORD +0 -9
- {diracx_testing-0.0.1a23.dist-info → diracx_testing-0.0.1a25.dist-info}/top_level.txt +0 -0
diracx/testing/__init__.py
CHANGED
@@ -1,751 +1,47 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
from typing import TYPE_CHECKING
|
19
|
-
from urllib.parse import parse_qs, urljoin, urlparse
|
20
|
-
from uuid import uuid4
|
21
|
-
|
22
|
-
import pytest
|
23
|
-
import requests
|
24
|
-
|
25
|
-
if TYPE_CHECKING:
|
26
|
-
from diracx.core.settings import DevelopmentSettings
|
27
|
-
from diracx.routers.jobs.sandboxes import SandboxStoreSettings
|
28
|
-
from diracx.routers.utils.users import AuthorizedUserInfo, AuthSettings
|
29
|
-
|
30
|
-
|
31
|
-
# to get a string like this run:
|
32
|
-
# openssl rand -hex 32
|
33
|
-
ALGORITHM = "HS256"
|
34
|
-
ISSUER = "http://lhcbdirac.cern.ch/"
|
35
|
-
AUDIENCE = "dirac"
|
36
|
-
ACCESS_TOKEN_EXPIRE_MINUTES = 30
|
37
|
-
|
38
|
-
|
39
|
-
def pytest_addoption(parser):
|
40
|
-
parser.addoption(
|
41
|
-
"--regenerate-client",
|
42
|
-
action="store_true",
|
43
|
-
default=False,
|
44
|
-
help="Regenerate the AutoREST client",
|
45
|
-
)
|
46
|
-
parser.addoption(
|
47
|
-
"--demo-dir",
|
48
|
-
type=Path,
|
49
|
-
default=None,
|
50
|
-
help="Path to a diracx-charts directory with the demo running",
|
51
|
-
)
|
52
|
-
|
53
|
-
|
54
|
-
def pytest_collection_modifyitems(config, items):
|
55
|
-
"""Disable the test_regenerate_client if not explicitly asked for."""
|
56
|
-
if config.getoption("--regenerate-client"):
|
57
|
-
# --regenerate-client given in cli: allow client re-generation
|
58
|
-
return
|
59
|
-
skip_regen = pytest.mark.skip(reason="need --regenerate-client option to run")
|
60
|
-
for item in items:
|
61
|
-
if item.name == "test_regenerate_client":
|
62
|
-
item.add_marker(skip_regen)
|
63
|
-
|
64
|
-
|
65
|
-
@pytest.fixture(scope="session")
|
66
|
-
def private_key_pem() -> str:
|
67
|
-
from cryptography.hazmat.primitives import serialization
|
68
|
-
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
69
|
-
|
70
|
-
private_key = Ed25519PrivateKey.generate()
|
71
|
-
return private_key.private_bytes(
|
72
|
-
encoding=serialization.Encoding.PEM,
|
73
|
-
format=serialization.PrivateFormat.PKCS8,
|
74
|
-
encryption_algorithm=serialization.NoEncryption(),
|
75
|
-
).decode()
|
76
|
-
|
77
|
-
|
78
|
-
@pytest.fixture(scope="session")
|
79
|
-
def fernet_key() -> str:
|
80
|
-
from cryptography.fernet import Fernet
|
81
|
-
|
82
|
-
return Fernet.generate_key().decode()
|
83
|
-
|
84
|
-
|
85
|
-
@pytest.fixture(scope="session")
|
86
|
-
def test_dev_settings() -> DevelopmentSettings:
|
87
|
-
from diracx.core.settings import DevelopmentSettings
|
88
|
-
|
89
|
-
yield DevelopmentSettings()
|
90
|
-
|
91
|
-
|
92
|
-
@pytest.fixture(scope="session")
|
93
|
-
def test_auth_settings(private_key_pem, fernet_key) -> AuthSettings:
|
94
|
-
from diracx.routers.utils.users import AuthSettings
|
95
|
-
|
96
|
-
yield AuthSettings(
|
97
|
-
token_algorithm="EdDSA",
|
98
|
-
token_key=private_key_pem,
|
99
|
-
state_key=fernet_key,
|
100
|
-
allowed_redirects=[
|
101
|
-
"http://diracx.test.invalid:8000/api/docs/oauth2-redirect",
|
102
|
-
],
|
103
|
-
)
|
104
|
-
|
105
|
-
|
106
|
-
@pytest.fixture(scope="session")
|
107
|
-
def aio_moto(worker_id):
|
108
|
-
"""Start the moto server in a separate thread and return the base URL.
|
109
|
-
|
110
|
-
The mocking provided by moto doesn't play nicely with aiobotocore so we use
|
111
|
-
the server directly. See https://github.com/aio-libs/aiobotocore/issues/755
|
112
|
-
"""
|
113
|
-
from moto.server import ThreadedMotoServer
|
114
|
-
|
115
|
-
port = 27132
|
116
|
-
if worker_id != "master":
|
117
|
-
port += int(worker_id.replace("gw", "")) + 1
|
118
|
-
server = ThreadedMotoServer(port=port)
|
119
|
-
server.start()
|
120
|
-
yield {
|
121
|
-
"endpoint_url": f"http://localhost:{port}",
|
122
|
-
"aws_access_key_id": "testing",
|
123
|
-
"aws_secret_access_key": "testing",
|
124
|
-
}
|
125
|
-
server.stop()
|
126
|
-
|
127
|
-
|
128
|
-
@pytest.fixture(scope="session")
|
129
|
-
def test_sandbox_settings(aio_moto) -> SandboxStoreSettings:
|
130
|
-
from diracx.routers.jobs.sandboxes import SandboxStoreSettings
|
131
|
-
|
132
|
-
yield SandboxStoreSettings(
|
133
|
-
bucket_name="sandboxes",
|
134
|
-
s3_client_kwargs=aio_moto,
|
135
|
-
auto_create_bucket=True,
|
136
|
-
)
|
137
|
-
|
138
|
-
|
139
|
-
class UnavailableDependency:
|
140
|
-
def __init__(self, key):
|
141
|
-
self.key = key
|
142
|
-
|
143
|
-
def __call__(self):
|
144
|
-
raise NotImplementedError(
|
145
|
-
f"{self.key} has not been made available to this test!"
|
146
|
-
)
|
147
|
-
|
148
|
-
|
149
|
-
class ClientFactory:
|
150
|
-
|
151
|
-
def __init__(
|
152
|
-
self,
|
153
|
-
tmp_path_factory,
|
154
|
-
with_config_repo,
|
155
|
-
test_auth_settings,
|
156
|
-
test_sandbox_settings,
|
157
|
-
test_dev_settings,
|
158
|
-
):
|
159
|
-
from diracx.core.config import ConfigSource
|
160
|
-
from diracx.core.extensions import select_from_extension
|
161
|
-
from diracx.core.settings import ServiceSettingsBase
|
162
|
-
from diracx.db.os.utils import BaseOSDB
|
163
|
-
from diracx.db.sql.utils import BaseSQLDB
|
164
|
-
from diracx.routers import create_app_inner
|
165
|
-
from diracx.routers.access_policies import BaseAccessPolicy
|
166
|
-
|
167
|
-
from .mock_osdb import fake_available_osdb_implementations
|
168
|
-
|
169
|
-
class AlwaysAllowAccessPolicy(BaseAccessPolicy):
|
170
|
-
"""Dummy access policy."""
|
171
|
-
|
172
|
-
async def policy(
|
173
|
-
policy_name: str, user_info: AuthorizedUserInfo, /, **kwargs
|
174
|
-
):
|
175
|
-
pass
|
176
|
-
|
177
|
-
def enrich_tokens(access_payload: dict, refresh_payload: dict):
|
178
|
-
|
179
|
-
return {"PolicySpecific": "OpenAccessForTest"}, {}
|
180
|
-
|
181
|
-
enabled_systems = {
|
182
|
-
e.name for e in select_from_extension(group="diracx.services")
|
183
|
-
}
|
184
|
-
database_urls = {
|
185
|
-
e.name: "sqlite+aiosqlite:///:memory:"
|
186
|
-
for e in select_from_extension(group="diracx.db.sql")
|
187
|
-
}
|
188
|
-
# TODO: Monkeypatch this in a less stupid way
|
189
|
-
# TODO: Only use this if opensearch isn't available
|
190
|
-
os_database_conn_kwargs = {
|
191
|
-
e.name: {"sqlalchemy_dsn": "sqlite+aiosqlite:///:memory:"}
|
192
|
-
for e in select_from_extension(group="diracx.db.os")
|
193
|
-
}
|
194
|
-
BaseOSDB.available_implementations = partial(
|
195
|
-
fake_available_osdb_implementations,
|
196
|
-
real_available_implementations=BaseOSDB.available_implementations,
|
197
|
-
)
|
198
|
-
|
199
|
-
self._cache_dir = tmp_path_factory.mktemp("empty-dbs")
|
200
|
-
|
201
|
-
self.test_auth_settings = test_auth_settings
|
202
|
-
self.test_dev_settings = test_dev_settings
|
203
|
-
|
204
|
-
all_access_policies = {
|
205
|
-
e.name: [AlwaysAllowAccessPolicy]
|
206
|
-
+ BaseAccessPolicy.available_implementations(e.name)
|
207
|
-
for e in select_from_extension(group="diracx.access_policies")
|
208
|
-
}
|
209
|
-
|
210
|
-
self.app = create_app_inner(
|
211
|
-
enabled_systems=enabled_systems,
|
212
|
-
all_service_settings=[
|
213
|
-
test_auth_settings,
|
214
|
-
test_sandbox_settings,
|
215
|
-
test_dev_settings,
|
216
|
-
],
|
217
|
-
database_urls=database_urls,
|
218
|
-
os_database_conn_kwargs=os_database_conn_kwargs,
|
219
|
-
config_source=ConfigSource.create_from_url(
|
220
|
-
backend_url=f"git+file://{with_config_repo}"
|
221
|
-
),
|
222
|
-
all_access_policies=all_access_policies,
|
223
|
-
)
|
224
|
-
|
225
|
-
self.all_dependency_overrides = self.app.dependency_overrides.copy()
|
226
|
-
self.app.dependency_overrides = {}
|
227
|
-
for obj in self.all_dependency_overrides:
|
228
|
-
assert issubclass(
|
229
|
-
obj.__self__,
|
230
|
-
(
|
231
|
-
ServiceSettingsBase,
|
232
|
-
BaseSQLDB,
|
233
|
-
BaseOSDB,
|
234
|
-
ConfigSource,
|
235
|
-
BaseAccessPolicy,
|
236
|
-
),
|
237
|
-
), obj
|
238
|
-
|
239
|
-
self.all_lifetime_functions = self.app.lifetime_functions[:]
|
240
|
-
self.app.lifetime_functions = []
|
241
|
-
for obj in self.all_lifetime_functions:
|
242
|
-
assert isinstance(
|
243
|
-
obj.__self__, (ServiceSettingsBase, BaseSQLDB, BaseOSDB, ConfigSource)
|
244
|
-
), obj
|
245
|
-
|
246
|
-
@contextlib.contextmanager
|
247
|
-
def configure(self, enabled_dependencies):
|
248
|
-
|
249
|
-
assert (
|
250
|
-
self.app.dependency_overrides == {} and self.app.lifetime_functions == []
|
251
|
-
), "configure cannot be nested"
|
252
|
-
for k, v in self.all_dependency_overrides.items():
|
253
|
-
|
254
|
-
class_name = k.__self__.__name__
|
255
|
-
|
256
|
-
if class_name in enabled_dependencies:
|
257
|
-
self.app.dependency_overrides[k] = v
|
258
|
-
else:
|
259
|
-
self.app.dependency_overrides[k] = UnavailableDependency(class_name)
|
260
|
-
|
261
|
-
for obj in self.all_lifetime_functions:
|
262
|
-
# TODO: We should use the name of the entry point instead of the class name
|
263
|
-
if obj.__self__.__class__.__name__ in enabled_dependencies:
|
264
|
-
self.app.lifetime_functions.append(obj)
|
265
|
-
|
266
|
-
# Add create_db_schemas to the end of the lifetime_functions so that the
|
267
|
-
# other lifetime_functions (i.e. those which run db.engine_context) have
|
268
|
-
# already been ran
|
269
|
-
self.app.lifetime_functions.append(self.create_db_schemas)
|
270
|
-
|
271
|
-
try:
|
272
|
-
yield
|
273
|
-
finally:
|
274
|
-
self.app.dependency_overrides = {}
|
275
|
-
self.app.lifetime_functions = []
|
276
|
-
|
277
|
-
@contextlib.asynccontextmanager
|
278
|
-
async def create_db_schemas(self):
|
279
|
-
"""Create DB schema's based on the DBs available in app.dependency_overrides."""
|
280
|
-
import aiosqlite
|
281
|
-
import sqlalchemy
|
282
|
-
from sqlalchemy.util.concurrency import greenlet_spawn
|
283
|
-
|
284
|
-
from diracx.db.sql.utils import BaseSQLDB
|
285
|
-
|
286
|
-
for k, v in self.app.dependency_overrides.items():
|
287
|
-
# Ignore dependency overrides which aren't BaseSQLDB.transaction
|
288
|
-
if (
|
289
|
-
isinstance(v, UnavailableDependency)
|
290
|
-
or k.__func__ != BaseSQLDB.transaction.__func__
|
291
|
-
):
|
292
|
-
continue
|
293
|
-
# The first argument of the overridden BaseSQLDB.transaction is the DB object
|
294
|
-
db = v.args[0]
|
295
|
-
assert isinstance(db, BaseSQLDB), (k, db)
|
296
|
-
|
297
|
-
# set PRAGMA foreign_keys=ON if sqlite
|
298
|
-
if db.engine.url.drivername.startswith("sqlite"):
|
299
|
-
|
300
|
-
def set_sqlite_pragma(dbapi_connection, connection_record):
|
301
|
-
cursor = dbapi_connection.cursor()
|
302
|
-
cursor.execute("PRAGMA foreign_keys=ON")
|
303
|
-
cursor.close()
|
304
|
-
|
305
|
-
sqlalchemy.event.listen(
|
306
|
-
db.engine.sync_engine, "connect", set_sqlite_pragma
|
307
|
-
)
|
308
|
-
|
309
|
-
# We maintain a cache of the populated DBs in empty_db_dir so that
|
310
|
-
# we don't have to recreate them for every test. This speeds up the
|
311
|
-
# tests by a considerable amount.
|
312
|
-
ref_db = self._cache_dir / f"{k.__self__.__name__}.db"
|
313
|
-
if ref_db.exists():
|
314
|
-
async with aiosqlite.connect(ref_db) as ref_conn:
|
315
|
-
conn = await db.engine.raw_connection()
|
316
|
-
await ref_conn.backup(conn.driver_connection)
|
317
|
-
await greenlet_spawn(conn.close)
|
318
|
-
else:
|
319
|
-
async with db.engine.begin() as conn:
|
320
|
-
await conn.run_sync(db.metadata.create_all)
|
321
|
-
|
322
|
-
async with aiosqlite.connect(ref_db) as ref_conn:
|
323
|
-
conn = await db.engine.raw_connection()
|
324
|
-
await conn.driver_connection.backup(ref_conn)
|
325
|
-
await greenlet_spawn(conn.close)
|
326
|
-
|
327
|
-
yield
|
328
|
-
|
329
|
-
@contextlib.contextmanager
|
330
|
-
def unauthenticated(self):
|
331
|
-
from fastapi.testclient import TestClient
|
332
|
-
|
333
|
-
with TestClient(self.app) as client:
|
334
|
-
yield client
|
335
|
-
|
336
|
-
@contextlib.contextmanager
|
337
|
-
def normal_user(self):
|
338
|
-
from diracx.core.properties import NORMAL_USER
|
339
|
-
from diracx.routers.auth.token import create_token
|
340
|
-
|
341
|
-
with self.unauthenticated() as client:
|
342
|
-
payload = {
|
343
|
-
"sub": "testingVO:yellow-sub",
|
344
|
-
"exp": datetime.now(tz=timezone.utc)
|
345
|
-
+ timedelta(self.test_auth_settings.access_token_expire_minutes),
|
346
|
-
"iss": ISSUER,
|
347
|
-
"dirac_properties": [NORMAL_USER],
|
348
|
-
"jti": str(uuid4()),
|
349
|
-
"preferred_username": "preferred_username",
|
350
|
-
"dirac_group": "test_group",
|
351
|
-
"vo": "lhcb",
|
352
|
-
}
|
353
|
-
token = create_token(payload, self.test_auth_settings)
|
354
|
-
|
355
|
-
client.headers["Authorization"] = f"Bearer {token}"
|
356
|
-
client.dirac_token_payload = payload
|
357
|
-
yield client
|
358
|
-
|
359
|
-
@contextlib.contextmanager
|
360
|
-
def admin_user(self):
|
361
|
-
from diracx.core.properties import JOB_ADMINISTRATOR
|
362
|
-
from diracx.routers.auth.token import create_token
|
363
|
-
|
364
|
-
with self.unauthenticated() as client:
|
365
|
-
payload = {
|
366
|
-
"sub": "testingVO:yellow-sub",
|
367
|
-
"iss": ISSUER,
|
368
|
-
"dirac_properties": [JOB_ADMINISTRATOR],
|
369
|
-
"jti": str(uuid4()),
|
370
|
-
"preferred_username": "preferred_username",
|
371
|
-
"dirac_group": "test_group",
|
372
|
-
"vo": "lhcb",
|
373
|
-
}
|
374
|
-
token = create_token(payload, self.test_auth_settings)
|
375
|
-
client.headers["Authorization"] = f"Bearer {token}"
|
376
|
-
client.dirac_token_payload = payload
|
377
|
-
yield client
|
378
|
-
|
379
|
-
|
380
|
-
@pytest.fixture(scope="session")
|
381
|
-
def session_client_factory(
|
3
|
+
from .entrypoints import verify_entry_points
|
4
|
+
from .utils import (
|
5
|
+
ClientFactory,
|
6
|
+
aio_moto,
|
7
|
+
cli_env,
|
8
|
+
client_factory,
|
9
|
+
demo_dir,
|
10
|
+
demo_kubectl_env,
|
11
|
+
demo_urls,
|
12
|
+
do_device_flow_with_dex,
|
13
|
+
fernet_key,
|
14
|
+
private_key_pem,
|
15
|
+
pytest_addoption,
|
16
|
+
pytest_collection_modifyitems,
|
17
|
+
session_client_factory,
|
382
18
|
test_auth_settings,
|
19
|
+
test_dev_settings,
|
20
|
+
test_login,
|
383
21
|
test_sandbox_settings,
|
22
|
+
with_cli_login,
|
384
23
|
with_config_repo,
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
""
|
389
|
-
|
390
|
-
|
391
|
-
""
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
@pytest.fixture(scope="session")
|
412
|
-
def with_config_repo(tmp_path_factory):
|
413
|
-
from git import Repo
|
414
|
-
|
415
|
-
from diracx.core.config import Config
|
416
|
-
|
417
|
-
tmp_path = tmp_path_factory.mktemp("cs-repo")
|
418
|
-
|
419
|
-
repo = Repo.init(tmp_path, initial_branch="master")
|
420
|
-
cs_file = tmp_path / "default.yml"
|
421
|
-
example_cs = Config.model_validate(
|
422
|
-
{
|
423
|
-
"DIRAC": {},
|
424
|
-
"Registry": {
|
425
|
-
"lhcb": {
|
426
|
-
"DefaultGroup": "lhcb_user",
|
427
|
-
"DefaultProxyLifeTime": 432000,
|
428
|
-
"DefaultStorageQuota": 2000,
|
429
|
-
"IdP": {
|
430
|
-
"URL": "https://idp-server.invalid",
|
431
|
-
"ClientID": "test-idp",
|
432
|
-
},
|
433
|
-
"Users": {
|
434
|
-
"b824d4dc-1f9d-4ee8-8df5-c0ae55d46041": {
|
435
|
-
"PreferedUsername": "chaen",
|
436
|
-
"Email": None,
|
437
|
-
},
|
438
|
-
"c935e5ed-2g0e-5ff9-9eg6-d1bf66e57152": {
|
439
|
-
"PreferedUsername": "albdr",
|
440
|
-
"Email": None,
|
441
|
-
},
|
442
|
-
},
|
443
|
-
"Groups": {
|
444
|
-
"lhcb_user": {
|
445
|
-
"Properties": ["NormalUser", "PrivateLimitedDelegation"],
|
446
|
-
"Users": [
|
447
|
-
"b824d4dc-1f9d-4ee8-8df5-c0ae55d46041",
|
448
|
-
"c935e5ed-2g0e-5ff9-9eg6-d1bf66e57152",
|
449
|
-
],
|
450
|
-
},
|
451
|
-
"lhcb_prmgr": {
|
452
|
-
"Properties": ["NormalUser", "ProductionManagement"],
|
453
|
-
"Users": ["b824d4dc-1f9d-4ee8-8df5-c0ae55d46041"],
|
454
|
-
},
|
455
|
-
"lhcb_tokenmgr": {
|
456
|
-
"Properties": ["NormalUser", "ProxyManagement"],
|
457
|
-
"Users": ["c935e5ed-2g0e-5ff9-9eg6-d1bf66e57152"],
|
458
|
-
},
|
459
|
-
},
|
460
|
-
}
|
461
|
-
},
|
462
|
-
"Operations": {"Defaults": {}},
|
463
|
-
"Systems": {
|
464
|
-
"WorkloadManagement": {
|
465
|
-
"Production": {
|
466
|
-
"Databases": {
|
467
|
-
"JobDB": {
|
468
|
-
"DBName": "xyz",
|
469
|
-
"Host": "xyz",
|
470
|
-
"Port": 9999,
|
471
|
-
"MaxRescheduling": 3,
|
472
|
-
},
|
473
|
-
"JobLoggingDB": {
|
474
|
-
"DBName": "xyz",
|
475
|
-
"Host": "xyz",
|
476
|
-
"Port": 9999,
|
477
|
-
},
|
478
|
-
"PilotAgentsDB": {
|
479
|
-
"DBName": "xyz",
|
480
|
-
"Host": "xyz",
|
481
|
-
"Port": 9999,
|
482
|
-
},
|
483
|
-
"SandboxMetadataDB": {
|
484
|
-
"DBName": "xyz",
|
485
|
-
"Host": "xyz",
|
486
|
-
"Port": 9999,
|
487
|
-
},
|
488
|
-
"TaskQueueDB": {
|
489
|
-
"DBName": "xyz",
|
490
|
-
"Host": "xyz",
|
491
|
-
"Port": 9999,
|
492
|
-
},
|
493
|
-
"ElasticJobParametersDB": {
|
494
|
-
"DBName": "xyz",
|
495
|
-
"Host": "xyz",
|
496
|
-
"Port": 9999,
|
497
|
-
},
|
498
|
-
"VirtualMachineDB": {
|
499
|
-
"DBName": "xyz",
|
500
|
-
"Host": "xyz",
|
501
|
-
"Port": 9999,
|
502
|
-
},
|
503
|
-
},
|
504
|
-
},
|
505
|
-
},
|
506
|
-
},
|
507
|
-
}
|
508
|
-
)
|
509
|
-
cs_file.write_text(example_cs.model_dump_json())
|
510
|
-
repo.index.add([cs_file]) # add it to the index
|
511
|
-
repo.index.commit("Added a new file")
|
512
|
-
yield tmp_path
|
513
|
-
|
514
|
-
|
515
|
-
@pytest.fixture(scope="session")
|
516
|
-
def demo_dir(request) -> Path:
|
517
|
-
demo_dir = request.config.getoption("--demo-dir")
|
518
|
-
if demo_dir is None:
|
519
|
-
pytest.skip("Requires a running instance of the DiracX demo")
|
520
|
-
demo_dir = (demo_dir / ".demo").resolve()
|
521
|
-
yield demo_dir
|
522
|
-
|
523
|
-
|
524
|
-
@pytest.fixture(scope="session")
|
525
|
-
def demo_urls(demo_dir):
|
526
|
-
import yaml
|
527
|
-
|
528
|
-
helm_values = yaml.safe_load((demo_dir / "values.yaml").read_text())
|
529
|
-
yield helm_values["developer"]["urls"]
|
530
|
-
|
531
|
-
|
532
|
-
@pytest.fixture(scope="session")
|
533
|
-
def demo_kubectl_env(demo_dir):
|
534
|
-
"""Get the dictionary of environment variables for kubectl to control the demo."""
|
535
|
-
kube_conf = demo_dir / "kube.conf"
|
536
|
-
if not kube_conf.exists():
|
537
|
-
raise RuntimeError(f"Could not find {kube_conf}, is the demo running?")
|
538
|
-
|
539
|
-
env = {
|
540
|
-
**os.environ,
|
541
|
-
"KUBECONFIG": str(kube_conf),
|
542
|
-
"PATH": f"{demo_dir}:{os.environ['PATH']}",
|
543
|
-
}
|
544
|
-
|
545
|
-
# Check that we can run kubectl
|
546
|
-
pods_result = subprocess.check_output(
|
547
|
-
["kubectl", "get", "pods"], env=env, text=True
|
548
|
-
)
|
549
|
-
assert "diracx" in pods_result
|
550
|
-
|
551
|
-
yield env
|
552
|
-
|
553
|
-
|
554
|
-
@pytest.fixture
|
555
|
-
def cli_env(monkeypatch, tmp_path, demo_urls, demo_dir):
|
556
|
-
"""Set up the environment for the CLI."""
|
557
|
-
import httpx
|
558
|
-
|
559
|
-
from diracx.core.preferences import get_diracx_preferences
|
560
|
-
|
561
|
-
diracx_url = demo_urls["diracx"]
|
562
|
-
ca_path = demo_dir / "demo-ca.pem"
|
563
|
-
if not ca_path.exists():
|
564
|
-
raise RuntimeError(f"Could not find {ca_path}, is the demo running?")
|
565
|
-
|
566
|
-
# Ensure the demo is working
|
567
|
-
|
568
|
-
r = httpx.get(
|
569
|
-
f"{diracx_url}/api/openapi.json",
|
570
|
-
verify=ssl.create_default_context(cafile=ca_path),
|
571
|
-
)
|
572
|
-
r.raise_for_status()
|
573
|
-
assert r.json()["info"]["title"] == "Dirac"
|
574
|
-
|
575
|
-
env = {
|
576
|
-
"DIRACX_URL": diracx_url,
|
577
|
-
"DIRACX_CA_PATH": str(ca_path),
|
578
|
-
"HOME": str(tmp_path),
|
579
|
-
}
|
580
|
-
for key, value in env.items():
|
581
|
-
monkeypatch.setenv(key, value)
|
582
|
-
yield env
|
583
|
-
|
584
|
-
# The DiracX preferences are cached however when testing this cache is invalid
|
585
|
-
get_diracx_preferences.cache_clear()
|
586
|
-
|
587
|
-
|
588
|
-
@pytest.fixture
|
589
|
-
async def with_cli_login(monkeypatch, capfd, cli_env, tmp_path):
|
590
|
-
try:
|
591
|
-
credentials = await test_login(monkeypatch, capfd, cli_env)
|
592
|
-
except Exception as e:
|
593
|
-
pytest.skip(f"Login failed, fix test_login to re-enable this test: {e!r}")
|
594
|
-
|
595
|
-
credentials_path = tmp_path / "credentials.json"
|
596
|
-
credentials_path.write_text(credentials)
|
597
|
-
monkeypatch.setenv("DIRACX_CREDENTIALS_PATH", str(credentials_path))
|
598
|
-
yield
|
599
|
-
|
600
|
-
|
601
|
-
async def test_login(monkeypatch, capfd, cli_env):
|
602
|
-
from diracx import cli
|
603
|
-
|
604
|
-
poll_attempts = 0
|
605
|
-
|
606
|
-
def fake_sleep(*args, **kwargs):
|
607
|
-
nonlocal poll_attempts
|
608
|
-
|
609
|
-
# Keep track of the number of times this is called
|
610
|
-
poll_attempts += 1
|
611
|
-
|
612
|
-
# After polling 5 times, do the actual login
|
613
|
-
if poll_attempts == 5:
|
614
|
-
# The login URL should have been printed to stdout
|
615
|
-
captured = capfd.readouterr()
|
616
|
-
match = re.search(rf"{cli_env['DIRACX_URL']}[^\n]+", captured.out)
|
617
|
-
assert match, captured
|
618
|
-
|
619
|
-
do_device_flow_with_dex(match.group(), cli_env["DIRACX_CA_PATH"])
|
620
|
-
|
621
|
-
# Ensure we don't poll forever
|
622
|
-
assert poll_attempts <= 100
|
623
|
-
|
624
|
-
# Reduce the sleep duration to zero to speed up the test
|
625
|
-
return unpatched_sleep(0)
|
626
|
-
|
627
|
-
# We monkeypatch asyncio.sleep to provide a hook to run the actions that
|
628
|
-
# would normally be done by a user. This includes capturing the login URL
|
629
|
-
# and doing the actual device flow with dex.
|
630
|
-
unpatched_sleep = asyncio.sleep
|
631
|
-
|
632
|
-
expected_credentials_path = Path(
|
633
|
-
cli_env["HOME"], ".cache", "diracx", "credentials.json"
|
634
|
-
)
|
635
|
-
# Ensure the credentials file does not exist before logging in
|
636
|
-
assert not expected_credentials_path.exists()
|
637
|
-
|
638
|
-
# Run the login command
|
639
|
-
with monkeypatch.context() as m:
|
640
|
-
m.setattr("asyncio.sleep", fake_sleep)
|
641
|
-
await cli.login(vo="diracAdmin", group=None, property=None)
|
642
|
-
captured = capfd.readouterr()
|
643
|
-
assert "Login successful!" in captured.out
|
644
|
-
assert captured.err == ""
|
645
|
-
|
646
|
-
# Ensure the credentials file exists after logging in
|
647
|
-
assert expected_credentials_path.exists()
|
648
|
-
|
649
|
-
# Return the credentials so this test can also be used by the
|
650
|
-
# "with_cli_login" fixture
|
651
|
-
return expected_credentials_path.read_text()
|
652
|
-
|
653
|
-
|
654
|
-
def do_device_flow_with_dex(url: str, ca_path: str) -> None:
|
655
|
-
"""Do the device flow with dex."""
|
656
|
-
|
657
|
-
class DexLoginFormParser(HTMLParser):
|
658
|
-
def handle_starttag(self, tag, attrs):
|
659
|
-
nonlocal action_url
|
660
|
-
if "form" in str(tag):
|
661
|
-
assert action_url is None
|
662
|
-
action_url = urljoin(login_page_url, dict(attrs)["action"])
|
663
|
-
|
664
|
-
# Get the login page
|
665
|
-
r = requests.get(url, verify=ca_path)
|
666
|
-
r.raise_for_status()
|
667
|
-
login_page_url = r.url # This is not the same as URL as we redirect to dex
|
668
|
-
login_page_body = r.text
|
669
|
-
|
670
|
-
# Search the page for the login form so we know where to post the credentials
|
671
|
-
action_url = None
|
672
|
-
DexLoginFormParser().feed(login_page_body)
|
673
|
-
assert action_url is not None, login_page_body
|
674
|
-
|
675
|
-
# Do the actual login
|
676
|
-
r = requests.post(
|
677
|
-
action_url,
|
678
|
-
data={"login": "admin@example.com", "password": "password"},
|
679
|
-
verify=ca_path,
|
680
|
-
)
|
681
|
-
r.raise_for_status()
|
682
|
-
approval_url = r.url # This is not the same as URL as we redirect to dex
|
683
|
-
# Do the actual approval
|
684
|
-
r = requests.post(
|
685
|
-
approval_url,
|
686
|
-
{"approval": "approve", "req": parse_qs(urlparse(r.url).query)["req"][0]},
|
687
|
-
verify=ca_path,
|
688
|
-
)
|
689
|
-
|
690
|
-
# This should have redirected to the DiracX page that shows the login is complete
|
691
|
-
assert "Please close the window" in r.text
|
692
|
-
|
693
|
-
|
694
|
-
def get_installed_entry_points():
|
695
|
-
"""Retrieve the installed entry points from the environment."""
|
696
|
-
entry_pts = entry_points()
|
697
|
-
diracx_eps = defaultdict(dict)
|
698
|
-
for group in entry_pts.groups:
|
699
|
-
if "diracx" in group:
|
700
|
-
for ep in entry_pts.select(group=group):
|
701
|
-
diracx_eps[group][ep.name] = ep.value
|
702
|
-
return dict(diracx_eps)
|
703
|
-
|
704
|
-
|
705
|
-
def get_entry_points_from_toml(toml_file):
|
706
|
-
"""Parse entry points from pyproject.toml."""
|
707
|
-
with open(toml_file, "rb") as f:
|
708
|
-
pyproject = tomllib.load(f)
|
709
|
-
package_name = pyproject["project"]["name"]
|
710
|
-
return package_name, pyproject.get("project", {}).get("entry-points", {})
|
711
|
-
|
712
|
-
|
713
|
-
def get_current_entry_points(repo_base) -> bool:
|
714
|
-
"""Create current entry points dict for comparison."""
|
715
|
-
current_eps = {}
|
716
|
-
for toml_file in repo_base.glob("diracx-*/pyproject.toml"):
|
717
|
-
package_name, entry_pts = get_entry_points_from_toml(f"{toml_file}")
|
718
|
-
# Ignore packages that are not installed
|
719
|
-
try:
|
720
|
-
distribution(package_name)
|
721
|
-
except PackageNotFoundError:
|
722
|
-
continue
|
723
|
-
# Merge the entry points
|
724
|
-
for key, value in entry_pts.items():
|
725
|
-
current_eps[key] = current_eps.get(key, {}) | value
|
726
|
-
return current_eps
|
727
|
-
|
728
|
-
|
729
|
-
@pytest.fixture(scope="session", autouse=True)
|
730
|
-
def verify_entry_points(request, pytestconfig):
|
731
|
-
try:
|
732
|
-
ini_toml_name = tomllib.loads(pytestconfig.inipath.read_text())["project"][
|
733
|
-
"name"
|
734
|
-
]
|
735
|
-
except tomllib.TOMLDecodeError:
|
736
|
-
return
|
737
|
-
if ini_toml_name == "diracx":
|
738
|
-
repo_base = pytestconfig.inipath.parent
|
739
|
-
elif ini_toml_name.startswith("diracx-"):
|
740
|
-
repo_base = pytestconfig.inipath.parent.parent
|
741
|
-
else:
|
742
|
-
return
|
743
|
-
|
744
|
-
installed_eps = get_installed_entry_points()
|
745
|
-
current_eps = get_current_entry_points(repo_base)
|
746
|
-
|
747
|
-
if installed_eps != current_eps:
|
748
|
-
pytest.fail(
|
749
|
-
"Project and installed entry-points are not consistent. "
|
750
|
-
"You should run `pip install -r requirements-dev.txt`",
|
751
|
-
)
|
24
|
+
)
|
25
|
+
|
26
|
+
__all__ = (
|
27
|
+
"verify_entry_points",
|
28
|
+
"ClientFactory",
|
29
|
+
"do_device_flow_with_dex",
|
30
|
+
"test_login",
|
31
|
+
"pytest_addoption",
|
32
|
+
"pytest_collection_modifyitems",
|
33
|
+
"private_key_pem",
|
34
|
+
"fernet_key",
|
35
|
+
"test_dev_settings",
|
36
|
+
"test_auth_settings",
|
37
|
+
"aio_moto",
|
38
|
+
"test_sandbox_settings",
|
39
|
+
"session_client_factory",
|
40
|
+
"client_factory",
|
41
|
+
"with_config_repo",
|
42
|
+
"demo_dir",
|
43
|
+
"demo_urls",
|
44
|
+
"demo_kubectl_env",
|
45
|
+
"cli_env",
|
46
|
+
"with_cli_login",
|
47
|
+
)
|