ctao-bdms-clients 0.0.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bdms/__init__.py +7 -0
- bdms/_dev_version/__init__.py +9 -0
- bdms/_version.py +16 -0
- bdms/tests/__init__.py +0 -0
- bdms/tests/conftest.py +53 -0
- bdms/tests/test_basic_rucio_functionality.py +143 -0
- bdms/tests/test_dpps_rel_0_0.py +90 -0
- bdms/tests/test_file_replicas.py +72 -0
- bdms/tests/test_metadata.py +110 -0
- bdms/version.py +23 -0
- ctao_bdms_clients-0.0.0a0.dist-info/LICENSE +29 -0
- ctao_bdms_clients-0.0.0a0.dist-info/METADATA +35 -0
- ctao_bdms_clients-0.0.0a0.dist-info/RECORD +16 -0
- ctao_bdms_clients-0.0.0a0.dist-info/WHEEL +5 -0
- ctao_bdms_clients-0.0.0a0.dist-info/entry_points.txt +2 -0
- ctao_bdms_clients-0.0.0a0.dist-info/top_level.txt +1 -0
bdms/__init__.py
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
# Try to use setuptools_scm to get the current version; this is only used
|
2
|
+
# in development installations from the git repository.
|
3
|
+
# see ../version.py for details
|
4
|
+
try:
|
5
|
+
from setuptools_scm import get_version
|
6
|
+
|
7
|
+
version = get_version(root="../../..", relative_to=__file__)
|
8
|
+
except Exception as e:
|
9
|
+
raise ImportError(f"setuptools_scm broken or not installed: {e}")
|
bdms/_version.py
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
# file generated by setuptools_scm
|
2
|
+
# don't change, don't track in version control
|
3
|
+
TYPE_CHECKING = False
|
4
|
+
if TYPE_CHECKING:
|
5
|
+
from typing import Tuple, Union
|
6
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
7
|
+
else:
|
8
|
+
VERSION_TUPLE = object
|
9
|
+
|
10
|
+
version: str
|
11
|
+
__version__: str
|
12
|
+
__version_tuple__: VERSION_TUPLE
|
13
|
+
version_tuple: VERSION_TUPLE
|
14
|
+
|
15
|
+
__version__ = version = '0.0.0a0'
|
16
|
+
__version_tuple__ = version_tuple = (0, 0, 0)
|
bdms/tests/__init__.py
ADDED
File without changes
|
bdms/tests/conftest.py
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
import os
|
2
|
+
import subprocess as sp
|
3
|
+
from datetime import datetime
|
4
|
+
from secrets import token_hex
|
5
|
+
|
6
|
+
import pytest
|
7
|
+
from rucio.client.scopeclient import ScopeClient
|
8
|
+
|
9
|
+
USER_CERT = os.getenv("RUCIO_CFG_CLIENT_CERT", "/opt/rucio/etc/usercert.pem")
|
10
|
+
USER_KEY = os.getenv("RUCIO_CFG_CLIENT_KEY", "/opt/rucio/etc/userkey.pem")
|
11
|
+
|
12
|
+
|
13
|
+
@pytest.fixture(scope="session")
|
14
|
+
def test_user():
|
15
|
+
return "root"
|
16
|
+
|
17
|
+
|
18
|
+
@pytest.fixture(scope="session")
|
19
|
+
def _auth_proxy(tmp_path_factory):
|
20
|
+
"""Auth proxy needed for accessing RSEs"""
|
21
|
+
# Key has to have 0o600 permissions, but due to the way we
|
22
|
+
# we create and mount it, it does not. We copy to a tmp file
|
23
|
+
# set correct permissions and then create the proxy
|
24
|
+
sp.run(
|
25
|
+
[
|
26
|
+
"voms-proxy-init",
|
27
|
+
"-valid",
|
28
|
+
"9999:00",
|
29
|
+
"-cert",
|
30
|
+
USER_CERT,
|
31
|
+
"-key",
|
32
|
+
USER_KEY,
|
33
|
+
],
|
34
|
+
check=True,
|
35
|
+
)
|
36
|
+
|
37
|
+
|
38
|
+
@pytest.fixture(scope="session")
|
39
|
+
def test_vo():
|
40
|
+
return "ctao.dpps.test"
|
41
|
+
|
42
|
+
|
43
|
+
@pytest.fixture(scope="session")
|
44
|
+
def test_scope(test_user):
|
45
|
+
"""To avoid name conflicts and old state, use a unique scope for the tests"""
|
46
|
+
# length of scope is limited to 25 characters
|
47
|
+
random_hash = token_hex(2)
|
48
|
+
date_str = f"{datetime.now():%Y%m%d_%H%M%S}"
|
49
|
+
scope = f"t_{date_str}_{random_hash}"
|
50
|
+
|
51
|
+
sc = ScopeClient()
|
52
|
+
sc.add_scope(test_user, scope)
|
53
|
+
return scope
|
@@ -0,0 +1,143 @@
|
|
1
|
+
import time
|
2
|
+
|
3
|
+
import pytest
|
4
|
+
from rucio.client import Client
|
5
|
+
from rucio.client.client import ReplicaClient, RuleClient
|
6
|
+
from rucio.client.didclient import DIDClient
|
7
|
+
from rucio.client.uploadclient import UploadClient
|
8
|
+
|
9
|
+
|
10
|
+
def test_server_version():
|
11
|
+
"""Test the expected version of rucio is running"""
|
12
|
+
client = Client()
|
13
|
+
result = client.ping()
|
14
|
+
assert result["version"].startswith("35")
|
15
|
+
|
16
|
+
|
17
|
+
def test_authentication():
|
18
|
+
"""Test we authenticated successfully"""
|
19
|
+
client = Client()
|
20
|
+
result = client.whoami()
|
21
|
+
|
22
|
+
assert result["account"] == "root"
|
23
|
+
|
24
|
+
|
25
|
+
def test_rses():
|
26
|
+
"""Test the expected RSEs are configured"""
|
27
|
+
client = Client()
|
28
|
+
result = list(client.list_rses())
|
29
|
+
|
30
|
+
rses = [r["rse"] for r in result]
|
31
|
+
assert "STORAGE-1" in rses
|
32
|
+
assert "STORAGE-2" in rses
|
33
|
+
assert "STORAGE-3" in rses
|
34
|
+
|
35
|
+
|
36
|
+
def test_add_dataset(test_vo, test_scope):
|
37
|
+
"""Test adding a simple dataset works"""
|
38
|
+
dataset_name = f"/{test_vo}/{test_scope}/dataset_aiv_basic"
|
39
|
+
|
40
|
+
did_client = DIDClient()
|
41
|
+
success = did_client.add_dataset(
|
42
|
+
scope=test_scope, name=dataset_name, rse="STORAGE-1", lifetime=2
|
43
|
+
)
|
44
|
+
assert success
|
45
|
+
|
46
|
+
names = list(did_client.list_dids(scope=test_scope, filters={}))
|
47
|
+
assert dataset_name in names
|
48
|
+
|
49
|
+
|
50
|
+
@pytest.mark.usefixtures("_auth_proxy")
|
51
|
+
def test_upload_file(test_vo, test_scope, tmp_path):
|
52
|
+
"""Test uploading a simple file works"""
|
53
|
+
name = "file_aiv_basic"
|
54
|
+
path = tmp_path / "file_aiv_basic"
|
55
|
+
path.write_text("Hello, World!")
|
56
|
+
did_name = f"/{test_vo}/{test_scope}/{name}"
|
57
|
+
|
58
|
+
upload_client = UploadClient()
|
59
|
+
|
60
|
+
upload_spec = {
|
61
|
+
"path": path,
|
62
|
+
"rse": "STORAGE-2",
|
63
|
+
"did_scope": test_scope,
|
64
|
+
"did_name": did_name,
|
65
|
+
}
|
66
|
+
# 0 means success
|
67
|
+
assert upload_client.upload([upload_spec]) == 0
|
68
|
+
|
69
|
+
|
70
|
+
@pytest.mark.parametrize(
|
71
|
+
"timeout",
|
72
|
+
[
|
73
|
+
pytest.param(
|
74
|
+
60,
|
75
|
+
marks=pytest.mark.xfail(
|
76
|
+
reason="sometimes there is an extra 300s timeout somewhere in FTS"
|
77
|
+
),
|
78
|
+
),
|
79
|
+
(600,),
|
80
|
+
],
|
81
|
+
)
|
82
|
+
def wait_for_replication_status(rule, status, timeout=600, poll=5):
|
83
|
+
rule_client = RuleClient()
|
84
|
+
|
85
|
+
start = time.perf_counter()
|
86
|
+
|
87
|
+
current_status = None
|
88
|
+
result = None
|
89
|
+
|
90
|
+
while (time.perf_counter() - start) < timeout:
|
91
|
+
result = rule_client.get_replication_rule(rule)
|
92
|
+
current_status = result["state"]
|
93
|
+
|
94
|
+
if current_status == status:
|
95
|
+
return
|
96
|
+
|
97
|
+
time.sleep(poll)
|
98
|
+
|
99
|
+
msg = (
|
100
|
+
f"Rule {rule} did not reach status '{status}' within {timeout} seconds."
|
101
|
+
f" Current status is '{current_status}'.\nFull output: {result}"
|
102
|
+
)
|
103
|
+
raise TimeoutError(msg)
|
104
|
+
|
105
|
+
|
106
|
+
@pytest.mark.usefixtures("_auth_proxy")
|
107
|
+
def test_replication(test_vo, test_scope, tmp_path):
|
108
|
+
dataset = "transfer_test"
|
109
|
+
path = tmp_path / f"{dataset}.dat"
|
110
|
+
path.write_text("I am a test for replication rules.")
|
111
|
+
|
112
|
+
dataset_did = f"/{test_vo}/{test_scope}/{dataset}"
|
113
|
+
file_did = f"/{test_vo}/{test_scope}/{dataset}/{path.name}"
|
114
|
+
|
115
|
+
main_rse = "STORAGE-1"
|
116
|
+
replica_rse = "STORAGE-2"
|
117
|
+
|
118
|
+
client = Client()
|
119
|
+
upload_client = UploadClient()
|
120
|
+
did_client = DIDClient()
|
121
|
+
rule_client = RuleClient()
|
122
|
+
replica_client = ReplicaClient()
|
123
|
+
|
124
|
+
upload_spec = {
|
125
|
+
"path": path,
|
126
|
+
"rse": main_rse,
|
127
|
+
"did_scope": test_scope,
|
128
|
+
"did_name": file_did,
|
129
|
+
}
|
130
|
+
# return value of 0 means success
|
131
|
+
assert upload_client.upload([upload_spec]) == 0
|
132
|
+
assert did_client.add_dataset(scope=test_scope, name=dataset_did)
|
133
|
+
dids = [{"scope": test_scope, "name": file_did}]
|
134
|
+
assert client.attach_dids(scope=test_scope, name=dataset_did, dids=dids)
|
135
|
+
|
136
|
+
dids = [{"scope": test_scope, "name": dataset_did}]
|
137
|
+
rule = rule_client.add_replication_rule(
|
138
|
+
dids=dids, copies=1, rse_expression=replica_rse
|
139
|
+
)[0]
|
140
|
+
|
141
|
+
wait_for_replication_status(rule=rule, status="OK")
|
142
|
+
replicas = next(replica_client.list_replicas(dids))
|
143
|
+
assert replicas["states"] == {"STORAGE-1": "AVAILABLE", "STORAGE-2": "AVAILABLE"}
|
@@ -0,0 +1,90 @@
|
|
1
|
+
import zlib
|
2
|
+
|
3
|
+
import pytest
|
4
|
+
from rucio.client import Client
|
5
|
+
from rucio.client.downloadclient import DownloadClient
|
6
|
+
from rucio.client.uploadclient import UploadClient
|
7
|
+
|
8
|
+
|
9
|
+
@pytest.fixture(scope="session")
|
10
|
+
def generic_data_product(_auth_proxy, test_vo, test_scope, tmp_path_factory):
|
11
|
+
"""A generic test data product ingested into the BDMS.
|
12
|
+
|
13
|
+
Used to fulfill precondition of an existing data product for query and retrieve
|
14
|
+
tests and part the implementation part of UC-110-1.1.3.
|
15
|
+
|
16
|
+
Returns
|
17
|
+
-------
|
18
|
+
lfn: str
|
19
|
+
the lfn of the test file
|
20
|
+
content: str
|
21
|
+
content of the file
|
22
|
+
checksum: str
|
23
|
+
adler32 checksum as hex string
|
24
|
+
"""
|
25
|
+
name = "test_lfn"
|
26
|
+
lfn = f"/{test_vo}/{test_scope}/{name}"
|
27
|
+
path = tmp_path_factory.mktemp("upload") / name
|
28
|
+
content = "A wizard is never late"
|
29
|
+
path.write_text(content)
|
30
|
+
checksum = f"{zlib.adler32(content.encode('utf-8')):x}"
|
31
|
+
|
32
|
+
upload_spec = {
|
33
|
+
"path": path,
|
34
|
+
"rse": "STORAGE-2",
|
35
|
+
"did_scope": test_scope,
|
36
|
+
"did_name": lfn,
|
37
|
+
}
|
38
|
+
|
39
|
+
# upload, this is part of UC 110-1.1.3
|
40
|
+
upload_client = UploadClient()
|
41
|
+
# 0 means success
|
42
|
+
assert upload_client.upload([upload_spec]) == 0
|
43
|
+
return lfn, content, checksum
|
44
|
+
|
45
|
+
|
46
|
+
@pytest.mark.verifies_usecase("DPPS-UC-110-1.1.3")
|
47
|
+
@pytest.mark.verifies_usecase("DPPS-UC-110-1.2.1")
|
48
|
+
def test_query_by_lfn(test_scope, generic_data_product):
|
49
|
+
"""
|
50
|
+
Test for getting the metadata of a product by LFN.
|
51
|
+
|
52
|
+
Ingestion of the generic data product is executed in the fixture.
|
53
|
+
|
54
|
+
Performing UC-110-1.2.1 is part of the postconditions of UC-110-1.1.3,
|
55
|
+
so we use the same test to verify both usecases.
|
56
|
+
"""
|
57
|
+
test_lfn, _, expected_checksum = generic_data_product
|
58
|
+
|
59
|
+
# 1. We select the "test_lfn"
|
60
|
+
# 2. - 5. metadata query and execution
|
61
|
+
client = Client()
|
62
|
+
meta = client.get_metadata(test_scope, test_lfn)
|
63
|
+
# 6. verify expectations
|
64
|
+
assert meta["adler32"] == expected_checksum
|
65
|
+
|
66
|
+
|
67
|
+
@pytest.mark.verifies_usecase("DPPS-UC-110-1.1.3")
|
68
|
+
@pytest.mark.verifies_usecase("DPPS-UC-110-1.3.1")
|
69
|
+
@pytest.mark.usefixtures("_auth_proxy")
|
70
|
+
def test_retrieve_by_lfn(test_scope, generic_data_product, tmp_path):
|
71
|
+
"""
|
72
|
+
Test for retrieving product by LFN.
|
73
|
+
|
74
|
+
Ingestion of the generic data product is executed in the fixture.
|
75
|
+
|
76
|
+
Performing UC-110-1.3.1 is part of the postconditions of UC-110-1.1.3,
|
77
|
+
so we use the same test to verify both usecases.
|
78
|
+
"""
|
79
|
+
test_lfn, expected_content, _ = generic_data_product
|
80
|
+
|
81
|
+
# 1. is done by using the "auth_proxy" fixture
|
82
|
+
# 2. - 3. download query and execution
|
83
|
+
query = [{"did": f"{test_scope}:{test_lfn}", "base_dir": tmp_path}]
|
84
|
+
download_client = DownloadClient()
|
85
|
+
download_client.download_dids(query)
|
86
|
+
|
87
|
+
# 4. inspect data product
|
88
|
+
expected_path = tmp_path / test_scope / test_lfn.lstrip("/")
|
89
|
+
assert expected_path.is_file(), "File not downloaded to expected location"
|
90
|
+
assert expected_path.read_text() == expected_content
|
@@ -0,0 +1,72 @@
|
|
1
|
+
import pytest
|
2
|
+
from rucio.client.downloadclient import DownloadClient
|
3
|
+
from rucio.client.replicaclient import ReplicaClient
|
4
|
+
from rucio.client.uploadclient import UploadClient
|
5
|
+
|
6
|
+
TEST_RSE = "STORAGE-1"
|
7
|
+
|
8
|
+
|
9
|
+
@pytest.mark.usefixtures("_auth_proxy")
|
10
|
+
@pytest.mark.verifies_requirement("C-BDMS-0330")
|
11
|
+
def test_file_localization(test_scope, tmp_path):
|
12
|
+
"""
|
13
|
+
Test that a file ingested in Rucio can be correctly localized using the list-replicas command.
|
14
|
+
"""
|
15
|
+
# Create a dummy file
|
16
|
+
file_name = "file_with_localization_test.txt"
|
17
|
+
lfn = f"/example.namespace/{test_scope}/{file_name}"
|
18
|
+
path = tmp_path / file_name
|
19
|
+
path.write_text("Test content for file localization")
|
20
|
+
|
21
|
+
# Upload the file to Rucio
|
22
|
+
upload_client = UploadClient()
|
23
|
+
upload_spec = {
|
24
|
+
"path": path,
|
25
|
+
"rse": TEST_RSE,
|
26
|
+
"did_scope": test_scope,
|
27
|
+
"did_name": lfn,
|
28
|
+
}
|
29
|
+
# Uploading the file
|
30
|
+
assert upload_client.upload([upload_spec]) == 0, "File upload failed"
|
31
|
+
|
32
|
+
# Verify file localization using list-replicas
|
33
|
+
replica_client = ReplicaClient()
|
34
|
+
replicas = list(replica_client.list_replicas([{"scope": test_scope, "name": lfn}]))
|
35
|
+
assert len(replicas) == 1, f"Expected 1 replica, found {len(replicas)}"
|
36
|
+
|
37
|
+
replica = replicas[0]
|
38
|
+
assert replica["name"] == lfn, f"Replica name mismatch: {replica['name']} != {lfn}"
|
39
|
+
assert TEST_RSE in replica["rses"], f"Replica not found in RSE {TEST_RSE}"
|
40
|
+
|
41
|
+
# Validate PFN and download the file
|
42
|
+
pfns = replica["rses"][TEST_RSE]
|
43
|
+
assert len(pfns) > 0, "No PFNs returned for the replica"
|
44
|
+
pfn = list(pfns)[0] # Extract the first PFN as a string
|
45
|
+
|
46
|
+
# Log the PFN for debugging
|
47
|
+
print(f"Using PFN: {pfn}")
|
48
|
+
|
49
|
+
# Prepare the input for download_pfns
|
50
|
+
download_spec = {
|
51
|
+
"pfn": pfn,
|
52
|
+
"did": f"{test_scope}:{lfn}",
|
53
|
+
"base_dir": str(tmp_path), # Ensure `dir` is correctly set
|
54
|
+
"rse": TEST_RSE, # Add `rse` if required by your Rucio setup
|
55
|
+
"no_subdir": True,
|
56
|
+
}
|
57
|
+
|
58
|
+
# Download the file using PFN
|
59
|
+
download_client = DownloadClient()
|
60
|
+
download_client.download_pfns([download_spec])
|
61
|
+
|
62
|
+
# Verify the contents of the downloaded file
|
63
|
+
download_path = (
|
64
|
+
tmp_path / file_name
|
65
|
+
) # The downloaded file should match the original name
|
66
|
+
assert download_path.exists(), f"Downloaded file does not exist at {download_path}"
|
67
|
+
downloaded_content = download_path.read_text()
|
68
|
+
original_content = path.read_text()
|
69
|
+
assert downloaded_content == original_content, (
|
70
|
+
f"Downloaded file content does not match the original. "
|
71
|
+
f"Expected: {original_content}, Got: {downloaded_content}"
|
72
|
+
)
|
@@ -0,0 +1,110 @@
|
|
1
|
+
from itertools import product
|
2
|
+
|
3
|
+
import pytest
|
4
|
+
from rucio.client.didclient import DIDClient
|
5
|
+
from rucio.client.uploadclient import UploadClient
|
6
|
+
|
7
|
+
TEST_RSE = "STORAGE-1"
|
8
|
+
|
9
|
+
|
10
|
+
@pytest.mark.usefixtures("_auth_proxy")
|
11
|
+
def test_add_metadata(test_scope, tmp_path):
|
12
|
+
"""Test adding/getting metadata works"""
|
13
|
+
name = "file_with_metadata"
|
14
|
+
lfn = f"/ctao.dpps.test/{test_scope}/name"
|
15
|
+
path = tmp_path / name
|
16
|
+
path.write_text("Hello, World!")
|
17
|
+
|
18
|
+
upload_client = UploadClient()
|
19
|
+
upload_spec = {
|
20
|
+
"path": path,
|
21
|
+
"rse": TEST_RSE,
|
22
|
+
"did_scope": test_scope,
|
23
|
+
"did_name": lfn,
|
24
|
+
}
|
25
|
+
# return value of 0 means success
|
26
|
+
assert upload_client.upload([upload_spec]) == 0
|
27
|
+
|
28
|
+
did_client = DIDClient()
|
29
|
+
meta = {
|
30
|
+
"obs_id": 200000001,
|
31
|
+
"tel_id": 1,
|
32
|
+
"category": "A",
|
33
|
+
"format": "zfits",
|
34
|
+
"data_levels": ["DL0", "DL1"],
|
35
|
+
"data_type": "event",
|
36
|
+
}
|
37
|
+
did_client.set_metadata_bulk(scope=test_scope, name=lfn, meta=meta)
|
38
|
+
|
39
|
+
# default for plugin is "DIDColumn", which only includes the internal rucio metadata
|
40
|
+
result = did_client.get_metadata(scope=test_scope, name=lfn, plugin="ALL")
|
41
|
+
|
42
|
+
# check all our metadata was received correctly
|
43
|
+
for key, value in meta.items():
|
44
|
+
assert key in result, f"Key {key} missing in retrieved metadata"
|
45
|
+
assert (
|
46
|
+
result[key] == value
|
47
|
+
), f"Key {key} has wrong value, expected {value}, got {result[key]}"
|
48
|
+
|
49
|
+
|
50
|
+
@pytest.fixture
|
51
|
+
def _metadata_test_dids(tmp_path, test_scope):
|
52
|
+
"""Setup a set of example files to test querying by metadata"""
|
53
|
+
upload_client = UploadClient()
|
54
|
+
did_client = DIDClient()
|
55
|
+
|
56
|
+
def add_file(obs_id, tel_id, data_type):
|
57
|
+
name = f"data_obs{obs_id}_tel{tel_id:03d}.{data_type}.txt"
|
58
|
+
lfn = f"/ctao.dpps.test/{test_scope}/{name}"
|
59
|
+
path = tmp_path / name
|
60
|
+
path.write_text(name)
|
61
|
+
|
62
|
+
spec = {
|
63
|
+
"did_scope": test_scope,
|
64
|
+
"did_name": lfn,
|
65
|
+
"path": path,
|
66
|
+
"rse": TEST_RSE,
|
67
|
+
}
|
68
|
+
upload_client.upload([spec])
|
69
|
+
meta = {"obs_id": obs_id, "tel_id": tel_id, "data_type": data_type}
|
70
|
+
did_client.set_metadata_bulk(test_scope, lfn, meta=meta)
|
71
|
+
|
72
|
+
obs_ids = (200000001, 200000002, 200000003)
|
73
|
+
tel_ids = (1, 2, 3, 4)
|
74
|
+
data_types = ("event", "monitoring", "service")
|
75
|
+
|
76
|
+
for obs_id, tel_id, data_type in product(obs_ids, tel_ids, data_types):
|
77
|
+
add_file(obs_id, tel_id, data_type)
|
78
|
+
|
79
|
+
|
80
|
+
@pytest.mark.usefixtures("_auth_proxy", "_metadata_test_dids")
|
81
|
+
@pytest.mark.verifies_requirement("C-BDMS-0210")
|
82
|
+
def test_dataset_retrieval_by_metadata(test_scope):
|
83
|
+
"""Test querying dids by metadata attributes"""
|
84
|
+
|
85
|
+
did_client = DIDClient()
|
86
|
+
|
87
|
+
# query for by one attribute, should return len(tel_ids) * len(data_types) files
|
88
|
+
obs_id = 200000002
|
89
|
+
dids = list(
|
90
|
+
did_client.list_dids(test_scope, filters={"obs_id": obs_id}, did_type="file")
|
91
|
+
)
|
92
|
+
assert len(dids) == 12
|
93
|
+
assert all(str(obs_id) in did for did in dids)
|
94
|
+
|
95
|
+
# query for by two attributes
|
96
|
+
obs_id = 200000002
|
97
|
+
data_type = "event"
|
98
|
+
query = {"obs_id": obs_id, "data_type": data_type}
|
99
|
+
dids = list(did_client.list_dids(test_scope, filters=query, did_type="file"))
|
100
|
+
assert len(dids) == 4
|
101
|
+
assert all(str(obs_id) in did and data_type in did for did in dids)
|
102
|
+
|
103
|
+
# query using comparison operator
|
104
|
+
# should only return entries for obs_id > 200000002, so only for 200000003
|
105
|
+
obs_id = 200000002
|
106
|
+
expected_obs_id = 200000003
|
107
|
+
query = {"obs_id.gt": obs_id, "data_type": data_type}
|
108
|
+
dids = list(did_client.list_dids(test_scope, filters=query, did_type="file"))
|
109
|
+
assert len(dids) == 4
|
110
|
+
assert all(str(expected_obs_id) in did and data_type in did for did in dids)
|
bdms/version.py
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
"""Version information."""
|
2
|
+
# this is adapted from https://github.com/astropy/astropy/blob/main/astropy/version.py
|
3
|
+
# see https://github.com/astropy/astropy/pull/10774 for a discussion on why this needed.
|
4
|
+
|
5
|
+
try:
|
6
|
+
try:
|
7
|
+
from ._dev_version import version
|
8
|
+
except Exception:
|
9
|
+
from ._version import version
|
10
|
+
except Exception:
|
11
|
+
import warnings
|
12
|
+
|
13
|
+
warnings.warn(
|
14
|
+
"Could not determine version; this indicates a broken installation."
|
15
|
+
" Install from PyPI, using conda or from a local git repository."
|
16
|
+
" Installing github's autogenerated source release tarballs "
|
17
|
+
" does not include version information and should be avoided.",
|
18
|
+
)
|
19
|
+
del warnings
|
20
|
+
version = "0.0.0"
|
21
|
+
|
22
|
+
__version__ = version
|
23
|
+
__all__ = ["__version__"]
|
@@ -0,0 +1,29 @@
|
|
1
|
+
BSD 3-Clause License
|
2
|
+
|
3
|
+
Copyright (c) 2022, Cherenkov Telescope Array Observatory Consortium
|
4
|
+
All rights reserved.
|
5
|
+
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
8
|
+
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
10
|
+
list of conditions and the following disclaimer.
|
11
|
+
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
14
|
+
and/or other materials provided with the distribution.
|
15
|
+
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
17
|
+
contributors may be used to endorse or promote products derived from
|
18
|
+
this software without specific prior written permission.
|
19
|
+
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
@@ -0,0 +1,35 @@
|
|
1
|
+
Metadata-Version: 2.2
|
2
|
+
Name: ctao-bdms-clients
|
3
|
+
Version: 0.0.0a0
|
4
|
+
Summary: Client module for the CTAO DPPS Bulk Data Management System
|
5
|
+
Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
|
6
|
+
License: BSD-3-Clause
|
7
|
+
Project-URL: repository, https://github.com/cta-observatory/...
|
8
|
+
Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/documentation/...
|
9
|
+
Requires-Python: >=3.9
|
10
|
+
Description-Content-Type: text/markdown
|
11
|
+
License-File: LICENSE
|
12
|
+
Requires-Dist: rucio-clients~=35.4.1
|
13
|
+
Requires-Dist: ctao-bdms-rucio-policy==0.1.0
|
14
|
+
Provides-Extra: test
|
15
|
+
Requires-Dist: pytest; extra == "test"
|
16
|
+
Requires-Dist: pytest-cov; extra == "test"
|
17
|
+
Requires-Dist: pytest-mock; extra == "test"
|
18
|
+
Requires-Dist: pytest-requirements; extra == "test"
|
19
|
+
Provides-Extra: doc
|
20
|
+
Requires-Dist: sphinx; extra == "doc"
|
21
|
+
Requires-Dist: numpydoc; extra == "doc"
|
22
|
+
Requires-Dist: ctao-sphinx-theme; extra == "doc"
|
23
|
+
Requires-Dist: myst-parser; extra == "doc"
|
24
|
+
Requires-Dist: sphinx-changelog; extra == "doc"
|
25
|
+
Provides-Extra: dev
|
26
|
+
Requires-Dist: setuptools_scm; extra == "dev"
|
27
|
+
Requires-Dist: sphinx-autobuild; extra == "dev"
|
28
|
+
Provides-Extra: all
|
29
|
+
Requires-Dist: bdms[dev,doc,test]; extra == "all"
|
30
|
+
|
31
|
+
# Bulk Data Management System
|
32
|
+
|
33
|
+
[](https://gitlab.cta-observatory.org/dpps/bdms/bdms/-/pipelines/main/latest)
|
34
|
+
[](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/jobs/artifacts/main/file/test_report.pdf?job=build-test-report)
|
35
|
+
[](http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms/)
|
@@ -0,0 +1,16 @@
|
|
1
|
+
bdms/__init__.py,sha256=7btE6tNhFqXSv2eUhZ-0m1J3nTTs4Xo6HWcQI4eh5Do,142
|
2
|
+
bdms/_version.py,sha256=PfEj3cQaCLALpwUXFQl23PozAfDHPiMS029TqAVVXCg,413
|
3
|
+
bdms/version.py,sha256=mTfi1WzbIs991NyImM6mcMg1R39a6U1W2pKnk-Tt5Vw,765
|
4
|
+
bdms/_dev_version/__init__.py,sha256=3qlzT1l_MfLxHuRphBwNwkb2WRttg3hGooj5n3BBZi4,369
|
5
|
+
bdms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
|
+
bdms/tests/conftest.py,sha256=vN96UXhO2503svkg3V2jxfq5SSUtlRCWZt5MQgblu7Y,1374
|
7
|
+
bdms/tests/test_basic_rucio_functionality.py,sha256=Sm6Kif8g5xPaw6tIIJ1TavDxVje2YaYYC9PnhVbVQFI,3999
|
8
|
+
bdms/tests/test_dpps_rel_0_0.py,sha256=MnbuBoS_kUUiMcHE3-jqOzekQNUa-wcsjCJqJQ2J9S4,2957
|
9
|
+
bdms/tests/test_file_replicas.py,sha256=NqutrSJa5ME50JpmyATNPSLqq1AOq1ruv84XSY3PKLI,2635
|
10
|
+
bdms/tests/test_metadata.py,sha256=f0tSqNGlYe-ydoSDJw0k1De2kHoPl6g-GYBj_jP6kCY,3728
|
11
|
+
ctao_bdms_clients-0.0.0a0.dist-info/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
|
12
|
+
ctao_bdms_clients-0.0.0a0.dist-info/METADATA,sha256=IgEbSv7mrsdcVdq3vqH8sPrUmNSY66zsv8kk7ewcJME,2124
|
13
|
+
ctao_bdms_clients-0.0.0a0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
14
|
+
ctao_bdms_clients-0.0.0a0.dist-info/entry_points.txt,sha256=QTq7DkTh87oL0SpuNEooUuNSeU8Y5C3mUTbwufMvBRA,53
|
15
|
+
ctao_bdms_clients-0.0.0a0.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
|
16
|
+
ctao_bdms_clients-0.0.0a0.dist-info/RECORD,,
|
@@ -0,0 +1 @@
|
|
1
|
+
bdms
|