rucio-clients 32.8.6__py3-none-any.whl

rucio/common/test_rucio_server.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from os import remove
+from os.path import basename
+
+from rucio.common.utils import generate_uuid as uuid, execute
+
+
+def file_generator(size=2048, namelen=10):
+    """ Create a bogus file and returns it's name.
+    :param size: size in bytes
+    :returns: The name of the generated file.
+    """
+    fn = '/tmp/rucio_testfile_' + uuid()
+    execute('dd if=/dev/urandom of={0} count={1} bs=1'.format(fn, size))
+    return fn
+
+
+def get_scope_and_rses():
+    """
+    Check if xrd containers rses for xrootd are available in the testing environment.
+
+    :return: A tuple (scope, rses) for the rucio client where scope is mock/test and rses is a list or (None, [None]) if no suitable rse exists.
+    """
+    cmd = "rucio list-rses --rses 'test_container_xrd=True'"
+    print(cmd)
+    exitcode, out, err = execute(cmd)
+    print(out, err)
+    rses = out.split()
+    if len(rses) == 0:
+        return None, [None]
+    return 'test', rses
+
+
+def delete_rules(did):
+    # get the rules for the file
+    print('Deleting rules')
+    cmd = "rucio list-rules {0} | grep {0} | cut -f1 -d\\ ".format(did)
+    print(cmd)
+    exitcode, out, err = execute(cmd)
+    print(out, err)
+    rules = out.split()
+    # delete the rules for the file
+    for rule in rules:
+        cmd = "rucio delete-rule {0}".format(rule)
+        print(cmd)
+        exitcode, out, err = execute(cmd)
+
+
+class TestRucioServer(unittest.TestCase):
+
+    def setUp(self):
+        self.marker = '$ > '
+        self.scope, self.rses = get_scope_and_rses()
+        self.rse = self.rses[0]
+        self.generated_dids = []
+
+    def tearDown(self):
+        for did in self.generated_dids:
+            delete_rules(did)
+        self.generated_dids = []
+
+    def test_ping(self):
+        """CLIENT (USER): rucio ping"""
+        cmd = 'rucio ping'
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out, err)
+        self.assertEqual(exitcode, 0)
+
+    def test_whoami(self):
+        """CLIENT (USER): rucio whoami"""
+        cmd = 'rucio whoami'
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out, err)
+        self.assertEqual(exitcode, 0)
+
+    def test_upload_download(self):
+        """CLIENT(USER): rucio upload files to dataset/download dataset"""
+        if self.rse is None:
+            return
+
+        tmp_file1 = file_generator()
+        tmp_file2 = file_generator()
+        tmp_file3 = file_generator()
+        tmp_dsn = 'tests.rucio_client_test_server_' + uuid()
+
+        # Adding files to a new dataset
+        cmd = 'rucio upload --rse {0} --scope {1} {2} {3} {4} {1}:{5}'.format(self.rse, self.scope, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        remove(tmp_file1)
+        remove(tmp_file2)
+        remove(tmp_file3)
+        self.assertEqual(exitcode, 0)
+
+        # List the files
+        cmd = 'rucio list-files {0}:{1}'.format(self.scope, tmp_dsn)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        self.assertEqual(exitcode, 0)
+
+        # List the replicas
+        cmd = 'rucio list-file-replicas {0}:{1}'.format(self.scope, tmp_dsn)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        self.assertEqual(exitcode, 0)
+
+        # Downloading dataset
+        cmd = 'rucio download --dir /tmp/ {0}:{1}'.format(self.scope, tmp_dsn)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        # The files should be there
+        cmd = 'ls /tmp/{0}/rucio_testfile_*'.format(tmp_dsn)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(err, out)
+        self.assertEqual(exitcode, 0)
+
+        # cleaning
+        remove('/tmp/{0}/'.format(tmp_dsn) + basename(tmp_file1))
+        remove('/tmp/{0}/'.format(tmp_dsn) + basename(tmp_file2))
+        remove('/tmp/{0}/'.format(tmp_dsn) + basename(tmp_file3))
+        added_dids = ['{0}:{1}'.format(self.scope, did) for did in (basename(tmp_file1), basename(tmp_file2), basename(tmp_file3), tmp_dsn)]
+        self.generated_dids += added_dids

rucio/common/types.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Optional, TypedDict, Union
+
+
+class InternalType(object):
+    '''
+    Base for Internal representations of string types
+    '''
+    def __init__(self, value, vo='def', fromExternal=True):
+        if value is None:
+            self.external = None
+            self.internal = None
+            self.vo = vo
+        elif not isinstance(value, str):
+            raise TypeError('Expected value to be string type, got %s' % type(value))
+        elif fromExternal:
+            self.external = value
+            self.vo = vo
+            self.internal = self._calc_internal()
+        else:
+            self.internal = value
+            vo, external = self._calc_external()
+            self.external = external
+            self.vo = vo
+
+    def __repr__(self):
+        return self.internal
+
+    def __str__(self):
+        return self.external
+
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            return self.internal == other.internal
+        return NotImplemented
+
+    def __ne__(self, other):
+        val = self == other
+        if val is NotImplemented:
+            return NotImplemented
+        return not val
+
+    def __le__(self, other):
+        val = self.external <= other.external
+        if val is NotImplemented:
+            return NotImplemented
+        return val
+
+    def __lt__(self, other):
+        val = self.external < other.external
+        if val is NotImplemented:
+            return NotImplemented
+        return val
+
+    def __hash__(self):
+        return hash(self.internal)
+
+    def _calc_external(self):
+        ''' Utility to convert between internal and external representations'''
+        split = self.internal.split('@', 1)
+        if len(split) == 1:  # if cannot convert, vo is '' and this is single vo
+            vo = 'def'
+            external = split[0]
+        else:
+            vo = split[1]
+            external = split[0]
+        return vo, external
+
+    def _calc_internal(self):
+        ''' Utility to convert between internal and external representations'''
+        if self.vo == 'def':
+            return self.external
+        internal = '{}@{}'.format(self.external, self.vo)
+        return internal
+
+
+class InternalAccount(InternalType):
+    '''
+    Internal representation of an account
+    '''
+    def __init__(self, account, vo='def', fromExternal=True):
+        super(InternalAccount, self).__init__(value=account, vo=vo, fromExternal=fromExternal)
+
+
+class InternalScope(InternalType):
+    '''
+    Internal representation of a scope
+    '''
+    def __init__(self, scope, vo='def', fromExternal=True):
+        super(InternalScope, self).__init__(value=scope, vo=vo, fromExternal=fromExternal)
+
+
+class RSEDomainLANDict(TypedDict):
+    read: Optional[int]
+    write: Optional[int]
+    delete: Optional[int]
+
+
+class RSEDomainWANDict(TypedDict):
+    read: Optional[int]
+    write: Optional[int]
+    delete: Optional[int]
+    third_party_copy_read: Optional[int]
+    third_party_copy_write: Optional[int]
+
+
+class RSEDomainsDict(TypedDict):
+    lan: RSEDomainLANDict
+    wan: RSEDomainWANDict
+
+
+class RSEProtocolDict(TypedDict):
+    auth_token: Optional[str]  # FIXME: typing.NotRequired
+    hostname: str
+    scheme: str
+    port: int
+    prefix: str
+    impl: str
+    domains: RSEDomainsDict
+    extended_attributes: Optional[Union[str, dict[str, Any]]]
+
+
+class RSESettingsDict(TypedDict):
+    availability_delete: bool
+    availability_read: bool
+    availability_write: bool
+    credentials: Optional[dict[str, Any]]
+    lfn2pfn_algorithm: str
+    qos_class: Optional[str]
+    staging_area: bool
+    rse_type: str
+    sign_url: Optional[str]
+    read_protocol: int
+    write_protocol: int
+    delete_protocol: int
+    third_party_copy_read_protocol: int
+    third_party_copy_write_protocol: int
+    id: str
+    rse: str
+    volatile: bool
+    verify_checksum: bool
+    deterministic: bool
+    domain: list[str]
+    protocols: list[RSEProtocolDict]