panda-server-source 0.8.0__py3-none-any.whl

pandaserver/pandamcp/mcp_test_client.py

@@ -0,0 +1,95 @@
+import argparse
+import asyncio
+
+from fastmcp import Client
+from fastmcp.client.transports import SSETransport, StreamableHttpTransport
+
+# argparse setup
+parser = argparse.ArgumentParser(description="MCP test client")
+parser.add_argument(
+    "--transport",
+    type=str,
+    choices=("streamable-http", "sse"),
+    default="streamable-http",
+    help="Transport to use: streamable-http or sse (default: streamable-http)",
+)
+parser.add_argument("--host", type=str, default="localhost", help="PanDA Server host (default: localhost)")
+parser.add_argument("--port", type=int, default=25443, help="PanDA Server port (default: 25443)")
+parser.add_argument("--use_http", action=argparse.BooleanOptionalAction, default=False, help="Use HTTP instead of HTTPS (default: False)")
+parser.add_argument("--token", type=str, default=None, help="OIDC ID token for write-operations (default: None)")
+parser.add_argument("--vo", type=str, default=None, help="Virtual organization with ID token is given (default: None)")
+parser.add_argument("--tool", type=str, default="is_alive", help="A tool name to test (default: is_alive)")
+parser.add_argument(
+    "--kv",
+    dest="kv",
+    metavar="KEY=VAL",
+    action="append",
+    help="Arguments to invoke the test function. Key=Value pairs repeat for multiple (e.g. --kv a=1 --kv b=2 --kv c=True)",
+)
+args = parser.parse_args()
+
+# parse --kv KEY=VAL (repeatable) into a dict stored back on args.kv
+raw_kv = args.kv or []
+kv = {}
+for item in raw_kv:
+    if "=" not in item:
+        parser.error(f"invalid --kv value {item!r}, expected KEY=VALUE")
+    k, v = item.split("=", 1)
+    v = v.strip()
+    if v == "True":
+        v = True
+    elif v == "False":
+        v = False
+    kv[k] = v
+args.kv = kv
+
+# construct base URL
+if args.use_http:
+    base_url = f"http://{args.host}:{args.port}/mcp/"
+else:
+    base_url = f"https://{args.host}:{args.port}/mcp/"
+
+headers = {"Origin": args.vo} if args.token else None
+
+# select transport
+if args.transport == "streamable-http":
+    transport = StreamableHttpTransport(url=base_url, auth=args.token, headers=headers)
+else:
+    transport = SSETransport(url=base_url, auth=args.token, headers=headers)
+
+# create client
+client = Client(transport)
+
+
+# test function
+async def cl():
+    # Connection is established here
+    async with client:
+        if client.is_connected():
+            print("Client connected")
+        else:
+            print("Client failed to connect")
+            return
+
+        # Make MCP calls within the context
+        tools = await client.list_tools()
+        print(f"\nAvailable tools:")
+        for tool in tools:
+            print(f"- {tool.name} -\n")
+            print(f" Description: {tool.description}")
+
+        print("\n" * 2)
+        print(f"Testing {args.tool}:")
+        result = await client.call_tool(args.tool, args.kv)
+        print(f"Result: {result}\n")
+
+    # Connection is closed automatically here
+    if not client.is_connected():
+        print("Client disconnected")
+    else:
+        print("Client still connected")
+    print("Done")
+
+
+if __name__ == "__main__":
+    asyncio.run(cl())

pandaserver/pandamcp/mcp_utils.py

@@ -0,0 +1,74 @@
+import inspect
+import os.path
+from collections.abc import Callable
+
+from fastmcp.server.dependencies import get_http_headers
+from fastmcp.tools.tool import Tool
+
+from pandaserver.api.v1.http_client import HttpClient, api_url, api_url_ssl
+from pandaserver.srvcore.panda_request import PandaRequest
+
+
+def create_tool(func: Callable) -> Tool:
+    """
+    Create an MCP tool that wraps the API call.
+
+    :param func: The function to wrap. It should be a callable that takes PandaRequest as the first argument.
+    :return: An MCP tool.
+    """
+
+    # construct the URL based on the module and function name
+    mod_path = inspect.getfile(inspect.getmodule(func))
+    mod_name = os.path.basename(mod_path).split("_")[0]
+
+    # determine http method based on the docstring
+    http_method = None
+    for line in func.__doc__.splitlines():
+        line = line.strip()
+        if line.startswith("HTTP Method:"):
+            http_method = line.split(":")[1].strip().lower()
+            if http_method not in ["get", "post"]:
+                raise ValueError(f"Unsupported HTTP method: {http_method}")
+            break
+    if http_method is None:
+        raise ValueError("HTTP Method not specified in the function docstring")
+
+    if http_method == "get":
+        url = f"{api_url}/{mod_name}/{func.__name__}"
+    else:
+        url = f"{api_url_ssl}/{mod_name}/{func.__name__}"
+
+    # remove PandaRequest from the signature and annotations
+    sig = inspect.signature(func)
+    params = []
+    for p in sig.parameters.values():
+        if p.annotation != PandaRequest:
+            params.append(p)
+
+    annotations = {}
+    for k, v in func.__annotations__.items():
+        if v != PandaRequest:
+            annotations[k] = v
+
+    # create a new function that wraps the API call
+    def wrapped_func(**kwarg):
+        nonlocal url, http_method
+        # extract the id_token and auth_vo from the headers
+        original_headers = get_http_headers()
+        id_token = original_headers.get("authorization")
+        if id_token and id_token.startswith("Bearer "):
+            id_token = original_headers["authorization"].split(" ")[1]
+        auth_vo = original_headers.get("origin")
+        oidc = id_token is not None
+
+        # set the id_token and auth_vo in the HttpClient
+        http_client = HttpClient()
+        http_client.override_oidc(oidc, id_token, auth_vo)
+        status, output = getattr(http_client, http_method)(url, kwarg)
+        return output
+
+    # set the signature and annotations to the wrapped function to align with the original API call
+    wrapped_func.__signature__ = sig.replace(parameters=params)
+    wrapped_func.__annotations__ = annotations
+
+    return Tool.from_function(wrapped_func, name=func.__name__, description=func.__doc__)

pandaserver/proxycache/panda_proxy_cache.py

@@ -0,0 +1,240 @@
+import datetime
+import hashlib
+import os
+import shutil
+import subprocess
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+_logger = PandaLogger().getLogger("ProxyCache")
+
+
+def execute(program, log_stream):
+    """Run a program on the command line. Return stderr, stdout and status."""
+    log_stream.info(f"executable: {program}")
+    pipe = subprocess.Popen(
+        program,
+        bufsize=-1,
+        shell=True,
+        close_fds=False,
+        cwd="/tmp",
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = pipe.communicate()
+    return stdout, stderr, pipe.wait()
+
+
+def cat(filename):
+    """Given filename, print its text contents."""
+    with open(filename, "r") as f:
+        out = f.read()
+    return out
+
+
+class MyProxyInterface(object):
+    """Class to store and retrieve proxies from my proxies."""
+
+    def __init__(self):
+        self.__target_path = "/tmp/proxies"
+        self.__cred_name = "panda"
+        if not os.path.exists(self.__target_path):
+            os.makedirs(self.__target_path)
+
+    def store(
+        self,
+        user_dn,
+        cred_name,
+        production=False,
+        server_name="myproxy.cern.ch",
+        role=None,
+        log_stream=None,
+    ):
+        log_stream.info("store proxy")
+
+        # Retrieve proxy from myproxy
+        proxy_path = os.path.join(
+            self.__target_path,
+            hashlib.sha1(f"{user_dn}.plain".encode("utf-8")).hexdigest(),
+        )
+
+        # check if empty dummy file
+        if os.path.exists(proxy_path) and os.stat(proxy_path).st_size == 0:
+            if datetime.datetime.now(datetime.timezone.utc) - datetime.datetime.fromtimestamp(
+                os.path.getctime(proxy_path), datetime.timezone.utc
+            ) < datetime.timedelta(hours=1):
+                log_stream.info(f"skip too early to try again according to timestamp of {proxy_path}")
+                return 2
+        cmd = f"myproxy-logon -s {server_name} --no_passphrase --out {proxy_path} -l '{user_dn}' -k {cred_name} -t 0"
+
+        stdout, stderr, status = execute(cmd, log_stream)
+        if stdout:
+            log_stream.info(f"stdout is {stdout} ")
+        if stderr:
+            log_stream.info(f"stderr is {stderr} ")
+            # make a dummy to avoid too early reattempt
+            open(proxy_path, "w").close()
+        log_stream.info(f"test the status of plain... {status}")
+        if status == 1:
+            return status
+
+        if role is not None:
+            log_stream.info(f"proxy needs {role} - need to add voms attributes and store it in the cache")
+            tmpExtension = self.getExtension(role)
+            prodproxy_path = os.path.join(
+                self.__target_path,
+                str(hashlib.sha1((user_dn + tmpExtension).encode("utf-8")).hexdigest()),
+            )
+            prodcmd = f"voms-proxy-init -vomses /etc/vomses -valid 96:00 -rfc -cert {proxy_path} -key {proxy_path} -out {prodproxy_path} -n -voms {role}"
+            stdout, stderr, status = execute(prodcmd, log_stream)
+            if stdout:
+                log_stream.info(f"stdout is {stdout} ")
+            if stderr:
+                log_stream.info(f"stderr is {stderr} ")
+            log_stream.debug(f"test the status of production... {status}")
+        elif production:
+            log_stream.info("production proxy needed - need to add voms attributes and store it in the cache")
+            prodproxy_path = os.path.join(
+                self.__target_path,
+                str(hashlib.sha1(f"{user_dn}.prod".encode("utf-8")).hexdigest()),
+            )
+            log_stream.info(prodproxy_path)
+            prodcmd = f"voms-proxy-init -vomses /etc/vomses -valid 96:00 -rfc -cert {proxy_path} -key {proxy_path} -out {prodproxy_path} -n -voms atlas:/atlas/Role=production"
+            stdout, stderr, status = execute(prodcmd, log_stream)
+            if stdout:
+                log_stream.info(f"stdout is {stdout} ")
+            if stderr:
+                log_stream.info(f"stderr is {stderr} ")
+            log_stream.info(f"test the status of production... {status}")
+        else:
+            # Now we need to add atlas roles and store it
+            atlasproxy_path = os.path.join(self.__target_path, hashlib.sha1(user_dn.encode("utf-8")).hexdigest())
+            atlasrolescmd = f"voms-proxy-init -vomses /etc/vomses -valid 96:00 -rfc -cert {proxy_path} -key {proxy_path} -out {atlasproxy_path} -n -voms atlas"
+            stdout, stderr, status = execute(atlasrolescmd, log_stream)
+            if stdout:
+                log_stream.info(f"stdout is {stdout} ")
+            if stderr:
+                log_stream.info(f"stderr is {stderr} ")
+            log_stream.info(f"test the status of atlas... {status}")
+        # make dummy to avoid too early attempts
+        if status != 0 and not os.path.exists(proxy_path):
+            open(proxy_path, "w").close()
+        return status
+
+    def retrieve(self, user_dn, production=False, role=None):
+        """Retrieve proxy from proxy cache."""
+        if role is not None:
+            tmpExtension = self.getExtension(role)
+            proxy_path = os.path.join(
+                self.__target_path,
+                str(hashlib.sha1((user_dn + tmpExtension).encode("utf-8")).hexdigest()),
+            )
+        elif production:
+            proxy_path = os.path.join(
+                self.__target_path,
+                str(hashlib.sha1(f"{user_dn}.prod".encode("utf-8")).hexdigest()),
+            )
+        else:
+            proxy_path = os.path.join(self.__target_path, hashlib.sha1(user_dn.encode("utf-8")).hexdigest())
+        if os.path.isfile(proxy_path):
+            return cat(proxy_path)
+        else:
+            _logger.warning(f"proxy file does not exist : DN:{user_dn} role:{role} file:{proxy_path}")
+
+    # get proxy path
+    def get_proxy_path(self, user_dn, production, role):
+        if role is not None:
+            tmpExtension = self.getExtension(role)
+            return os.path.join(
+                self.__target_path,
+                str(hashlib.sha1((user_dn + tmpExtension).encode("utf-8")).hexdigest()),
+            )
+        elif production:
+            return os.path.join(
+                self.__target_path,
+                str(hashlib.sha1(f"{user_dn}.prod".encode("utf-8")).hexdigest()),
+            )
+        else:
+            return os.path.join(
+                self.__target_path,
+                hashlib.sha1(user_dn.encode("utf-8")).hexdigest(),
+            )
+
+    def checkProxy(self, user_dn, production=False, role=None, name=None):
+        log_stream = LogWrapper(_logger, f'< name="{name}" role={role} >')
+        log_stream.info(f"check proxy for {user_dn}")
+
+        # Check the validity of a proxy
+        proxy_path = self.get_proxy_path(user_dn, production, role)
+        is_ok = False
+        if os.path.isfile(proxy_path):
+            log_stream.info("proxy is there. Need to check validity")
+            cmd = f"voms-proxy-info -exists -hours 94 -file {proxy_path}"
+            stdout, stderr, status = execute(cmd, log_stream)
+            if stdout:
+                log_stream.info(f"stdout is {stdout} ")
+            if stderr:
+                log_stream.info(f"stderr is {stderr} ")
+            if status == 1:
+                log_stream.info("proxy expires in 94h or less. We need to renew proxy!")
+                ret = self.store(
+                    user_dn,
+                    self.__cred_name,
+                    production,
+                    role=role,
+                    log_stream=log_stream,
+                )
+                if ret == 0:
+                    log_stream.info("proxy retrieval successful")
+                    # copy with compact name
+                    alt_proxy_path = self.get_proxy_path(name, production, role)
+                    shutil.copyfile(proxy_path, alt_proxy_path)
+                    is_ok = True
+                elif ret == 2:
+                    log_stream.info("proxy retrieval on hold")
+                else:
+                    log_stream.error("proxy retrieval failed")
+            else:
+                log_stream.info("proxy is valid for more than 3 days")
+                is_ok = True
+        else:
+            log_stream.info("proxy is not in the cache repo. will try to get it from myproxy")
+            ret = self.store(user_dn, self.__cred_name, production, role=role, log_stream=log_stream)
+            if ret == 0:
+                log_stream.info("proxy stored successfully")
+                alt_proxy_path = self.get_proxy_path(name, production, role)
+                shutil.copyfile(proxy_path, alt_proxy_path)
+                is_ok = True
+            elif ret == 2:
+                log_stream.info("proxy retrieval on hold")
+            else:
+                log_stream.error("proxy retrieval failed")
+        if is_ok:
+            plain_path = os.path.join(
+                self.__target_path,
+                hashlib.sha1(f"{user_dn}.plain".encode("utf-8")).hexdigest(),
+            )
+            if os.path.isfile(plain_path):
+                return self.checkValidity(plain_path, log_stream)
+            else:
+                log_stream.error("plain proxy not there at the moment!")
+
+    def checkValidity(self, proxy_path, log_stream):
+        log_stream.info("Need to check validity and expiry!")
+        time_left_thresholds = [24, 94, 168]
+        status = 0
+        for threshold in time_left_thresholds:
+            cmd = f"voms-proxy-info -exists -hours {threshold} -file {proxy_path}"
+            stdout, stderr, status = execute(cmd, log_stream)
+            if status == 1:
+                log_stream.warning(f"proxy expires in {threshold} hours")
+                return threshold
+
+        return status
+
+    # get extension
+    def getExtension(self, role):
+        if role is not None:
+            return "." + role.split("=")[-1]
+        return None

pandaserver/proxycache/token_cache.py

@@ -0,0 +1,141 @@
+"""
+download access tokens for OIDC token exchange flow
+"""
+
+import datetime
+import json
+import os.path
+import pathlib
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+from pandaserver.config import panda_config
+from pandaserver.srvcore.oidc_utils import get_access_token
+
+# logger
+_logger = PandaLogger().getLogger("token_cache")
+
+
+class TokenCache:
+    """
+    A class used to download and give access tokens for OIDC token exchange flow
+
+    """
+
+    # constructor
+    def __init__(self, target_path: str = None, file_prefix: str = None, refresh_interval: int = 60, task_buffer=None):
+        """
+        Constructs all the necessary attributes for the TokenCache object.
+
+        :param target_path: The base path to store the access tokens
+        :param file_prefix: The prefix of the access token files
+        :param refresh_interval: The interval to refresh the access tokens (default is 60 minutes)
+        :param task_buffer: TaskBuffer object
+        """
+        if target_path:
+            self.target_path = target_path
+        else:
+            self.target_path = "/tmp/proxies"
+        if not os.path.exists(self.target_path):
+            os.makedirs(self.target_path)
+        if file_prefix:
+            self.file_prefix = file_prefix
+        else:
+            self.file_prefix = "access_token_"
+        self.refresh_interval = refresh_interval
+        self.task_buffer = task_buffer
+        # cache for access tokens
+        self.cached_access_tokens = {}
+
+    # construct target path
+    def construct_target_path(self, client_name: str) -> str:
+        """
+        Constructs the target path to store an access token
+
+        :param client_name: client name
+        :return: the target path
+        """
+        return os.path.join(self.target_path, f"{self.file_prefix}{client_name}")
+
+    # main
+    def run(self):
+        """ "
+        Main function to download access tokens
+        """
+        tmp_log = LogWrapper(_logger)
+        tmp_log.debug("================= start ==================")
+        try:
+            # check config
+            if not hasattr(panda_config, "token_cache_config") or not panda_config.token_cache_config:
+                tmp_log.debug("token_cache_config is not set in panda_config")
+            # check config path
+            elif not os.path.exists(panda_config.token_cache_config):
+                tmp_log.debug(f"config file {panda_config.token_cache_config} not found")
+            # read config
+            else:
+                with open(panda_config.token_cache_config) as f:
+                    token_cache_config = json.load(f)
+                for client_name, client_config in token_cache_config.items():
+                    tmp_log.debug(f"client_name={client_name}")
+                    # token file path
+                    token_file_path = client_config.get("token_file_path")
+                    if not token_file_path:
+                        token_file_path = self.construct_target_path(client_name)
+                    # check if fresh
+                    is_fresh = False
+                    if os.path.exists(token_file_path):
+                        mod_time = datetime.datetime.fromtimestamp(os.stat(token_file_path).st_mtime, datetime.timezone.utc)
+                        if datetime.datetime.now(datetime.timezone.utc) - mod_time < datetime.timedelta(minutes=self.refresh_interval):
+                            tmp_log.debug(f"skip since {token_file_path} is fresh")
+                            is_fresh = True
+                    # get access token
+                    if not is_fresh:
+                        status_code, output = get_access_token(
+                            client_config["endpoint"], client_config["client_id"], client_config["secret"], client_config.get("scope")
+                        )
+                        if status_code:
+                            with open(token_file_path, "w") as f:
+                                f.write(output)
+                            tmp_log.debug(f"dump access token to {token_file_path}")
+                        else:
+                            tmp_log.error(output)
+                            # touch file to avoid immediate reattempt
+                            pathlib.Path(token_file_path).touch()
+                            tmp_log.debug(f"touch {token_file_path} to avoid immediate reattempt")
+                    # register token keys
+                    if client_config.get("use_token_key") is True and self.task_buffer is not None:
+                        token_key_lifetime = client_config.get("token_key_lifetime", 96)
+                        tmp_log.debug(f"register token key for {client_name}")
+                        tmp_stat = self.task_buffer.register_token_key(client_name, token_key_lifetime)
+                        if not tmp_stat:
+                            tmp_log.error("failed")
+        except Exception as e:
+            tmp_log.error(f"failed with {str(e)}")
+        tmp_log.debug("================= end ==================")
+        tmp_log.debug("done")
+        return
+
+    # get access token for a client
+    def get_access_token(self, client_name: str) -> str | None:
+        """
+        Get an access token string for a client. None is returned if the access token is not found
+
+        :param client_name : client name
+        :return: the access token
+        """
+        time_now = naive_utcnow()
+        if client_name in self.cached_access_tokens and self.cached_access_tokens[client_name]["last_update"] + datetime.timedelta(minutes=10) < time_now:
+            # use cached token since it is still fresh
+            pass
+        else:
+            target_path = self.construct_target_path(client_name)
+            token = None
+            if os.path.exists(target_path):
+                with open(target_path) as f:
+                    token = f.read()
+            if not token:
+                token = None
+            self.cached_access_tokens[client_name] = {"token": token, "last_update": time_now}
+        return self.cached_access_tokens[client_name]["token"]

pandaserver/server/.gacl

@@ -0,0 +1,6 @@
+<gacl>
+<entry>
+  <any-user/>
+  <allow><read/><list/></allow>
+</entry>
+</gacl>