panda-server-source 0.8.0__py3-none-any.whl

pandaserver/srvcore/allowed_methods.py

@@ -0,0 +1,100 @@
+# List of methods that can be executed by the clients
+
+allowed_methods = []
+
+# methods from pandaserver.taskbuffer.Utils
+allowed_methods += [
+    "isAlive",
+    "putFile",
+    "deleteFile",
+    "getServer",
+    "updateLog",
+    "fetchLog",
+    "touchFile",
+    "getVomsAttr",
+    "putEventPickingRequest",
+    "getAttr",
+    "uploadLog",
+    "put_checkpoint",
+    "delete_checkpoint",
+    "put_file_recovery_request",
+    "put_workflow_request",
+]
+# methods from pandaserver.jobdispatcher.JobDispatcher
+allowed_methods += [
+    "getJob",
+    "updateJob",
+    "getStatus",
+    "getEventRanges",
+    "updateEventRange",
+    "getKeyPair",
+    "updateEventRanges",
+    "getProxy",
+    "get_access_token",
+    "get_token_key",
+    "getCommands",
+    "ackCommands",
+    "checkJobStatus",
+    "checkEventsAvailability",
+    "updateJobsInBulk",
+    "getResourceTypes",
+    "updateWorkerPilotStatus",
+    "get_max_worker_id",
+    "get_events_status",
+]
+
+# methods from pandaserver.userinterface.UserIF
+allowed_methods += [
+    "submitJobs",
+    "getJobStatus",
+    "killJobs",
+    "reassignJobs",
+    "getJobStatisticsPerSite",
+    "getFullJobStatus",
+    "getScriptOfflineRunning",
+    "setDebugMode",
+    "insertSandboxFileInfo",
+    "checkSandboxFile",
+    "insertTaskParams",
+    "killTask",
+    "finishTask",
+    "getJediTasksInTimeRange",
+    "getJediTaskDetails",
+    "retryTask",
+    "changeTaskPriority",
+    "reassignTask",
+    "changeTaskAttributePanda",
+    "pauseTask",
+    "resumeTask",
+    "increaseAttemptNrPanda",
+    "killUnfinishedJobs",
+    "changeTaskSplitRulePanda",
+    "changeTaskModTimePanda",
+    "avalancheTask",
+    "getPandaIDsWithTaskID",
+    "reactivateTask",
+    "getTaskStatus",
+    "reassignShare",
+    "getTaskParamsMap",
+    "updateWorkers",
+    "harvesterIsAlive",
+    "reportWorkerStats_jobtype",
+    "getWorkerStats",
+    "addHarvesterDialogs",
+    "setNumSlotsForWP",
+    "reloadInput",
+    "enableJumboJobs",
+    "updateServiceMetrics",
+    "getUserJobMetadata",
+    "getJumboJobDatasets",
+    "sweepPQ",
+    "get_job_statistics_per_site_label_resource",
+    "relay_idds_command",
+    "send_command_to_job",
+    "execute_idds_workflow_command",
+    "set_user_secret",
+    "get_user_secrets",
+    "get_ban_users",
+    "get_files_in_datasets",
+    "release_task",
+]

pandaserver/srvcore/oidc_utils.py

@@ -0,0 +1,152 @@
+import base64
+import datetime
+from threading import Lock
+
+import jwt
+import requests
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicNumbers
+from jwt.exceptions import InvalidTokenError
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+
+def decode_value(val):
+    if isinstance(val, str):
+        val = val.encode()
+    decoded = base64.urlsafe_b64decode(val + b"==")
+    return int.from_bytes(decoded, "big")
+
+
+def rsa_pem_from_jwk(jwk):
+    public_num = RSAPublicNumbers(n=decode_value(jwk["n"]), e=decode_value(jwk["e"]))
+    public_key = public_num.public_key(default_backend())
+    pem = public_key.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+    return pem
+
+
+def get_jwk(kid, jwks):
+    for jwk in jwks.get("keys", []):
+        if jwk.get("kid") == kid:
+            return jwk
+    raise InvalidTokenError("JWK not found for kid={0}".format(kid, str(jwks)))
+
+
+# token decoder
+class TokenDecoder:
+    # constructor
+    def __init__(self, refresh_interval=10):
+        self.lock = Lock()
+        self.data = {}
+        self.refresh_interval = refresh_interval
+
+    # get cached data
+    def get_data(self, url, log_stream):
+        try:
+            with self.lock:
+                if url not in self.data or naive_utcnow() - self.data[url]["last_update"] > datetime.timedelta(minutes=self.refresh_interval):
+                    log_stream.debug(f"to refresh {url}")
+                    tmp_data = requests.get(url).json()
+                    log_stream.debug("refreshed")
+                    self.data[url] = {
+                        "data": tmp_data,
+                        "last_update": naive_utcnow(),
+                    }
+                return self.data[url]["data"]
+        except Exception as e:
+            log_stream.error(f"failed to refresh with {str(e)}")
+            raise
+
+    # decode and verify JWT token
+    def deserialize_token(self, token, auth_config, vo, log_stream, legacy_token_issuers):
+        try:
+            # check audience
+            unverified = jwt.decode(token, verify=False, options={"verify_signature": False})
+            conf_key = None
+            audience = None
+            if "aud" in unverified:
+                audience = unverified["aud"]
+                if audience in auth_config:
+                    conf_key = audience
+            if not conf_key:
+                # use sub as config key for access token
+                conf_key = unverified["sub"]
+            discovery_endpoint = auth_config[conf_key]["oidc_config_url"]
+            # decode headers
+            headers = jwt.get_unverified_header(token)
+            # get key id
+            if headers is None or "kid" not in headers:
+                raise jwt.exceptions.InvalidTokenError("cannot extract kid from headers")
+            kid = headers["kid"]
+            # retrieve OIDC configuration and JWK set
+            oidc_config = self.get_data(discovery_endpoint, log_stream)
+            jwks = self.get_data(oidc_config["jwks_uri"], log_stream)
+            # get JWK and public key
+            jwk = get_jwk(kid, jwks)
+            public_key = rsa_pem_from_jwk(jwk)
+            # decode token only with RS256
+            if unverified["iss"] and unverified["iss"] != oidc_config["issuer"] and oidc_config["issuer"].startswith(unverified["iss"]):
+                # iss is missing the last '/' in access tokens
+                issuer = unverified["iss"]
+            else:
+                issuer = oidc_config["issuer"]
+            if legacy_token_issuers:
+                issuers = list(dict.fromkeys([issuer] + legacy_token_issuers))
+            else:
+                issuers = [issuer]
+            decoded = None
+            err_msg = None
+            for tmp_issuer in issuers:
+                try:
+                    decoded = jwt.decode(
+                        token,
+                        public_key,
+                        verify=True,
+                        algorithms="RS256",
+                        audience=audience,
+                        issuer=tmp_issuer,
+                    )
+                    break
+                except jwt.exceptions.InvalidIssuerError as e:
+                    err_msg = str(e)
+            if not decoded:
+                raise jwt.exceptions.InvalidTokenError(f"failed to decode: {err_msg}")
+            if vo is not None:
+                decoded["vo"] = vo
+            else:
+                decoded["vo"] = auth_config[conf_key]["vo"]
+            return decoded
+        except Exception:
+            raise
+
+
+# get an access token with client_credentials flow
+def get_access_token(token_endpoint: str, client_id: str, client_secret: str, scope: str = None, timeout: int = 180) -> tuple[bool, str]:
+    """
+    Get an access token with client_credentials flow
+
+    :param token_endpoint: URL for token request
+    :param client_id: client ID
+    :param client_secret: client secret
+    :param scope: space separated string of scopes
+    :param timeout: timeout in seconds
+
+    :return: (True, access_token) or (False, error_str)
+    """
+    try:
+        token_request = {
+            "grant_type": "client_credentials",
+            "client_id": client_id,
+            "client_secret": client_secret,
+        }
+        if scope:
+            token_request["scope"] = scope
+        token_response = requests.post(token_endpoint, data=token_request, timeout=timeout)
+        token_response.raise_for_status()
+        return True, token_response.json()["access_token"]
+    except Exception as e:
+        error_str = f"failed to get access token with {str(e)}"
+        return False, error_str

pandaserver/srvcore/panda_request.py

@@ -0,0 +1,196 @@
+from pandaserver.config import panda_config
+from pandaserver.srvcore.CoreUtils import CacheDict
+
+if panda_config.token_authType is None:
+    pass
+elif panda_config.token_authType == "scitokens":
+    import scitokens
+else:
+    from pandaserver.srvcore.oidc_utils import TokenDecoder
+
+    token_decoder = TokenDecoder()
+
+import traceback
+
+import jwt
+
+cache_dict = CacheDict()
+
+
+# decode token
+def decode_token(serialized_token, env, tmp_log):
+    authenticated = False
+    message_str = None
+    subprocess_env = {}
+    try:
+        vo = None
+        role = None
+        if panda_config.token_authType == "oidc":
+            if "HTTP_ORIGIN" in env:
+                # two formats, vo:role and vo.role
+                vo = env["HTTP_ORIGIN"]
+                # only vo.role for auth filename which is a key of auth_vo_dict
+            else:
+                vo = None
+            # only vo.role for auth filename which is a key of auth_vo_dict
+            vo_role = vo.replace(":", ".")
+            token = token_decoder.deserialize_token(serialized_token, panda_config.auth_config, vo, tmp_log, panda_config.legacy_token_issuers)
+            # extract role
+            if "vo" in token:
+                vo_raw = token["vo"]  # Original input value of vo
+                tmp_log.debug(f"Raw VO from token: {vo_raw}")
+
+                if vo_raw.startswith("vo."):
+                    # Handle vo names starting with "vo."
+                    if ":" in vo_raw:
+                        vo, role = vo_raw.split(":", 1)  # Split by ':' into vo and role
+                    else:
+                        vo, role = vo_raw, None  # No role present
+                else:
+                    # Handle cases without "vo." prefix
+                    if ":" in vo_raw:
+                        vo, role = vo_raw.split(":", 1)
+                    elif "." in vo_raw:
+                        parts = vo_raw.rsplit(".", 1)  # Split only on the last '.'
+                        vo, role = parts[0], parts[1]
+                    else:
+                        vo, role = vo_raw, None  # Single part, no role
+
+            tmp_log.debug(f"Parsed VO: {vo}, role: {role}")
+
+            # check vo
+            if vo not in panda_config.auth_policies:
+                tmp_log.error(f"VO '{vo}' not found in auth_policies: {list(panda_config.auth_policies.keys())}")
+                message_str = f"Unknown vo : {vo}"
+            else:
+                # groups claim name based on JWT profile
+                groups_claim_name = "groups"
+                if vo_role in panda_config.auth_vo_dict:
+                    jwt_profile = panda_config.auth_vo_dict[vo_role].get("jwt_profile")
+                    if jwt_profile == "wlcg":
+                        groups_claim_name = "wlcg.groups"
+                # robot
+                if vo_role in panda_config.auth_vo_dict and "robot_ids" in panda_config.auth_vo_dict[vo_role]:
+                    robot_ids = panda_config.auth_vo_dict[vo_role].get("robot_ids")
+                    if isinstance(robot_ids, str):
+                        robot_ids = robot_ids.split(",")
+                    if not robot_ids:
+                        robot_ids = []
+                    robot_ids = [i for i in robot_ids if i]
+                    if token["sub"] in robot_ids:
+                        if groups_claim_name not in token:
+                            if role:
+                                token[groups_claim_name] = [f"{vo}/{role}"]
+                            else:
+                                token[groups_claim_name] = [f"{vo}"]
+                        if "name" not in token:
+                            token["name"] = f"robot {role}"
+                # check role
+                if role:
+                    if f"{vo}/{role}" not in token[groups_claim_name] and f"/{vo}/{role}" not in token[groups_claim_name]:
+                        message_str = f"Not a member of the {vo}/{role} group"
+                    else:
+                        subprocess_env["PANDA_OIDC_VO"] = vo
+                        subprocess_env["PANDA_OIDC_GROUP"] = role
+                        subprocess_env["PANDA_OIDC_ROLE"] = role
+                        authenticated = True
+                else:
+                    for member_string, member_info in panda_config.auth_policies[vo]:
+                        if member_string in token[groups_claim_name] or f"/{member_string}" in token[groups_claim_name]:
+                            subprocess_env["PANDA_OIDC_VO"] = vo
+                            subprocess_env["PANDA_OIDC_GROUP"] = member_info["group"]
+                            subprocess_env["PANDA_OIDC_ROLE"] = member_info["role"]
+                            authenticated = True
+                            break
+                    if not authenticated:
+                        message_str = f"Not a member of the {vo} group"
+        else:
+            token = scitokens.SciToken.deserialize(serialized_token, audience=panda_config.token_audience)
+
+        # check issuer
+        if "iss" not in token:
+            message_str = "Issuer is undefined in the token"
+        else:
+            if panda_config.token_authType == "scitokens":
+                items = token.claims()
+            else:
+                items = token.items()
+            for c, v in items:
+                subprocess_env[f"PANDA_OIDC_CLAIM_{str(c)}"] = str(v)
+            # use sub and scope as DN and FQAN
+            if "SSL_CLIENT_S_DN" not in env:
+                if "name" in token:
+                    subprocess_env["SSL_CLIENT_S_DN"] = " ".join([t[:1].upper() + t[1:].lower() for t in str(token["name"]).split()])
+                    if "preferred_username" in token:
+                        subprocess_env["SSL_CLIENT_S_DN"] += f"/CN=nickname:{token['preferred_username']}"
+                else:
+                    subprocess_env["SSL_CLIENT_S_DN"] = str(token["sub"])
+                i = 0
+                for scope in token.get("scope", "").split():
+                    if scope.startswith("role:"):
+                        subprocess_env[f"GRST_CRED_AUTH_TOKEN_{i}"] = "VOMS " + str(scope.split(":")[-1])
+                        i += 1
+                if role:
+                    subprocess_env[f"GRST_CRED_AUTH_TOKEN_{i}"] = f"VOMS /{vo}/Role={role}"
+                    i += 1
+            else:
+                # protection against cached decisions that miss x509-related variables due to token+x509 access
+                subprocess_env["SSL_CLIENT_S_DN"] = env["SSL_CLIENT_S_DN"]
+                for key in env:
+                    if key.startswith("GRST_CRED_") or key.startswith("GRST_CONN_"):
+                        subprocess_env[key] = env[key]
+
+    except jwt.exceptions.InvalidTokenError as e:
+        message_str = f"Invalid token. {str(e)}"
+    except Exception as e:
+        message_str = f"Token decode failure. {str(e)} {traceback.format_exc()}"
+    return {"authenticated": authenticated, "message": message_str, "subprocess_env": subprocess_env}
+
+
+# PanDA request object
+class PandaRequest:
+    def __init__(self, env, tmp_log):
+        # environment
+        self.subprocess_env = env
+        # header
+        self.headers_in = {}
+        # authentication
+        self.authenticated = True
+        # message
+        self.message = None
+
+        # content-length
+        if "CONTENT_LENGTH" in self.subprocess_env:
+            self.headers_in["content-length"] = self.subprocess_env["CONTENT_LENGTH"]
+
+        # tokens
+        try:
+            if panda_config.token_authType in ["scitokens", "oidc"] and "HTTP_AUTHORIZATION" in env:
+                serialized_token = env["HTTP_AUTHORIZATION"].split()[1]
+                decision_key = f"""{serialized_token} : {env.get("HTTP_ORIGIN", None)} : {env.get("SSL_CLIENT_S_DN", None)}"""
+                cached_decision = cache_dict.get(decision_key, tmp_log, decode_token, serialized_token, env, tmp_log)
+                self.authenticated = cached_decision["authenticated"]
+                self.message = cached_decision["message"]
+                if self.message:
+                    tmp_log.error(f"""{self.message} - Origin: {env.get("HTTP_ORIGIN", None)}, Token: {env["HTTP_AUTHORIZATION"]}""")
+                self.subprocess_env.update(cached_decision["subprocess_env"])
+        except Exception as e:
+            self.message = f"Failed to instantiate reqeust object. {str(e)}"
+            tmp_log.error(
+                f"""{self.message} - Origin: {env.get("HTTP_ORIGIN", None)}, Token: {env.get("HTTP_AUTHORIZATION", None)}\n{traceback.format_exc()}"""
+            )
+
+    # get remote host
+    def get_remote_host(self):
+        if "REMOTE_HOST" in self.subprocess_env:
+            return self.subprocess_env["REMOTE_HOST"]
+        return ""
+
+    # accept json
+    def acceptJson(self):
+        try:
+            if "HTTP_ACCEPT" in self.subprocess_env:
+                return "application/json" in self.subprocess_env["HTTP_ACCEPT"]
+        except Exception:
+            pass
+        return False

pandaserver/srvcore/srv_msg_utils.py

@@ -0,0 +1,42 @@
+import datetime
+import json
+
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+
+# make message
+def make_message(msg_type, **kwargs):
+    msg_dict = {"msg_type": msg_type}
+    msg_dict.update(kwargs)
+    msg_dict["timestamp"] = int(naive_utcnow().timestamp())
+    return json.dumps(msg_dict)
+
+
+# send a job message
+def send_job_message(msg_queue, msg_topic, task_id, job_id):
+    # make message
+    msg = make_message("get_job", taskid=task_id, jobid=job_id)
+    # use job ID for selector
+    headers = {"type": job_id}
+    # send message to topic first
+    msg_topic.send(msg)
+    # send the same message to queue
+    msg_queue.send(msg, headers=headers)
+
+
+# delete a job message
+def delete_job_message(msg_queue, job_id, time_out=10):
+    # job ID for selector
+    headers = {"selector": "type='{0}' OR JMSType='{0}'".format(job_id)}
+    # subscribe to remove job messages
+    msg_queue.add_remover(headers, time_out)
+    # delete old removers
+    msg_queue.purge_removers()
+
+
+# send a task message
+def send_task_message(msg_topic, command_str, task_id):
+    # make message
+    msg = make_message(f"{command_str}_task", taskid=task_id)
+    # send message to topic
+    msg_topic.send(msg)