panda-server-source 0.8.0__py3-none-any.whl

pandaserver/brokerage/broker.py

@@ -0,0 +1,272 @@
+import datetime
+import functools
+import time
+import traceback
+import uuid
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+from pandaserver.config import panda_config
+from pandaserver.dataservice import DataServiceUtils
+
+_log = PandaLogger().getLogger("broker")
+
+# all known sites
+_allSites = []
+
+# processingType to skip brokerage
+skipBrokerageProTypes = ["prod_test"]
+
+
+# comparison function for sort
+def _compFunc(job_a, job_b):
+    # This comparison helps in sorting the jobs based on the order of their computing sites in the _allSites list.
+    # append site if not in list
+    if job_a.computingSite not in _allSites:
+        _allSites.append(job_a.computingSite)
+    if job_b.computingSite not in _allSites:
+        _allSites.append(job_b.computingSite)
+
+    # compare site indices
+    index_a = _allSites.index(job_a.computingSite)
+    index_b = _allSites.index(job_b.computingSite)
+
+    if index_a > index_b:
+        return 1
+    elif index_a < index_b:
+        return -1
+    else:
+        return 0
+
+
+def schedule(jobs, siteMapper):
+    timestamp = naive_utcnow().isoformat("/")
+    tmp_log = LogWrapper(_log, f"start_ts={timestamp}")
+
+    try:
+        # no jobs
+        if len(jobs) == 0:
+            tmp_log.debug("finished : no jobs")
+            return
+
+        max_jobs = 20
+        max_files = 20
+
+        iJob = 0
+        fileList = []
+        chosen_panda_queue = None
+        prodDBlock = None
+        computingSite = None
+        dispatchDBlock = None
+        previousCloud = None
+        prevProType = None
+        prevSourceLabel = None
+        prevDirectAcc = None
+        prevIsJEDI = None
+        prevBrokerageSiteList = None
+
+        indexJob = 0
+
+        # sort jobs by siteID. Some jobs may already define computingSite
+        jobs = sorted(jobs, key=functools.cmp_to_key(_compFunc))
+
+        # loop over all jobs + terminator(None)
+        for job in jobs + [None]:
+            indexJob += 1
+
+            # ignore failed jobs
+            if job and job.jobStatus == "failed":
+                continue
+
+            # list of sites for special brokerage
+            specialBrokerageSiteList = []
+
+            # manually set site
+            if job and job.computingSite != "NULL" and job.prodSourceLabel in ("test", "managed") and specialBrokerageSiteList == []:
+                specialBrokerageSiteList = [job.computingSite]
+
+            overwriteSite = False
+
+            # check JEDI
+            isJEDI = False
+            if job and job.lockedby == "jedi":
+                isJEDI = True
+
+            # new bunch or terminator
+            if (
+                job is None
+                or len(fileList) >= max_files
+                or (dispatchDBlock is None and job.homepackage.startswith("AnalysisTransforms"))
+                or prodDBlock != job.prodDBlock
+                or job.computingSite != computingSite
+                or iJob > max_jobs
+                or previousCloud != job.getCloud()
+                or (prevProType in skipBrokerageProTypes and iJob > 0)
+                or prevDirectAcc != job.transferType
+                or prevProType != job.processingType
+                or prevBrokerageSiteList != specialBrokerageSiteList
+                or prevIsJEDI != isJEDI
+            ):
+                if indexJob > 1:
+                    tmp_log.debug("new bunch")
+                    tmp_log.debug(f"  iJob           {iJob}")
+                    tmp_log.debug(f"  cloud          {previousCloud}")
+                    tmp_log.debug(f"  sourceLabel    {prevSourceLabel}")
+                    tmp_log.debug(f"  prodDBlock     {prodDBlock}")
+                    tmp_log.debug(f"  computingSite  {computingSite}")
+                    tmp_log.debug(f"  processingType {prevProType}")
+                    tmp_log.debug(f"  transferType   {prevDirectAcc}")
+
+                if (iJob != 0 and chosen_panda_queue == "TOBEDONE") or prevBrokerageSiteList not in [None, []]:
+                    # load balancing
+                    minSites = {}
+                    if prevBrokerageSiteList:
+                        # special brokerage
+                        scanSiteList = prevBrokerageSiteList
+                    else:
+                        if siteMapper.checkCloud(previousCloud):
+                            # use cloud sites
+                            scanSiteList = siteMapper.getCloud(previousCloud)["sites"]
+
+                    # loop over all sites
+                    for site in scanSiteList:
+                        tmp_log.debug(f"calculate weight for site:{site}")
+                        # _allSites may contain NULL after sort()
+                        if site == "NULL":
+                            tmp_log.debug("site is NULL")
+                            continue
+
+                        winv = 1
+
+                        tmp_log.debug(f"Site:{site} 1/Weight:{winv}")
+
+                        # choose largest nMinSites weights
+                        minSites[site] = winv
+
+                    # choose site
+                    tmp_log.debug(f"Min Sites:{minSites}")
+                    if len(fileList) == 0 or prevIsJEDI is True:
+                        # choose min 1/weight
+                        minSite = list(minSites)[0]
+                        chosen_panda_queue = siteMapper.getSite(minSite)
+
+                    # set job spec
+                    tmp_log.debug(f"indexJob      : {indexJob}")
+
+                    for tmpJob in jobs[indexJob - iJob - 1 : indexJob - 1]:
+                        # set computingSite
+                        tmpJob.computingSite = chosen_panda_queue.sitename
+                        tmp_log.debug(f"PandaID:{tmpJob.PandaID} -> site:{tmpJob.computingSite}")
+
+                # terminate
+                if job is None:
+                    break
+                # reset iJob
+                iJob = 0
+                # reset file list
+                fileList = []
+                # create new dispDBlock
+                if job.prodDBlock != "NULL":
+                    # get datatype
+                    try:
+                        tmpDataType = job.prodDBlock.split(":")[-1].split(".")[-2]
+                    except Exception:
+                        # default
+                        tmpDataType = "GEN"
+                    if len(tmpDataType) > 20:
+                        # avoid too long name
+                        tmpDataType = "GEN"
+                    transferType = "transfer"
+                    if job.useInputPrestaging():
+                        transferType = "prestaging"
+                    dispatchDBlock = f"panda.{job.taskID}.{time.strftime('%m.%d')}.{tmpDataType}.{transferType}.{str(uuid.uuid4())}_dis{job.PandaID}"
+                    tmp_log.debug(f"New dispatchDBlock: {dispatchDBlock}")
+                prodDBlock = job.prodDBlock
+                # already define computingSite
+                if job.computingSite != "NULL":
+                    # instantiate KnownSite
+                    chosen_panda_queue = siteMapper.getSite(job.computingSite)
+
+                    # if site doesn't exist, use the default site
+                    if job.homepackage.startswith("AnalysisTransforms"):
+                        if chosen_panda_queue.sitename == panda_config.def_sitename:
+                            chosen_panda_queue = siteMapper.getSite(panda_config.def_queue)
+                            overwriteSite = True
+                else:
+                    # default for Analysis jobs
+                    if job.homepackage.startswith("AnalysisTransforms"):
+                        chosen_panda_queue = siteMapper.getSite(panda_config.def_queue)
+                        overwriteSite = True
+                    else:
+                        # set chosen_panda_queue
+                        chosen_panda_queue = "TOBEDONE"
+            # increment iJob
+            iJob += 1
+            # reserve computingSite and cloud
+            computingSite = job.computingSite
+            previousCloud = job.getCloud()
+            prevProType = job.processingType
+            prevSourceLabel = job.prodSourceLabel
+            prevDirectAcc = job.transferType
+            prevBrokerageSiteList = specialBrokerageSiteList
+            prevIsJEDI = isJEDI
+
+            # assign site
+            if chosen_panda_queue != "TOBEDONE":
+                job.computingSite = chosen_panda_queue.sitename
+                tmp_log.debug(f"PandaID:{job.PandaID} -> preset site:{chosen_panda_queue.sitename}")
+                # set cloud
+                if job.cloud in ["NULL", None, ""]:
+                    job.cloud = chosen_panda_queue.cloud
+
+            # set destinationSE
+            destSE = job.destinationSE
+            if siteMapper.checkCloud(job.getCloud()):
+                # use cloud dest for non-existing sites
+                if job.prodSourceLabel != "user" and job.destinationSE not in siteMapper.siteSpecList and job.destinationSE != "local":
+                    if DataServiceUtils.checkJobDestinationSE(job):
+                        destSE = DataServiceUtils.checkJobDestinationSE(job)
+                    job.destinationSE = destSE
+
+            if overwriteSite:
+                # overwrite SE for analysis jobs which set non-existing sites
+                destSE = job.computingSite
+                job.destinationSE = destSE
+
+            # set dispatchDBlock and destinationSE
+            first = True
+            for file in job.Files:
+                # Set dispatch data block for pre-stating jobs too
+                if file.type == "input" and file.dispatchDBlock == "NULL" and file.status not in ["ready", "missing", "cached"]:
+                    if first:
+                        first = False
+                        job.dispatchDBlock = dispatchDBlock
+                    file.dispatchDBlock = dispatchDBlock
+                    file.status = "pending"
+                    if file.lfn not in fileList:
+                        fileList.append(file.lfn)
+
+                # destinationSE
+                if file.type in ["output", "log"] and destSE != "":
+                    if job.prodSourceLabel == "user" and job.computingSite == file.destinationSE:
+                        pass
+                    elif job.prodSourceLabel == "user" and prevIsJEDI is True and file.destinationSE not in ["", "NULL"]:
+                        pass
+                    elif destSE == "local":
+                        pass
+                    elif DataServiceUtils.getDistributedDestination(file.destinationDBlockToken):
+                        pass
+                    else:
+                        file.destinationSE = destSE
+
+                # pre-assign GUID to log
+                if file.type == "log":
+                    # generate GUID
+                    file.GUID = str(uuid.uuid4())
+
+        tmp_log.debug("finished")
+
+    except Exception as e:
+        tmp_log.error(f"schedule : {str(e)} {traceback.format_exc()}")

pandaserver/config/config_utils.py

@@ -0,0 +1,15 @@
+import json
+import os
+
+
+# load configmap
+def load_config_map(section, target_dict):
+    if "PANDA_HOME" not in os.environ:
+        return
+    config_map_name = "panda_server_config.json"
+    config_map_path = os.path.join(os.environ["PANDA_HOME"], "etc/config_json", config_map_name)
+    if os.path.exists(config_map_path):
+        with open(config_map_path) as f:
+            tmp_data = json.load(f)
+            if section in tmp_data:
+                target_dict.update(tmp_data[section])

pandaserver/config/daemon_config.py

@@ -0,0 +1,29 @@
+import re
+import sys
+
+from pandacommon.liveconfigparser.LiveConfigParser import (
+    LiveConfigParser,
+    expand_values,
+)
+
+from . import config_utils
+
+# get ConfigParser
+tmpConf = LiveConfigParser()
+
+# read
+tmpConf.read("panda_server.cfg")
+
+# get daemon section
+tmpDict = getattr(tmpConf, "daemon", {})
+
+# read configmap
+config_utils.load_config_map("daemon", tmpDict)
+
+# expand all values
+tmpSelf = sys.modules[__name__]
+expand_values(tmpSelf, tmpDict)
+
+# default values
+if "enable" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["enable"] = False

pandaserver/config/mcp_config.py

@@ -0,0 +1,38 @@
+import re
+import sys
+
+from pandacommon.liveconfigparser.LiveConfigParser import (
+    LiveConfigParser,
+    expand_values,
+)
+
+from . import config_utils
+
+# get ConfigParser
+tmp_conf = LiveConfigParser()
+
+# read
+tmp_conf.read("panda_server.cfg")
+
+# get daemon section
+tmp_dict = getattr(tmp_conf, "mcp", {})
+
+# read configmap
+config_utils.load_config_map("mcp", tmp_dict)
+
+# expand all values
+tmp_self = sys.modules[__name__]
+expand_values(tmp_self, tmp_dict)
+
+# default values
+if "transport" not in tmp_self.__dict__:
+    tmp_self.__dict__["transport"] = "streamable-http"
+
+if "endpoint_list_file" not in tmp_self.__dict__:
+    tmp_self.__dict__["endpoint_list_file"] = "/opt/panda/etc/panda/panda_mcp_endpoints.json"
+
+if "ssl_keyfile" not in tmp_self.__dict__:
+    tmp_self.__dict__["ssl_keyfile"] = None
+
+if "ssl_certfile" not in tmp_self.__dict__:
+    tmp_self.__dict__["ssl_certfile"] = None

pandaserver/config/panda_config.py

@@ -0,0 +1,233 @@
+import glob
+import json
+import os
+import re
+import socket
+import sys
+
+from pandacommon.liveconfigparser.LiveConfigParser import (
+    LiveConfigParser,
+    expand_values,
+)
+
+from . import config_utils
+
+# get ConfigParser
+tmpConf = LiveConfigParser()
+
+# read
+tmpConf.read("panda_server.cfg")
+
+# get server section
+tmpDict = tmpConf.server
+
+# read configmap
+config_utils.load_config_map("server", tmpDict)
+
+# expand all values
+tmpSelf = sys.modules[__name__]
+expand_values(tmpSelf, tmpDict)
+
+# set hostname
+if "pserverhost" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["pserverhost"] = socket.getfqdn()
+
+# set port for http
+if "pserverporthttp" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["pserverporthttp"] = 25080
+
+# set host for http
+if "pserverhosthttp" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["pserverhosthttp"] = tmpSelf.__dict__["pserverhost"]
+
+# disable http
+if "disableHTTP" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["disableHTTP"] = False
+
+# DB backend
+if "backend" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["backend"] = "oracle"
+if "dbport" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["dbport"] = 0
+if "dbtimeout" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["dbtimeout"] = 60
+
+# Directory for certs
+if "certdir" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["certdir"] = "/data/atlpan"
+
+# endpoint map file
+if "endpoint_mapfile" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["endpoint_mapfile"] = (
+        "/cvmfs/atlas.cern.ch/repo/sw/local/etc/cric_ddmendpoints.json" ",/cvmfs/atlas.cern.ch/repo/sw/local/etc/agis_ddmendpoints.json"
+    )
+
+# sandbox info
+if "record_sandbox_info" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["record_sandbox_info"] = True
+
+# secrets
+if "pilot_secrets" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["pilot_secrets"] = "pilot secrets"
+
+# schemas
+if "schemaPANDA" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["schemaPANDA"] = "ATLAS_PANDA"
+if "schemaPANDAARCH" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["schemaPANDAARCH"] = "ATLAS_PANDAARCH"
+if "schemaMETA" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["schemaMETA"] = "ATLAS_PANDAMETA"
+if "schemaJEDI" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["schemaJEDI"] = "ATLAS_PANDA"
+if "schemaDEFT" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["schemaDEFT"] = "ATLAS_DEFT"
+if "schemaGRISLI" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["schemaGRISLI"] = "ATLAS_GRISLI"
+if "schemaEI" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["schemaEI"] = "ATLAS_EVENTINDEX"
+
+# default site
+if "def_sitename" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["def_sitename"] = "BNL_ATLAS_1"
+if "def_queue" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["def_queue"] = "ANALY_BNL_ATLAS_1"
+if "def_nickname" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["def_nickname"] = "BNL_ATLAS_1-condor"
+if "def_ddm" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["def_ddm"] = "PANDA_UNDEFINED2"
+if "def_type" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["def_type"] = "production"
+if "def_status" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["def_status"] = "online"
+if "token_authType" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["token_authType"] = "oidc"
+if "auth_config" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["auth_config"] = "/opt/panda/etc/panda/auth/"
+if "token_audience" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["token_audience"] = "https://pandaserver.cern.ch"
+if "token_issuers" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["token_issuers"] = ""
+if "legacy_token_issuers" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["legacy_token_issuers"] = "https://atlas-auth.cern.ch/,https://atlas-auth.web.cern.ch/"
+if tmpSelf.__dict__["legacy_token_issuers"]:
+    tmpSelf.__dict__["legacy_token_issuers"] = tmpSelf.__dict__["legacy_token_issuers"].split(",")
+else:
+    tmpSelf.__dict__["legacy_token_issuers"] = None
+tmpSelf.__dict__["production_dns"] = [x for x in tmpSelf.__dict__.get("production_dns", "").split(",") if x]
+tmpSelf.__dict__["pilot_owners"] = [x for x in tmpSelf.__dict__.get("pilot_owners", "").split(",") if x]
+tmpSelf.__dict__["auth_policies"] = {}
+tmpSelf.__dict__["auth_vo_dict"] = {}
+try:
+    data_dict = {}
+    vo_data_dict = {}
+    policy_dict = {}
+    for name in glob.glob(os.path.join(tmpSelf.__dict__["auth_config"], "*_auth_config.json")):
+        with open(name) as f:
+            data = json.load(f)
+            data_dict[data["client_id"]] = data
+            if "secondary_ids" in data:
+                for tmp_id in data["secondary_ids"]:
+                    data_dict[tmp_id] = data
+        m = re.search("^(.+)_auth_config.json", os.path.basename(name))
+        if m:
+            # Extract the VO group from the filename
+            tmp_vo_group = m.group(1)
+
+            # Determine tmp_vo and tmp_group based on delimiters
+            if tmp_vo_group.startswith("vo.") and ":" in tmp_vo_group:
+                tmp_vo, tmp_group = tmp_vo_group.split(":", 1)
+            elif ":" in tmp_vo_group:
+                tmp_vo, tmp_group = tmp_vo_group.split(":", 1)
+            elif tmp_vo_group.startswith("vo."):
+                # If it starts with "vo." but has no ":", treat it as a single VO with "user" as the group
+                tmp_vo = tmp_vo_group
+                tmp_group = "user"
+            elif "." in tmp_vo_group:
+                tmp_vo, tmp_group = tmp_vo_group.split(".", 1)
+            else:
+                tmp_vo, tmp_group = tmp_vo_group, "user"
+
+            policy_dict.setdefault(tmp_vo, [])
+            policy_dict[tmp_vo].append([tmp_vo, {"group": tmp_group, "role": tmp_group}])
+            vo_data_dict[tmp_vo_group.replace(":", ".")] = data
+    tmpSelf.__dict__["auth_config"] = data_dict
+    tmpSelf.__dict__["auth_policies"] = policy_dict
+    tmpSelf.__dict__["auth_vo_dict"] = vo_data_dict
+except Exception:
+    tmpSelf.__dict__["auth_config"] = {}
+
+if "token_cache_config" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["token_cache_config"] = "/opt/panda/etc/panda/token_cache_config.json"
+
+# use cert in configurator
+if "configurator_use_cert" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["configurator_use_cert"] = True
+
+# adder serialization
+if "add_serialized" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["add_serialized"] = False
+
+# disable file aggregation
+if "disable_file_aggregation" not in tmpSelf.__dict__:
+    tmpSelf.__dict__["disable_file_aggregation"] = False
+
+# dict for plugins
+g_pluginMap = {}
+
+
+# parser for plugin setup
+def parsePluginConf(modConfigName):
+    global tmpSelf
+    global g_pluginMap
+    g_pluginMap.setdefault(modConfigName, {})
+    # parse plugin setup
+    try:
+        for configStr in getattr(tmpSelf, modConfigName).split(","):
+            configStr = configStr.strip()
+            items = configStr.split(":")
+            vos = items[0].split("|")
+            moduleName = items[1]
+            className = items[2]
+            if len(items) > 3:
+                group = items[3]
+            else:
+                group = None
+            for vo in vos:
+                # import
+                mod = __import__(moduleName)
+                for subModuleName in moduleName.split(".")[1:]:
+                    mod = getattr(mod, subModuleName)
+                # get class
+                cls = getattr(mod, className)
+                if group:
+                    vo_key = f"{vo}_{group}"
+                else:
+                    vo_key = vo
+                g_pluginMap[modConfigName][vo_key] = cls
+    except Exception:
+        pass
+
+
+# accessor for plugin
+def getPlugin(modConfigName, vo, group=None):
+    if modConfigName not in g_pluginMap:
+        return None
+    if group:
+        vo_group = f"{vo}_{group}"
+        if vo_group in g_pluginMap[modConfigName]:
+            # VO+group specified
+            return g_pluginMap[modConfigName][vo_group]
+    if vo in g_pluginMap[modConfigName]:
+        # VO specified
+        return g_pluginMap[modConfigName][vo]
+    if "any" in g_pluginMap[modConfigName]:
+        # catch all
+        return g_pluginMap[modConfigName]["any"]
+    # undefined
+    return None
+
+
+# plug-ins
+def setupPlugin():
+    parsePluginConf("adder_plugins")
+    parsePluginConf("setupper_plugins")

pandaserver/configurator/Carbon.py

@@ -0,0 +1,65 @@
+import datetime
+import threading
+import traceback
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandaserver.config import panda_config
+from pandaserver.configurator import aux
+
+_logger = PandaLogger().getLogger("carbon")
+
+
+class CarbonEmissions(threading.Thread):
+    """
+    Downloads the carbon information from the relevant sources
+    """
+
+    def __init__(self, taskBuffer):
+        threading.Thread.__init__(self)
+
+        self.bearer_token = None
+        if hasattr(panda_config, "CO2_BEARER_TOKEN"):
+            self.bearer_token = panda_config.CO2_BEARER_TOKEN
+
+        self.taskBuffer = taskBuffer
+
+    def download_region_emissions(self):
+        # Don't indent the query
+        query = """
+{"search_type": "query_then_fetch","ignore_unavailable": true,"index": ["monit_prod_green-it_raw_regionmetric*"]}
+{"query": {"range": {"metadata.timestamp": {"gte": "now-2h","lt": "now"}}}, "size": 100}}
+"""
+        try:
+            results = aux.query_grafana_proxy(query, self.bearer_token)
+            if not results:
+                _logger.error("download_region_emissions was not able to download data")
+                return None
+
+            status = results["responses"][0]["status"]
+            if status != 200:
+                _logger.error(f"download_region_emissions was not able to download data with status {status}")
+                return None
+
+            hits = results["responses"][0]["hits"]["hits"]  # That's how the json is structured...
+            results = []
+            for entry in hits:
+                simplified_entry = {
+                    "region": entry["_source"]["data"]["region"],
+                    # timestamps come in ms
+                    "timestamp": datetime.datetime.fromtimestamp(entry["_source"]["metadata"]["timestamp"] / 1000),
+                    "value": entry["_source"]["data"]["gCO2_perkWh"],
+                }
+                results.append(simplified_entry)
+            return results
+        except Exception:
+            _logger.error(f"download_region_emissions excepted with {traceback.format_exc()}")
+            return None
+
+    def run(self):
+        # download emissions and store them in the DB
+        results = self.download_region_emissions()
+        if results:
+            self.taskBuffer.carbon_write_region_emissions(results)
+
+            # aggregate the emissions for the grid
+            self.taskBuffer.carbon_aggregate_emissions()