codecarbon 2.4.2__tar.gz → 2.5.0__tar.gz

{codecarbon-2.4.2 → codecarbon-2.5.0}/.gitignore

@@ -110,6 +110,7 @@ venv.bak/
 
 # idea
 .idea/
+*.iml
 
 # Sphinx
 docs/_build/

{codecarbon-2.4.2 → codecarbon-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codecarbon
-Version: 2.4.2
+Version: 2.5.0
 Project-URL: Homepage, https://codecarbon.io/
 Project-URL: Repository, https://github.com/mlco2/codecarbon
 Project-URL: Dashboard, http://dashboard.codecarbon.io/

codecarbon-2.5.0/codecarbon/_version.py

@@ -0,0 +1 @@
+__version__ = "2.5.0"

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/core/api_client.py

@@ -52,6 +52,14 @@ class ApiClient:  # (AsyncClient)
         self.conf = conf
         if self.experiment_id is not None:
             self._create_run(self.experiment_id)
+        import warnings
+
+        # FIXME: remove this warning in the future, once the release is created
+        warnings.warn(
+            "Beta API will be reworked, and some features will be removed. If you have data persisted through the API, please be warned that it will be erased with the next API release",
+            DeprecationWarning,
+            stacklevel=2,
+        )
 
     def add_emission(self, carbon_emission: dict):
         assert self.experiment_id is not None

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/core/powermetrics.py

@@ -25,6 +25,14 @@ def is_powermetrics_available():
 
 
 def _has_powermetrics_sudo():
+    if shutil.which("sudo") is None:
+        logger.debug("sudo not available, we won't use Apple PowerMetrics.")
+        return False
+    if shutil.which("powermetrics") is None:
+        logger.info(
+            "Apple PowerMetrics not available. Please install it if you are using an Apple product."
+        )
+        return False
     process = subprocess.Popen(
         [
             "sudo",

codecarbon-2.5.0/codecarbon/data/cloud/impact.csv

@@ -0,0 +1,37 @@
+provider,providerName,offsetRatio,region,regionName,country_name,countryIsoCode,state,city,impact,source,comment
+gcp,Google Cloud Platform,100,asia-east1,,Taiwan,TWN,,Changhua County,453,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,asia-east2,,China,CHN,,Hong Kong,360,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,asia-northeast1,,Japan,JPN,,Tokyo,463,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,asia-northeast2,,Japan,JPN,,Osaka,383,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,asia-northeast3,,Korea,KOR,South Korea,Seoul,425,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,asia-south1,,India,IND,,Mumbai,555,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,asia-south2,,India,IND,,Delhi,632,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,asia-southeast1,,Singapore,SGP,,Jurong West,372,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,asia-southeast2,,Indonesia,IDN,,Jakarta,580,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,australia-southeast1,,Australia,AUS,,Sydney,538,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,australia-southeast2,,Australia,AUS,,Melbourne,490,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-central2,,Poland,POL,,Warsaw,738,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-north1,,Finland,FIN,,Hamina,112,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-southwest1,,Spain,ESP,,Madrid,160,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-west1,,Belgium,BEL,,St. Ghislain,123,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-west2,,United Kingdom,GBR,England,London,166,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-west3,,Germany,DEU,,Frankfurt,413,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-west4,,Netherlands,NLD,,Eemshaven,317,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-west6,,Switzerland,CHE,,Zurich,118,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-west8,,Italy,ITA,,Milan,323,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-west9,,France,FRA,,Paris,71,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,europe-west12,,Italy,ITA,,Turin,323,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,me-west1,,Israel,ISR,,Tel Aviv,476,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,northamerica-northeast1,,Canada,CAN,Quebec,Montreal,0,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,northamerica-northeast2,,Canada,CAN,,Toronto,36,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,southamerica-east1,,Brazil,BRA,,Sao Paulo,65,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,southamerica-west1,,Chili,CHL,,Santiago,165,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-central1,,USA,USA,Iowa,Council Bluffs,445,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-east1,,USA,USA,South Carolina,Moncks Corner,532,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-east4,,USA,USA,Northern Virginia,Ashburn,354,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-east5,,USA,USA,,Columbus,354,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-south1,,USA,USA,,Dallas,342,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-west1,,USA,USA,Oregon,The Dalles,67,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-west2,,USA,USA,California,Los Angeles,202,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-west3,,USA,USA,Utah,Salt Lake City,606,https://cloud.google.com/sustainability/region-carbon#data,
+gcp,Google Cloud Platform,100,us-west4,,USA,USA,Nevada,Las Vegas,396,https://cloud.google.com/sustainability/region-carbon#data,

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/emissions_tracker.py

@@ -4,7 +4,6 @@ OfflineEmissionsTracker and @track_emissions
 """
 
 import dataclasses
-import os
 import platform
 import time
 import uuid
@@ -32,6 +31,7 @@ from codecarbon.output import (
     EmissionsData,
     FileOutput,
     HTTPOutput,
+    LogfireOutput,
     LoggerOutput,
     PrometheusOutput,
 )
@@ -149,7 +149,9 @@ class BaseEmissionsTracker(ABC):
         save_to_logger: Optional[bool] = _sentinel,
         logging_logger: Optional[LoggerOutput] = _sentinel,
         save_to_prometheus: Optional[bool] = _sentinel,
+        save_to_logfire: Optional[bool] = _sentinel,
         prometheus_url: Optional[str] = _sentinel,
+        output_handlers: Optional[List[BaseOutput]] = _sentinel,
         gpu_ids: Optional[List] = _sentinel,
         emissions_endpoint: Optional[str] = _sentinel,
         experiment_id: Optional[str] = _sentinel,
@@ -187,6 +189,8 @@ class BaseEmissionsTracker(ABC):
                             or a Google Cloud logger.
         :param save_to_prometheus: Indicates if the emission artifacts should be
                             pushed to prometheus, defaults to False.
+        :param save_to_logfire: Indicates if the emission artifacts should be written
+                            to a logfire observability platform, defaults to False.
         :param prometheus_url: url of the prometheus server, defaults to `localhost:9091`.
         :param gpu_ids: User-specified known gpu ids to track.
                             Defaults to None, which means that all available gpus will be tracked.
@@ -234,7 +238,9 @@ class BaseEmissionsTracker(ABC):
         self._set_from_conf(save_to_logger, "save_to_logger", False, bool)
         self._set_from_conf(logging_logger, "logging_logger")
         self._set_from_conf(save_to_prometheus, "save_to_prometheus", False, bool)
+        self._set_from_conf(save_to_logfire, "save_to_logfire", False, bool)
         self._set_from_conf(prometheus_url, "prometheus_url", "localhost:9091")
+        self._set_from_conf(output_handlers, "output_handlers", [])
         self._set_from_conf(tracking_mode, "tracking_mode", "machine")
         self._set_from_conf(on_csv_write, "on_csv_write", "append")
         self._set_from_conf(logger_preamble, "logger_preamble", "")
@@ -257,8 +263,6 @@ class BaseEmissionsTracker(ABC):
         self._cpu_power: Power = Power.from_watts(watts=0)
         self._gpu_power: Power = Power.from_watts(watts=0)
         self._ram_power: Power = Power.from_watts(watts=0)
-        self._cc_api__out = None
-        self._cc_prometheus_out = None
         self._measure_occurrence: int = 0
         self._cloud = None
         self._previous_emissions = None
@@ -271,18 +275,19 @@ class BaseEmissionsTracker(ABC):
         self._tasks: Dict[str, Task] = {}
         self._active_task: Optional[str] = None
 
-        # If _gpu_ids is a string or a list of int, parse it to a list of ints
-        if isinstance(self._gpu_ids, str) or (
-            isinstance(self._gpu_ids, list)
-            and all(isinstance(gpu_id, int) for gpu_id in self._gpu_ids)
-        ):
-            self._gpu_ids: List[int] = parse_gpu_ids(self._gpu_ids)
-            self._conf["gpu_ids"] = self._gpu_ids
-            self._conf["gpu_count"] = len(self._gpu_ids)
-        else:
-            logger.warning(
-                "Invalid gpu_ids format. Expected a string or a list of ints."
-            )
+        if self._gpu_ids:
+            # If _gpu_ids is a string or a list of int, parse it to a list of ints
+            if isinstance(self._gpu_ids, str) or (
+                isinstance(self._gpu_ids, list)
+                and all(isinstance(gpu_id, int) for gpu_id in self._gpu_ids)
+            ):
+                self._gpu_ids: List[int] = parse_gpu_ids(self._gpu_ids)
+                self._conf["gpu_ids"] = self._gpu_ids
+                self._conf["gpu_count"] = len(self._gpu_ids)
+            else:
+                logger.warning(
+                    "Invalid gpu_ids format. Expected a string or a list of ints."
+                )
 
         logger.info("[setup] RAM Tracking...")
         ram = RAM(tracking_mode=self._tracking_mode)
@@ -399,38 +404,38 @@ class BaseEmissionsTracker(ABC):
         """
         Prepare the different output methods
         """
-        self.persistence_objs: List[BaseOutput] = list()
-
         if self._save_to_file:
-            self.persistence_objs.append(
+            self._output_handlers.append(
                 FileOutput(
-                    os.path.join(self._output_dir, self._output_file),
+                    self._output_file,
+                    self._output_dir,
                     self._on_csv_write,
                 )
             )
 
         if self._save_to_logger:
-            self.persistence_objs.append(self._logging_logger)
+            self._output_handlers.append(self._logging_logger)
 
         if self._emissions_endpoint:
-            self.persistence_objs.append(HTTPOutput(self._emissions_endpoint))
+            self._output_handlers.append(HTTPOutput(self._emissions_endpoint))
 
         if self._save_to_api:
-            self._cc_api__out = CodeCarbonAPIOutput(
+            cc_api__out = CodeCarbonAPIOutput(
                 endpoint_url=self._api_endpoint,
                 experiment_id=self._experiment_id,
                 api_key=api_key,
                 conf=self._conf,
             )
-            self.run_id = self._cc_api__out.run_id
-            self.persistence_objs.append(self._cc_api__out)
-
+            self.run_id = cc_api__out.run_id
+            self._output_handlers.append(cc_api__out)
         else:
             self.run_id = uuid.uuid4()
 
         if self._save_to_prometheus:
-            self._cc_prometheus_out = PrometheusOutput(self._prometheus_url)
-            self.persistence_objs.append(self._cc_prometheus_out)
+            self._output_handlers.append(PrometheusOutput(self._prometheus_url))
+
+        if self._save_to_logfire:
+            self._output_handlers.append(LogfireOutput())
 
     def service_shutdown(self, signum, frame):
         print("Caught signal %d" % signum)
@@ -476,7 +481,8 @@ class BaseEmissionsTracker(ABC):
         # Read initial energy for hardware
         for hardware in self._hardware:
             hardware.start()
-        _ = self._prepare_emissions_data(delta=True)
+        _ = self._prepare_emissions_data()
+        _ = self._compute_emissions_delta(_)
 
         self._tasks.update(
             {
@@ -496,13 +502,14 @@ class BaseEmissionsTracker(ABC):
         task_name = task_name if task_name else self._active_task
         self._measure_power_and_energy()
 
-        emissions_data = self._prepare_emissions_data(delta=True)
+        emissions_data = self._prepare_emissions_data()
+        emissions_data_delta = self._compute_emissions_delta(emissions_data)
 
         task_duration = Time.from_seconds(
             time.time() - self._tasks[task_name].start_time
         )
 
-        task_emission_data = emissions_data
+        task_emission_data = emissions_data_delta
         task_emission_data.duration = task_duration.seconds
         self._tasks[task_name].emissions_data = task_emission_data
         self._tasks[task_name].is_active = False
@@ -525,7 +532,11 @@ class BaseEmissionsTracker(ABC):
         self._measure_power_and_energy()
 
         emissions_data = self._prepare_emissions_data()
-        self._persist_data(emissions_data)
+        emissions_data_delta = self._compute_emissions_delta(emissions_data)
+
+        self._persist_data(
+            total_emissions=emissions_data, delta_emissions=emissions_data_delta
+        )
 
         return emissions_data.emissions
 
@@ -553,29 +564,34 @@ class BaseEmissionsTracker(ABC):
         self._measure_power_and_energy()
 
         emissions_data = self._prepare_emissions_data()
+        emissions_data_delta = self._compute_emissions_delta(emissions_data)
 
-        self._persist_data(emissions_data, experiment_name=self._experiment_name)
+        self._persist_data(
+            total_emissions=emissions_data,
+            delta_emissions=emissions_data_delta,
+            experiment_name=self._experiment_name,
+        )
 
         self.final_emissions_data = emissions_data
         self.final_emissions = emissions_data.emissions
         return emissions_data.emissions
 
-    def _persist_data(self, emissions_data, experiment_name=None):
-        for persistence in self.persistence_objs:
-            if isinstance(persistence, CodeCarbonAPIOutput):
-                emissions_data = self._prepare_emissions_data(delta=True)
-
-            persistence.out(emissions_data)
-            if isinstance(persistence, FileOutput):
-                if len(self._tasks) > 0:
-                    task_emissions_data = []
-                    for task in self._tasks:
-                        task_emissions_data.append(self._tasks[task].out())
-                    persistence.task_out(
-                        task_emissions_data, experiment_name, self._output_dir
-                    )
+    def _persist_data(
+        self,
+        total_emissions: EmissionsData,
+        delta_emissions: EmissionsData,
+        experiment_name=None,
+    ):
+        task_emissions_data = []
+        for task in self._tasks:
+            task_emissions_data.append(self._tasks[task].out())
+
+        for handler in self._output_handlers:
+            handler.out(total_emissions, delta_emissions)
+            if len(task_emissions_data) > 0:
+                handler.task_out(task_emissions_data, experiment_name)
 
-    def _prepare_emissions_data(self, delta=False) -> EmissionsData:
+    def _prepare_emissions_data(self) -> EmissionsData:
         """
         :delta: If 'True', return only the delta comsumption since the last call.
         """
@@ -635,21 +651,23 @@ class BaseEmissionsTracker(ABC):
             tracking_mode=self._conf.get("tracking_mode"),
             pue=self._pue,
         )
-        if delta:
-            if self._previous_emissions is None:
-                self._previous_emissions = total_emissions
-            else:
-                # Create a copy
-                delta_emissions = dataclasses.replace(total_emissions)
-                # Compute emissions rate from delta
-                delta_emissions.compute_delta_emission(self._previous_emissions)
-                # TODO : find a way to store _previous_emissions only when
-                # TODO : the API call succeeded
-                self._previous_emissions = total_emissions
-                total_emissions = delta_emissions
         logger.debug(total_emissions)
         return total_emissions
 
+    def _compute_emissions_delta(self, total_emissions: EmissionsData) -> EmissionsData:
+        delta_emissions: EmissionsData = total_emissions
+        if self._previous_emissions is None:
+            self._previous_emissions = total_emissions
+        else:
+            # Create a copy
+            delta_emissions = dataclasses.replace(total_emissions)
+            # Compute emissions rate from delta
+            delta_emissions.compute_delta_emission(self._previous_emissions)
+            # TODO : find a way to store _previous_emissions only when
+            # TODO : the API call succeeded
+            self._previous_emissions = total_emissions
+        return delta_emissions
+
     @abstractmethod
     def _get_geo_metadata(self) -> GeoMetadata:
         """
@@ -740,20 +758,21 @@ class BaseEmissionsTracker(ABC):
         self._do_measurements()
         self._last_measured_time = time.time()
         self._measure_occurrence += 1
+        # Special case: metrics and api calls are sent every `api_call_interval` measures
         if (
-            self._cc_api__out is not None or self._cc_prometheus_out is not None
-        ) and self._api_call_interval != -1:
-            if self._measure_occurrence >= self._api_call_interval:
-                emissions = self._prepare_emissions_data(delta=True)
-                logger.info(
-                    f"{emissions.emissions_rate * 1000:.6f} g.CO2eq/s mean an estimation of "
-                    + f"{emissions.emissions_rate * 3600 * 24 * 365:,} kg.CO2eq/year"
-                )
-                if self._cc_api__out:
-                    self._cc_api__out.out(emissions)
-                if self._cc_prometheus_out:
-                    self._cc_prometheus_out.out(emissions)
-                self._measure_occurrence = 0
+            self._api_call_interval != -1
+            and len(self._output_handlers) > 0
+            and self._measure_occurrence >= self._api_call_interval
+        ):
+            emissions = self._prepare_emissions_data()
+            emissions_delta = self._compute_emissions_delta(emissions)
+            logger.info(
+                f"{emissions_delta.emissions_rate * 1000:.6f} g.CO2eq/s mean an estimation of "
+                + f"{emissions_delta.emissions_rate * 3600 * 24 * 365:,} kg.CO2eq/year"
+            )
+            for handler in self._output_handlers:
+                handler.live_out(emissions, emissions_delta)
+            self._measure_occurrence = 0
         logger.debug(f"last_duration={last_duration}\n------------------------")
 
     def __enter__(self):
@@ -922,7 +941,9 @@ def track_emissions(
     save_to_api: Optional[bool] = _sentinel,
     save_to_logger: Optional[bool] = _sentinel,
     save_to_prometheus: Optional[bool] = _sentinel,
+    save_to_logfire: Optional[bool] = _sentinel,
     prometheus_url: Optional[str] = _sentinel,
+    output_handlers: Optional[List[BaseOutput]] = _sentinel,
     logging_logger: Optional[LoggerOutput] = _sentinel,
     offline: Optional[bool] = _sentinel,
     emissions_endpoint: Optional[str] = _sentinel,
@@ -956,6 +977,8 @@ def track_emissions(
                         to a dedicated logger, defaults to False.
     :param save_to_prometheus: Indicates if the emission artifacts should be
                             pushed to prometheus, defaults to False.
+    :param save_to_logfire: Indicates if the emission artifacts should be
+                            pushed to logfire, defaults to False.
     :param prometheus_url: url of the prometheus server, defaults to `localhost:9091`.
     :param logging_logger: LoggerOutput object encapsulating a logging.logger
                         or a Google Cloud logger.
@@ -1000,7 +1023,9 @@ def track_emissions(
                     save_to_file=save_to_file,
                     save_to_logger=save_to_logger,
                     save_to_prometheus=save_to_prometheus,
+                    save_to_logfire=save_to_logfire,
                     prometheus_url=prometheus_url,
+                    output_handlers=output_handlers,
                     logging_logger=logging_logger,
                     country_iso_code=country_iso_code,
                     region=region,
@@ -1021,7 +1046,9 @@ def track_emissions(
                     save_to_file=save_to_file,
                     save_to_logger=save_to_logger,
                     save_to_prometheus=save_to_prometheus,
+                    save_to_logfire=save_to_logfire,
                     prometheus_url=prometheus_url,
+                    output_handlers=output_handlers,
                     logging_logger=logging_logger,
                     gpu_ids=gpu_ids,
                     log_level=log_level,

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/external/geography.py

@@ -32,18 +32,22 @@ class CloudMetadata:
             return re.search(google_region_regex, zone).group(0)
 
         extract_region_for_provider: Dict[str, Callable] = {
-            "aws": lambda x: x["metadata"]["region"],
-            "azure": lambda x: x["metadata"]["compute"]["location"],
-            "gcp": lambda x: extract_gcp_region(x["metadata"]["zone"]),
+            "aws": lambda x: x["metadata"].get("region"),
+            "azure": lambda x: x["metadata"]["compute"].get("location"),
+            "gcp": lambda x: extract_gcp_region(x["metadata"].get("zone")),
         }
 
         cloud_metadata: Dict = get_env_cloud_details()
 
-        if cloud_metadata is None:
+        if cloud_metadata is None or cloud_metadata["metadata"] == {}:
             return cls(provider=None, region=None)
 
         provider: str = cloud_metadata["provider"].lower()
         region: str = extract_region_for_provider.get(provider)(cloud_metadata)
+        if region is None:
+            logger.warning(
+                f"Cloud provider '{provider}' detected, but unable to read region. Using country value instead."
+            )
         if provider in ["aws", "azure"]:
             logger.warning(
                 f"Cloud provider '{provider}' do not publish electricity carbon intensity. Using country value instead."

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/external/hardware.py

@@ -308,6 +308,9 @@ class RAM(BaseHardware):
 
     def _parse_scontrol(self, scontrol_str):
         mem_matches = re.findall(r"AllocTRES=.*?,mem=(\d+[A-Z])", scontrol_str)
+        if len(mem_matches) == 0:
+            # Try with TRES, see https://github.com/mlco2/codecarbon/issues/569#issuecomment-2167706145
+            mem_matches = re.findall(r"TRES=.*?,mem=(\d+[A-Z])", scontrol_str)
         if len(mem_matches) == 0:
             logger.warning(
                 "Could not find mem= after running `scontrol show job $SLURM_JOB_ID` "

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/output.py

@@ -21,8 +21,7 @@ from codecarbon.output_methods.logger import (  # noqa: F401
     GoogleCloudLoggerOutput,
     LoggerOutput,
 )
+from codecarbon.output_methods.metrics.logfire import LogfireOutput  # noqa: F401
 
 # output is sent to metrics
-from codecarbon.output_methods.metrics.prometheus.prometheus import (  # noqa: F401
-    PrometheusOutput,
-)
+from codecarbon.output_methods.metrics.prometheus import PrometheusOutput  # noqa: F401

codecarbon-2.5.0/codecarbon/output_methods/base_output.py

@@ -0,0 +1,24 @@
+from typing import List
+
+from codecarbon.output_methods.emissions_data import EmissionsData, TaskEmissionsData
+
+
+class BaseOutput:
+    """
+    An abstract class defining possible contracts for an output strategy, a strategy implementation can save emissions
+    data to a file, posting to Json Box, saving to a database, sending a Slack message etc.
+    Each method is responsible for a different part of the EmissionsData lifecycle:
+        - `out` is used by termination calls such as emissions_tracker.flush and emissions_tracker.stop
+        - `live_out` is used by live measurement events, e.g. the iterative update of prometheus metrics
+        - `task_out` is used by terminate calls such as emissions_tracker.flush and emissions_tracker.stop, but uses
+          emissions segregated by task
+    """
+
+    def out(self, total: EmissionsData, delta: EmissionsData):
+        pass
+
+    def live_out(self, total: EmissionsData, delta: EmissionsData):
+        pass
+
+    def task_out(self, data: List[TaskEmissionsData], experiment_name: str):
+        pass

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/output_methods/file.py

@@ -15,14 +15,21 @@ class FileOutput(BaseOutput):
     Saves experiment artifacts to a file
     """
 
-    def __init__(self, save_file_path: str, on_csv_write: str = "append"):
+    def __init__(
+        self, output_file_name: str, output_dir: str, on_csv_write: str = "append"
+    ):
         if on_csv_write not in {"append", "update"}:
             raise ValueError(
                 f"Unknown `on_csv_write` value: {on_csv_write}"
                 + " (should be one of 'append' or 'update'"
             )
+        self.output_file_name: str = output_file_name
+        self.output_dir: str = output_dir
         self.on_csv_write: str = on_csv_write
-        self.save_file_path: str = save_file_path
+        self.save_file_path = os.path.join(self.output_dir, self.output_file_name)
+        logger.info(
+            f"Saving emissions data to file {os.path.abspath(self.save_file_path)}"
+        )
 
     def has_valid_headers(self, data: EmissionsData):
         with open(self.save_file_path) as csv_file:
@@ -31,42 +38,42 @@ class FileOutput(BaseOutput):
             list_of_column_names = list(dict_from_csv.keys())
             return list(data.values.keys()) == list_of_column_names
 
-    def out(self, data: EmissionsData):
+    def out(self, total: EmissionsData, delta: EmissionsData):
         file_exists: bool = os.path.isfile(self.save_file_path)
-        if file_exists and not self.has_valid_headers(data):
+        if file_exists and not self.has_valid_headers(total):
             logger.info("Backing up old emission file")
             backup(self.save_file_path)
             file_exists = False
 
         if not file_exists:
-            df = pd.DataFrame(columns=data.values.keys())
-            df = pd.concat([df, pd.DataFrame.from_records([dict(data.values)])])
+            df = pd.DataFrame(columns=total.values.keys())
+            df = pd.concat([df, pd.DataFrame.from_records([dict(total.values)])])
         elif self.on_csv_write == "append":
             df = pd.read_csv(self.save_file_path)
-            df = pd.concat([df, pd.DataFrame.from_records([dict(data.values)])])
+            df = pd.concat([df, pd.DataFrame.from_records([dict(total.values)])])
         else:
             df = pd.read_csv(self.save_file_path)
-            df_run = df.loc[df.run_id == data.run_id]
+            df_run = df.loc[df.run_id == total.run_id]
             if len(df_run) < 1:
-                df = pd.concat([df, pd.DataFrame.from_records([dict(data.values)])])
+                df = pd.concat([df, pd.DataFrame.from_records([dict(total.values)])])
             elif len(df_run) > 1:
                 logger.warning(
                     f"CSV contains more than 1 ({len(df_run)})"
-                    + f" rows with current run ID ({data.run_id})."
+                    + f" rows with current run ID ({total.run_id})."
                     + "Appending instead of updating."
                 )
-                df = pd.concat([df, pd.DataFrame.from_records([dict(data.values)])])
+                df = pd.concat([df, pd.DataFrame.from_records([dict(total.values)])])
             else:
-                df.at[df.run_id == data.run_id, data.values.keys()] = (
-                    data.values.values()
+                df.at[df.run_id == total.run_id, total.values.keys()] = (
+                    total.values.values()
                 )
 
         df.to_csv(self.save_file_path, index=False)
 
-    def task_out(self, data: List[TaskEmissionsData], experiment_name: str, output_dir):
+    def task_out(self, data: List[TaskEmissionsData], experiment_name: str):
         run_id = data[0].run_id
         save_task_file_path = os.path.join(
-            output_dir, "emissions_" + experiment_name + "_" + run_id + ".csv"
+            self.output_dir, "emissions_" + experiment_name + "_" + run_id + ".csv"
         )
         df = pd.DataFrame(columns=data[0].values.keys())
         df = pd.concat(

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/output_methods/http.py

@@ -19,9 +19,9 @@ class HTTPOutput(BaseOutput):
     def __init__(self, endpoint_url: str):
         self.endpoint_url: str = endpoint_url
 
-    def out(self, data: EmissionsData):
+    def out(self, total: EmissionsData, delta: EmissionsData):
         try:
-            payload = dataclasses.asdict(data)
+            payload = dataclasses.asdict(total)
             payload["user"] = getpass.getuser()
             resp = requests.post(self.endpoint_url, json=payload, timeout=10)
             if resp.status_code != 201:
@@ -50,8 +50,8 @@ class CodeCarbonAPIOutput(BaseOutput):
         )
         self.run_id = self.api.run_id
 
-    def out(self, data: EmissionsData):
+    def out(self, total: EmissionsData, delta: EmissionsData):
         try:
-            self.api.add_emission(dataclasses.asdict(data))
+            self.api.add_emission(dataclasses.asdict(delta))
         except Exception as e:
             logger.error(e, exc_info=True)

{codecarbon-2.4.2 → codecarbon-2.5.0}/codecarbon/output_methods/logger.py

@@ -16,22 +16,28 @@ class LoggerOutput(BaseOutput):
         self.logger = logger
         self.logging_severity = severity
 
-    def out(self, data: EmissionsData):
+    def out(self, total: EmissionsData, delta: EmissionsData):
         try:
-            payload = dataclasses.asdict(data)
+            payload = dataclasses.asdict(total)
             self.logger.log(self.logging_severity, msg=json.dumps(payload))
         except Exception as e:
             logger.error(e, exc_info=True)
 
+    def live_out(self, total: EmissionsData, delta: EmissionsData):
+        self.out(total, delta)
+
 
 class GoogleCloudLoggerOutput(LoggerOutput):
     """
     Send emissions data to GCP Cloud Logging
     """
 
-    def out(self, data: EmissionsData):
+    def out(self, total: EmissionsData, delta: EmissionsData):
         try:
-            payload = dataclasses.asdict(data)
+            payload = dataclasses.asdict(total)
             self.logger.log_struct(payload, severity=self.logging_severity)
         except Exception as e:
             logger.error(e, exc_info=True)
+
+    def live_out(self, total: EmissionsData, delta: EmissionsData):
+        self.out(total, delta)

codecarbon-2.5.0/codecarbon/output_methods/metrics/logfire.py

@@ -0,0 +1,79 @@
+from codecarbon.external.logger import logger
+from codecarbon.output_methods.base_output import BaseOutput
+from codecarbon.output_methods.emissions_data import EmissionsData
+
+
+class LogfireOutput(BaseOutput):
+    """
+    Send emissions data to logfire
+    """
+
+    def __init__(self):
+        try:
+            from logfire import configure, metric_counter, metric_gauge
+
+            configure()
+        except ImportError:
+            logger.error(
+                "Logfire is not installed. Please install it using `pip install logfire`"
+            )
+            raise
+
+        # Counters
+        self.duration = metric_counter(
+            "codecarbon_duration", unit="(s)", description="Duration from last measure"
+        )
+        self.emissions = metric_counter(
+            "codecarbon_emissions",
+            unit="(kg)",
+            description="Emissions as CO₂-equivalents CO₂eq",
+        )
+        self.energy_consumed = metric_counter(
+            "codecarbon_energy_consumed",
+            unit="(kW)",
+            description="Sum of cpu_energy, gpu_energy and ram_energy",
+        )
+
+        # Gauges
+        self.emissions_rate = metric_gauge(
+            "codecarbon_emissions_rate",
+            unit="(Kg/s)",
+            description="Emissions divided per duration",
+        )
+        self.cpu_power = metric_gauge(
+            "codecarbon_cpu_power", unit="(W)", description="CPU power"
+        )
+        self.gpu_power = metric_gauge(
+            "codecarbon_gpu_power", unit="(W)", description="GPU power"
+        )
+        self.ram_power = metric_gauge(
+            "codecarbon_ram_power", unit="(W)", description="RAM power"
+        )
+        self.cpu_energy = metric_gauge(
+            "codecarbon_cpu_energy", unit="(kWh)", description="Energy used per CPU"
+        )
+        self.gpu_energy = metric_gauge(
+            "codecarbon_gpu_energy", unit="(kWh)", description="Energy used per GPU"
+        )
+        self.ram_energy = metric_gauge(
+            "codecarbon_ram_energy", unit="(kWh)", description="Energy used per RAM"
+        )
+
+    def out(self, total: EmissionsData, delta: EmissionsData):
+        try:
+            self.duration.add(delta.duration)
+            self.emissions.add(delta.emissions)
+            self.emissions_rate.set(delta.emissions_rate)
+            self.cpu_power.set(delta.cpu_power)
+            self.gpu_power.set(delta.gpu_power)
+            self.ram_power.set(delta.ram_power)
+            self.cpu_energy.set(delta.cpu_energy)
+            self.gpu_energy.set(delta.gpu_energy)
+            self.ram_energy.set(delta.ram_energy)
+            self.energy_consumed.add(delta.energy_consumed)
+            logger.debug("Data sent to logfire")
+        except Exception as e:
+            logger.error(e, exc_info=True)
+
+    def live_out(self, total: EmissionsData, delta: EmissionsData):
+        self.out(total, delta)

{codecarbon-2.4.2/codecarbon/output_methods/metrics/prometheus → codecarbon-2.5.0/codecarbon/output_methods/metrics}/prometheus.py

@@ -1,26 +1,76 @@
 import dataclasses
 import os
 
-from prometheus_client import push_to_gateway
+from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
 from prometheus_client.exposition import basic_auth_handler
 
 from codecarbon.external.logger import logger
 from codecarbon.output_methods.base_output import BaseOutput
 from codecarbon.output_methods.emissions_data import EmissionsData
-from codecarbon.output_methods.metrics.prometheus.metrics import (
-    cpu_energy_gauge,
-    cpu_power_gauge,
-    duration_gauge,
-    emissions_gauge,
-    emissions_rate_gauge,
-    energy_consumed_gauge,
-    gpu_energy_gauge,
-    gpu_power_gauge,
-    ram_energy_gauge,
-    ram_power_gauge,
-    registry,
+from codecarbon.output_methods.metrics.metric_docs import (
+    MetricDocumentation,
+    cpu_energy_doc,
+    cpu_power_doc,
+    duration_doc,
+    emissions_doc,
+    emissions_rate_doc,
+    energy_consumed_doc,
+    gpu_energy_doc,
+    gpu_power_doc,
+    ram_energy_doc,
+    ram_power_doc,
 )
 
+registry = CollectorRegistry()
+
+
+# TODO: add labelnames
+# timestamp: str
+# run_id: str
+# python_version: str
+# longitude: float
+# latitude: float
+# on_cloud: str = "N"
+
+# TODO: Set up the possible labels
+labelnames = [
+    "project_name",
+    "country_name",
+    "country_iso_code",
+    "region",
+    "cloud_provider",
+    "cloud_region",
+    "os",
+    "codecarbon_version",
+    "cpu_model",
+    "cpu_count",
+    "gpu_model",
+    "gpu_count",
+    "tracking_mode",
+    "ram_total_size",
+]
+
+
+def generate_gauge(metric_doc: MetricDocumentation):
+    return Gauge(
+        metric_doc.name,
+        metric_doc.description,
+        labelnames,
+        registry=registry,
+    )
+
+
+duration_gauge = generate_gauge(duration_doc)
+emissions_gauge = generate_gauge(emissions_doc)
+emissions_rate_gauge = generate_gauge(emissions_rate_doc)
+cpu_power_gauge = generate_gauge(cpu_power_doc)
+gpu_power_gauge = generate_gauge(gpu_power_doc)
+ram_power_gauge = generate_gauge(ram_power_doc)
+cpu_energy_gauge = generate_gauge(cpu_energy_doc)
+gpu_energy_gauge = generate_gauge(gpu_energy_doc)
+ram_energy_gauge = generate_gauge(ram_energy_doc)
+energy_consumed_gauge = generate_gauge(energy_consumed_doc)
+
 
 class PrometheusOutput(BaseOutput):
     """
@@ -30,12 +80,15 @@ class PrometheusOutput(BaseOutput):
     def __init__(self, prometheus_url: str):
         self.prometheus_url = prometheus_url
 
-    def out(self, data: EmissionsData):
+    def out(self, total: EmissionsData, delta: EmissionsData):
         try:
-            self.add_emission(dataclasses.asdict(data))
+            self.add_emission(dataclasses.asdict(delta))
         except Exception as e:
             logger.error(e, exc_info=True)
 
+    def live_out(self, total: EmissionsData, delta: EmissionsData):
+        self.out(total, delta)
+
     def _auth_handler(self, url, method, timeout, headers, data):
         username = os.getenv("PROMETHEUS_USERNAME")
         password = os.getenv("PROMETHEUS_PASSWORD")

codecarbon-2.4.2/codecarbon/_version.py

@@ -1 +0,0 @@
-__version__ = "2.4.2"

codecarbon-2.4.2/codecarbon/data/cloud/impact.csv

@@ -1,35 +0,0 @@
-provider,providerName,offsetRatio,region,regionName,country_name,countryIsoCode,state,city,impact,source,comment
-gcp,Google Cloud Platform,100,asia-east1,,Taiwan,TWN,,Changhua County,456,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,asia-east2,,China,CHN,,Hong Kong,360,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,asia-northeast1,,Japan,JPN,,Tokyo,464,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,asia-northeast2,,Japan,JPN,,Osaka,384,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,asia-northeast3,,Korea,KOR,South Korea,Seoul,425,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,asia-south1,,India,IND,,Mumbai,633,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,asia-south2,,India,IND,,Delhi,633,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,asia-southeast1,,Singapore,SGP,,Jurong West,372,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,asia-southeast2,,Indonesia,IDN,,Jakarta,580,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,australia-southeast1,,Australia,AUS,,Sydney,723,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,australia-southeast2,,Australia,AUS,,Melbourne,746,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-central2,,Poland,POL,,Warsaw,576,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-north1,,Finland,FIN,,Hamina,127,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-southwest1,,Spain,ESP,,Madrid,121,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-west1,,Belgium,BEL,,St. Ghislain,110,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-west2,,United Kingdom,GBR,England,London,172,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-west3,,Germany,DEU,,Frankfurt,269,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-west4,,Netherlands,NLD,,Eemshaven,282,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-west6,,Switzerland,CHE,,Zurich,86,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-west8,,Italy,ITA,,Milan,298,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,europe-west9,,France,FRA,,Paris,59,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,northamerica-northeast1,,Canada,CAN,Quebec,Montreal,142,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,northamerica-northeast2,,Canada,CAN,,Toronto,142,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,southamerica-east1,,Brazil,BRA,,Sao Paulo,129,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,southamerica-west1,,Chili,CHL,,Santiago,190,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-central1,,USA,USA,Iowa,Council Bluffs,394,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-east1,,USA,USA,South Carolina,Moncks Corner,434,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-east4,,USA,USA,Northern Virginia,Ashburn,309,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-east5,,USA,USA,,Columbus,309,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-south1,,USA,USA,,Dallas,296,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-west1,,USA,USA,Oregon,The Dalles,60,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-west2,,USA,USA,California,Los Angeles,190,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-west3,,USA,USA,Utah,Salt Lake City,448,https://cloud.google.com/sustainability/region-carbon?hl=fr,
-gcp,Google Cloud Platform,100,us-west4,,USA,USA,Nevada,Las Vegas,365,https://cloud.google.com/sustainability/region-carbon?hl=fr,

codecarbon-2.4.2/codecarbon/output_methods/base_output.py

@@ -1,15 +0,0 @@
-from abc import ABC, abstractmethod
-
-from codecarbon.output_methods.emissions_data import EmissionsData
-
-
-class BaseOutput(ABC):
-    """
-    An abstract class that requires children to inherit a single method,
-    `out` which is used for persisting data. This could be by saving it to a file,
-    posting to Json Box, saving to a database, sending a slack message etc.
-    """
-
-    @abstractmethod
-    def out(self, data: EmissionsData):
-        pass

codecarbon-2.4.2/codecarbon/output_methods/metrics/prometheus/metrics.py

@@ -1,65 +0,0 @@
-from prometheus_client import CollectorRegistry, Gauge
-
-from codecarbon.output_methods.metrics.metric_docs import (
-    MetricDocumentation,
-    cpu_energy_doc,
-    cpu_power_doc,
-    duration_doc,
-    emissions_doc,
-    emissions_rate_doc,
-    energy_consumed_doc,
-    gpu_energy_doc,
-    gpu_power_doc,
-    ram_energy_doc,
-    ram_power_doc,
-)
-
-registry = CollectorRegistry()
-
-
-# TODO: add labelnames
-# timestamp: str
-# run_id: str
-# python_version: str
-# longitude: float
-# latitude: float
-# on_cloud: str = "N"
-
-# TODO: Set up the possible labels
-labelnames = [
-    "project_name",
-    "country_name",
-    "country_iso_code",
-    "region",
-    "cloud_provider",
-    "cloud_region",
-    "os",
-    "codecarbon_version",
-    "cpu_model",
-    "cpu_count",
-    "gpu_model",
-    "gpu_count",
-    "tracking_mode",
-    "ram_total_size",
-]
-
-
-def generate_gauge(metric_doc: MetricDocumentation):
-    return Gauge(
-        metric_doc.name,
-        metric_doc.description,
-        labelnames,
-        registry=registry,
-    )
-
-
-duration_gauge = generate_gauge(duration_doc)
-emissions_gauge = generate_gauge(emissions_doc)
-emissions_rate_gauge = generate_gauge(emissions_rate_doc)
-cpu_power_gauge = generate_gauge(cpu_power_doc)
-gpu_power_gauge = generate_gauge(gpu_power_doc)
-ram_power_gauge = generate_gauge(ram_power_doc)
-cpu_energy_gauge = generate_gauge(cpu_energy_doc)
-gpu_energy_gauge = generate_gauge(gpu_energy_doc)
-ram_energy_gauge = generate_gauge(ram_energy_doc)
-energy_consumed_gauge = generate_gauge(energy_consumed_doc)