codecarbon 2.8.2__tar.gz → 3.0.0rc0__tar.gz

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/.gitignore

@@ -8,11 +8,9 @@ __pycache__/
 *intel_power_gadget_log.csv
 *powermetrics_log.txt
 !tests/test_data/mock*
-.codecarbon.config
 
 # C extensions
 *.so
-.codecarbon.config
 
 # Distribution / packaging
 .Python
@@ -132,4 +130,5 @@ tests/test_data/rapl/*
 *.cast
 
 # credentials
-credentials.json
+credentials*
+.codecarbon.config*

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codecarbon
-Version: 2.8.2
+Version: 3.0.0rc0
 Project-URL: Homepage, https://codecarbon.io/
 Project-URL: Repository, https://github.com/mlco2/codecarbon
 Project-URL: Dashboard, http://dashboard.codecarbon.io/
@@ -212,3 +212,9 @@ Here is a sample for BibTeX:
 # Contact 📝
 
 Maintainers are [@vict0rsch](https://github.com/vict0rsch) [@benoit-cty](https://github.com/benoit-cty) and [@SaboniAmine](https://github.com/saboniamine). Codecarbon is developed by volunteers from [**Mila**](http://mila.quebec) and the [**DataForGoodFR**](https://twitter.com/dataforgood_fr) community alongside donated professional time of engineers at [**Comet.ml**](https://comet.ml) and [**BCG GAMMA**](https://www.bcg.com/en-nl/beyond-consulting/bcg-gamma/default).
+
+## Star History
+
+Comparison of the number of stars accumulated by the different Python CO2 emissions projects:
+[![Star History Chart](https://api.star-history.com/svg?repos=mlco2/codecarbon,lfwa/carbontracker,sb-ai-lab/Eco2AI,fvaleye/tracarbon,Breakend/experiment-impact-tracker&type=Date)](https://star-history.com/#mlco2/codecarbon&lfwa/carbontracker&sb-ai-lab/Eco2AI&fvaleye/tracarbon&Breakend/experiment-impact-tracker&Date)
+

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/README.md

@@ -172,3 +172,9 @@ Here is a sample for BibTeX:
 # Contact 📝
 
 Maintainers are [@vict0rsch](https://github.com/vict0rsch) [@benoit-cty](https://github.com/benoit-cty) and [@SaboniAmine](https://github.com/saboniamine). Codecarbon is developed by volunteers from [**Mila**](http://mila.quebec) and the [**DataForGoodFR**](https://twitter.com/dataforgood_fr) community alongside donated professional time of engineers at [**Comet.ml**](https://comet.ml) and [**BCG GAMMA**](https://www.bcg.com/en-nl/beyond-consulting/bcg-gamma/default).
+
+## Star History
+
+Comparison of the number of stars accumulated by the different Python CO2 emissions projects:
+[![Star History Chart](https://api.star-history.com/svg?repos=mlco2/codecarbon,lfwa/carbontracker,sb-ai-lab/Eco2AI,fvaleye/tracarbon,Breakend/experiment-impact-tracker&type=Date)](https://star-history.com/#mlco2/codecarbon&lfwa/carbontracker&sb-ai-lab/Eco2AI&fvaleye/tracarbon&Breakend/experiment-impact-tracker&Date)
+

codecarbon-3.0.0rc0/codecarbon/_version.py

@@ -0,0 +1 @@
+__version__ = "3.0.0_rc0"

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/cli/main.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import time
 from pathlib import Path
 from typing import Optional
@@ -99,12 +100,14 @@ def show_config(path: Path = Path("./.codecarbon.config")) -> None:
         )
 
 
-fief = Fief(AUTH_SERVER_URL, AUTH_CLIENT_ID)
-fief_auth = FiefAuth(fief, "./credentials.json")
+def get_fief_auth():
+    fief = Fief(AUTH_SERVER_URL, AUTH_CLIENT_ID)
+    fief_auth = FiefAuth(fief, "./credentials.json")
+    return fief_auth
 
 
 def _get_access_token():
-    access_token_info = fief_auth.access_token_info()
+    access_token_info = get_fief_auth().access_token_info()
     access_token = access_token_info["access_token"]
     return access_token
 
@@ -124,7 +127,7 @@ def api_get():
 
 @codecarbon.command("login", short_help="Login to CodeCarbon")
 def login():
-    fief_auth.authorize()
+    get_fief_auth().authorize()
 
 
 def get_api_key(project_id: str):
@@ -327,7 +330,10 @@ def monitor(
     experiment_id = get_existing_local_exp_id()
     if api:
         if experiment_id is None:
-            print("ERROR: No experiment id, call 'codecarbon config' first.", err=True)
+            print(
+                "ERROR: No experiment id, call 'codecarbon config' first.",
+                file=sys.stderr,
+            )
     print("CodeCarbon is going in an infinite loop to monitor this machine.")
     with EmissionsTracker(
         measure_power_secs=measure_power_secs,

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/core/api_client.py

@@ -257,8 +257,8 @@ class ApiClient:  # (AsyncClient)
                 gpu_count=self.conf.get("gpu_count"),
                 gpu_model=self.conf.get("gpu_model"),
                 # Reduce precision for Privacy
-                longitude=round(self.conf.get("longitude"), 1),
-                latitude=round(self.conf.get("latitude"), 1),
+                longitude=round(self.conf.get("longitude", 0), 1),
+                latitude=round(self.conf.get("latitude", 0), 1),
                 region=self.conf.get("region"),
                 provider=self.conf.get("provider"),
                 ram_total_size=self.conf.get("ram_total_size"),

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/core/cpu.py

@@ -5,12 +5,14 @@ https://software.intel.com/content/www/us/en/develop/articles/intel-power-gadget
 """
 
 import os
+import re
 import shutil
 import subprocess
 import sys
 from typing import Dict, Optional, Tuple
 
 import pandas as pd
+import psutil
 from rapidfuzz import fuzz, process, utils
 
 from codecarbon.core.rapl import RAPLFile
@@ -58,6 +60,21 @@ def is_rapl_available() -> bool:
         return False
 
 
+def is_psutil_available():
+    try:
+        nice = psutil.cpu_times().nice
+        if nice > 0.1:
+            return True
+        else:
+            return False
+    except Exception as e:
+        logger.debug(
+            "Not using the psutil interface, an exception occurred while instantiating "
+            + f"psutil.cpu_percent : {e}",
+        )
+        return False
+
+
 class IntelPowerGadget:
     """
     A class to interface with Intel Power Gadget for monitoring CPU power consumption on Windows and (non-Apple Silicon) macOS.
@@ -294,7 +311,7 @@ class IntelRAPL:
                     logger.debug("We will read Intel RAPL files at %s", rapl_file)
                 except PermissionError as e:
                     raise PermissionError(
-                        "Unable to read Intel RAPL files for CPU power, we will use a constant for your CPU power."
+                        "PermissionError : Unable to read Intel RAPL files for CPU power, we will use a constant for your CPU power."
                         + " Please view https://github.com/mlco2/codecarbon/issues/244"
                         + " for workarounds : %s",
                         e,
@@ -332,8 +349,6 @@ class IntelRAPL:
         """
         Return CPU details without computing them.
         """
-        logger.debug("get_static_cpu_details %s", self._cpu_details)
-
         return self._cpu_details
 
     def start(self) -> None:
@@ -426,6 +441,18 @@ class TDP:
         start_cpu = model_raw.find(" CPU @ ")
         if start_cpu > 0:
             model_raw = model_raw[0:start_cpu]
+        model_raw = model_raw.replace(" CPU", "")
+        model_raw = re.sub(r" @\s*\d+\.\d+GHz", "", model_raw)
+        direct_match = process.extractOne(
+            model_raw,
+            cpu_df["Name"],
+            processor=lambda s: s.lower(),
+            scorer=fuzz.ratio,
+            score_cutoff=THRESHOLD_DIRECT,
+        )
+
+        if direct_match:
+            return direct_match[0]
         indirect_matches = process.extract(
             model_raw,
             cpu_df["Name"],

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/core/emissions.py

@@ -90,7 +90,7 @@ class Emissions:
         selected = df.loc[flags]
         if not len(selected):
             raise ValueError(
-                "Unable to find country name for "
+                "Unable to find country ISO Code for "
                 f"cloud_provider={cloud.provider}, "
                 f"cloud_region={cloud.region}"
             )
@@ -105,7 +105,7 @@ class Emissions:
         selected = df.loc[flags]
         if not len(selected):
             raise ValueError(
-                "Unable to find country name for "
+                "Unable to find State/City name for "
                 f"cloud_provider={cloud.provider}, "
                 f"cloud_region={cloud.region}"
             )

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/core/resource_tracker.py

@@ -4,7 +4,7 @@ from typing import List, Union
 from codecarbon.core import cpu, gpu, powermetrics
 from codecarbon.core.config import parse_gpu_ids
 from codecarbon.core.util import detect_cpu_model, is_linux_os, is_mac_os, is_windows_os
-from codecarbon.external.hardware import CPU, GPU, RAM, AppleSiliconChip
+from codecarbon.external.hardware import CPU, GPU, MODE_CPU_LOAD, RAM, AppleSiliconChip
 from codecarbon.external.logger import logger
 
 
@@ -23,6 +23,31 @@ class ResourceTracker:
 
     def set_CPU_tracking(self):
         logger.info("[setup] CPU Tracking...")
+        tdp = cpu.TDP()
+        if self.tracker._conf.get("force_mode_cpu_load", False) and tdp.tdp is not None:
+            if tdp.tdp is None:
+                logger.warning(
+                    "Force CPU load mode requested but TDP could not be calculated. Falling back to another mode."
+                )
+            elif cpu.is_psutil_available():
+                # Register a CPU with MODE_CPU_LOAD
+                power = tdp.tdp
+                model = tdp.model
+                hardware = CPU.from_utils(
+                    self.tracker._output_dir,
+                    MODE_CPU_LOAD,
+                    model,
+                    power,
+                    tracking_mode=self.tracker._tracking_mode,
+                )
+                self.cpu_tracker = MODE_CPU_LOAD
+                self.tracker._conf["cpu_model"] = hardware.get_model()
+                self.tracker._hardware.append(hardware)
+                return
+            else:
+                logger.warning(
+                    "Force CPU load mode requested but psutil is not available."
+                )
         if cpu.is_powergadget_available() and self.tracker._default_cpu_power is None:
             logger.info("Tracking Intel CPU via Power Gadget")
             self.cpu_tracker = "Power Gadget"
@@ -32,9 +57,15 @@ class ResourceTracker:
         elif cpu.is_rapl_available():
             logger.info("Tracking Intel CPU via RAPL interface")
             self.cpu_tracker = "RAPL"
-            hardware = CPU.from_utils(self.tracker._output_dir, "intel_rapl")
+            hardware = CPU.from_utils(
+                output_dir=self.tracker._output_dir, mode="intel_rapl"
+            )
             self.tracker._hardware.append(hardware)
             self.tracker._conf["cpu_model"] = hardware.get_model()
+            if "AMD Ryzen Threadripper" in self.tracker._conf["cpu_model"]:
+                logger.warning(
+                    "The RAPL energy and power reported is divided by 2 for all 'AMD Ryzen Threadripper' as it seems to give better results."
+                )
         # change code to check if powermetrics needs to be installed or just sudo setup
         elif (
             powermetrics.is_powermetrics_available()
@@ -72,7 +103,7 @@ class ResourceTracker:
             elif is_windows_os():
                 cpu_tracking_install_instructions = "Windows OS detected: Please install Intel Power Gadget to measure CPU"
             elif is_linux_os():
-                cpu_tracking_install_instructions = "Linux OS detected: Please ensure RAPL files exist at \\sys\\class\\powercap\\intel-rapl to measure CPU"
+                cpu_tracking_install_instructions = "Linux OS detected: Please ensure RAPL files exist at /sys/class/powercap/intel-rapl to measure CPU"
             logger.warning(
                 f"No CPU tracking mode found. Falling back on CPU constant mode. \n {cpu_tracking_install_instructions}\n"
             )
@@ -89,17 +120,46 @@ class ResourceTracker:
             logger.info(f"CPU Model on constant consumption mode: {model}")
             self.tracker._conf["cpu_model"] = model
             if tdp:
-                hardware = CPU.from_utils(
-                    self.tracker._output_dir, "constant", model, power
-                )
+                if cpu.is_psutil_available():
+                    logger.warning(
+                        "No CPU tracking mode found. Falling back on CPU load mode."
+                    )
+                    hardware = CPU.from_utils(
+                        self.tracker._output_dir,
+                        MODE_CPU_LOAD,
+                        model,
+                        power,
+                        tracking_mode=self.tracker._tracking_mode,
+                    )
+                    self.cpu_tracker = MODE_CPU_LOAD
+                else:
+                    logger.warning(
+                        "No CPU tracking mode found. Falling back on CPU constant mode."
+                    )
+                    hardware = CPU.from_utils(
+                        self.tracker._output_dir, "constant", model, power
+                    )
+                    self.cpu_tracker = "global constant"
                 self.tracker._hardware.append(hardware)
             else:
-                logger.warning(
-                    "Failed to match CPU TDP constant. "
-                    + "Falling back on a global constant."
-                )
-                self.cpu_tracker = "global constant"
-                hardware = CPU.from_utils(self.tracker._output_dir, "constant")
+                if cpu.is_psutil_available():
+                    logger.warning(
+                        "Failed to match CPU TDP constant. Falling back on CPU load mode."
+                    )
+                    hardware = CPU.from_utils(
+                        self.tracker._output_dir,
+                        MODE_CPU_LOAD,
+                        model,
+                        power,
+                        tracking_mode=self.tracker._tracking_mode,
+                    )
+                    self.cpu_tracker = MODE_CPU_LOAD
+                else:
+                    logger.warning(
+                        "Failed to match CPU TDP constant. Falling back on a global constant."
+                    )
+                    self.cpu_tracker = "global constant"
+                    hardware = CPU.from_utils(self.tracker._output_dir, "constant")
                 self.tracker._hardware.append(hardware)
 
     def set_GPU_tracking(self):
@@ -130,10 +190,15 @@ class ResourceTracker:
                 self.tracker._conf["gpu_count"] = len(
                     gpu_devices.devices.get_gpu_static_info()
                 )
+            self.gpu_tracker = "pynvml"
         else:
             logger.info("No GPU found.")
 
     def set_CPU_GPU_ram_tracking(self):
+        """
+        Set up CPU, GPU and RAM tracking based on the user's configuration.
+        param tracker: BaseEmissionsTracker object
+        """
         self.set_RAM_tracking()
         self.set_CPU_tracking()
         self.set_GPU_tracking()

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/core/units.py

@@ -145,3 +145,9 @@ class Power:
 
     def __mul__(self, factor: float) -> "Power":
         return Power(self.kW * factor)
+
+    def __truediv__(self, divisor: float) -> "Power":
+        return Power(self.kW / divisor)
+
+    def __floordiv__(self, divisor: float) -> "Power":
+        return Power(self.kW // divisor)

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/core/util.py

@@ -48,6 +48,8 @@ def resolve_path(path: Union[str, Path]) -> Path:
 def backup(file_path: Union[str, Path], ext: Optional[str] = ".bak") -> None:
     """
     Resolves the path to a path then backs it up, adding the extension provided.
+    Warning : this function will rename the file in place, it's the calling function that will write a new file at the original path.
+    This function will not overwrite existing backups but add a number.
 
     Args:
         file_path (Union[str, Path]): Path to a file to backup.
@@ -111,7 +113,7 @@ def count_cpus() -> int:
             "Error running `scontrol show job $SLURM_JOB_ID` "
             + "to count SLURM-available cpus. Using the machine's cpu count."
         )
-        return psutil.cpu_count()
+        return psutil.cpu_count(logical=True)
 
     num_cpus_matches = re.findall(r"NumCPUs=\d+", scontrol)
 
@@ -120,14 +122,14 @@ def count_cpus() -> int:
             "Could not find NumCPUs= after running `scontrol show job $SLURM_JOB_ID` "
             + "to count SLURM-available cpus. Using the machine's cpu count."
         )
-        return psutil.cpu_count()
+        return psutil.cpu_count(logical=True)
 
     if len(num_cpus_matches) > 1:
         logger.warning(
             "Unexpected output after running `scontrol show job $SLURM_JOB_ID` "
             + "to count SLURM-available cpus. Using the machine's cpu count."
         )
-        return psutil.cpu_count()
+        return psutil.cpu_count(logical=True)
 
     num_cpus = num_cpus_matches[0].replace("NumCPUs=", "")
     logger.debug(f"Detected {num_cpus} cpus available on SLURM.")

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/data/hardware/cpu_power.csv

@@ -563,6 +563,7 @@ AMD EPYC 7F32,180
 AMD EPYC 7F52,240
 AMD EPYC 7F72,240
 AMD EPYC 7H12,280
+AMD EPYC 8024P,90
 AMD EPYC Embedded 3251,50
 AMD FX-4100,95
 AMD FX-4120,95
@@ -2231,6 +2232,7 @@ Intel Core i7-1185G7E,28
 Intel Core i7-1185GRE,28
 Intel Core i7-1195G7,28
 Intel Core i7-1270P,64
+Intel Core i7-12700K,190
 Intel Core i7-1360P,28
 Intel Core i7-2600,95
 Intel Core i7-2600K,95
@@ -3188,6 +3190,7 @@ Intel Xeon E5-2618L v2,50
 Intel Xeon E5-2618L v4,75
 Intel Xeon E5-2620,95
 Intel Xeon E5-2620 v2,80
+Intel Xeon E5-2620 v3,85
 Intel Xeon E5-2620 v4,85
 Intel Xeon E5-2623 v4,85
 Intel Xeon E5-2628L,60
@@ -3751,6 +3754,7 @@ Intel Xeon Platinum 8376HL,205
 Intel Xeon Platinum 8380,270
 Intel Xeon Platinum 8380H,250
 Intel Xeon Platinum 8380HL,250
+Intel Xeon Platinum 8370C,205
 Intel Xeon Platinum 9221,250
 Intel Xeon Platinum 9222,250
 Intel Xeon Platinum 9242,350

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/emissions_tracker.py

@@ -64,6 +64,9 @@ class BaseEmissionsTracker(ABC):
     and `CarbonTracker.`
     """
 
+    _scheduler: Optional[PeriodicScheduler] = None
+    _scheduler_monitor_power: Optional[PeriodicScheduler] = None
+
     def _set_from_conf(
         self, var, name, default=None, return_type=None, prevent_setter=False
     ):
@@ -170,6 +173,7 @@ class BaseEmissionsTracker(ABC):
         logger_preamble: Optional[str] = _sentinel,
         default_cpu_power: Optional[int] = _sentinel,
         pue: Optional[int] = _sentinel,
+        force_mode_cpu_load: Optional[bool] = _sentinel,
         allow_multiple_runs: Optional[bool] = _sentinel,
     ):
         """
@@ -225,6 +229,7 @@ class BaseEmissionsTracker(ABC):
                                 messages. Defaults to "".
         :param default_cpu_power: cpu power to be used as default if the cpu is not known.
         :param pue: PUE (Power Usage Effectiveness) of the datacenter.
+        :param force_mode_cpu_load: Force the addition of a CPU in MODE_CPU_LOAD
         :param allow_multiple_runs: Allow multiple instances of codecarbon running in parallel. Defaults to False.
         """
 
@@ -274,6 +279,7 @@ class BaseEmissionsTracker(ABC):
         self._set_from_conf(logger_preamble, "logger_preamble", "")
         self._set_from_conf(default_cpu_power, "default_cpu_power")
         self._set_from_conf(pue, "pue", 1.0, float)
+        self._set_from_conf(force_mode_cpu_load, "force_mode_cpu_load", False)
         self._set_from_conf(
             experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1"
         )
@@ -330,6 +336,10 @@ class BaseEmissionsTracker(ABC):
             function=self._measure_power_and_energy,
             interval=self._measure_power_secs,
         )
+        self._scheduler_monitor_power = PeriodicScheduler(
+            function=self._monitor_power,
+            interval=1,
+        )
 
         self._data_source = DataSource()
 
@@ -417,6 +427,7 @@ class BaseEmissionsTracker(ABC):
             hardware.start()
 
         self._scheduler.start()
+        self._scheduler_monitor_power.start()
 
     def start_task(self, task_name=None) -> None:
         """
@@ -428,6 +439,9 @@ class BaseEmissionsTracker(ABC):
         if self._scheduler:
             self._scheduler.stop()
 
+        # Task background thread for measuring power
+        self._scheduler_monitor_power.start()
+
         if self._active_task:
             logger.info("A task is already under measure")
             return
@@ -457,6 +471,9 @@ class BaseEmissionsTracker(ABC):
         emissions.
         :return: None
         """
+        if self._scheduler_monitor_power:
+            self._scheduler_monitor_power.stop()
+
         task_name = task_name if task_name else self._active_task
         self._measure_power_and_energy()
 
@@ -524,6 +541,9 @@ class BaseEmissionsTracker(ABC):
         if self._scheduler:
             self._scheduler.stop()
             self._scheduler = None
+        if self._scheduler_monitor_power:
+            self._scheduler_monitor_power.stop()
+            self._scheduler_monitor_power = None
         else:
             logger.warning("Tracker already stopped !")
         for task_name in self._tasks:
@@ -652,6 +672,17 @@ class BaseEmissionsTracker(ABC):
         :return: Metadata containing cloud info
         """
 
+    def _monitor_power(self) -> None:
+        """
+        Monitor the power consumption of the hardware.
+        We do this for hardware that does not support energy monitoring.
+        So we could average the power consumption.
+        This method is called every 1 second. Even if we are in Task mode.
+        """
+        for hardware in self._hardware:
+            if isinstance(hardware, CPU):
+                hardware.monitor_power()
+
     def _do_measurements(self) -> None:
         for hardware in self._hardware:
             h_time = time.perf_counter()
@@ -668,8 +699,11 @@ class BaseEmissionsTracker(ABC):
                 self._total_cpu_energy += energy
                 self._cpu_power = power
                 logger.info(
-                    f"Energy consumed for all CPUs : {self._total_cpu_energy.kWh:.6f} kWh"
-                    + f". Total CPU Power : {self._cpu_power.W} W"
+                    f"Delta energy consumed for CPU with {hardware._mode} : {energy.kWh:.6f} kWh"
+                    + f", power : {self._cpu_power.W} W"
+                )
+                logger.info(
+                    f"Energy consumed for All CPU : {self._total_cpu_energy.kWh:.6f} kWh"
                 )
             elif isinstance(hardware, GPU):
                 self._total_gpu_energy += energy
@@ -704,8 +738,7 @@ class BaseEmissionsTracker(ABC):
                 logger.error(f"Unknown hardware type: {hardware} ({type(hardware)})")
             h_time = time.perf_counter() - h_time
             logger.debug(
-                f"{hardware.__class__.__name__} : {hardware.total_power().W:,.2f} "
-                + f"W during {last_duration:,.2f} s [measurement time: {h_time:,.4f}]"
+                f"Done measure for {hardware.__class__.__name__} - measurement time: {h_time:,.4f} s - last call {last_duration:,.2f} s"
             )
         logger.info(
             f"{self._total_energy.kWh:.6f} kWh of electricity used since the beginning."
@@ -717,7 +750,13 @@ class BaseEmissionsTracker(ABC):
         every `self._measure_power_secs` seconds.
         :return: None
         """
-        last_duration = time.perf_counter() - self._last_measured_time
+        try:
+            last_duration = time.perf_counter() - self._last_measured_time
+        except AttributeError as e:
+            logger.debug(
+                f"You need to start the tracker first before measuring. Or maybe you do multiple run at the same time ? Error: {e}"
+            )
+            raise e
 
         warning_duration = self._measure_power_secs * 3
         if last_duration > warning_duration:

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/external/hardware.py

@@ -2,6 +2,7 @@
 Encapsulates external dependencies to retrieve hardware metadata
 """
 
+import math
 import re
 import subprocess
 from abc import ABC, abstractmethod
@@ -14,7 +15,7 @@ from codecarbon.core.cpu import IntelPowerGadget, IntelRAPL
 from codecarbon.core.gpu import AllGPUDevices
 from codecarbon.core.powermetrics import ApplePowermetrics
 from codecarbon.core.units import Energy, Power, Time
-from codecarbon.core.util import SLURM_JOB_ID, detect_cpu_model
+from codecarbon.core.util import SLURM_JOB_ID, count_cpus, detect_cpu_model
 from codecarbon.external.logger import logger
 
 # default W value for a CPU if no model is found in the ref csv
@@ -25,6 +26,8 @@ CONSUMPTION_PERCENTAGE_CONSTANT = 0.5
 
 B_TO_GB = 1024 * 1024 * 1024
 
+MODE_CPU_LOAD = "cpu_load"
+
 
 @dataclass
 class BaseHardware(ABC):
@@ -147,12 +150,20 @@ class CPU(BaseHardware):
         model: str,
         tdp: int,
         rapl_dir: str = "/sys/class/powercap/intel-rapl",
+        tracking_mode: str = "machine",
     ):
+        assert tracking_mode in ["machine", "process"]
+        self._power_history: List[Power] = []
         self._output_dir = output_dir
         self._mode = mode
         self._model = model
         self._tdp = tdp
         self._is_generic_tdp = False
+        self._tracking_mode = tracking_mode
+        self._pid = psutil.Process().pid
+        self._cpu_count = count_cpus()
+        self._process = psutil.Process(self._pid)
+
         if self._mode == "intel_power_gadget":
             self._intel_interface = IntelPowerGadget(self._output_dir)
         elif self._mode == "intel_rapl":
@@ -169,12 +180,62 @@ class CPU(BaseHardware):
 
         return s + ")"
 
+    @staticmethod
+    def _calculate_power_from_cpu_load(tdp, cpu_load, model):
+        if "AMD Ryzen Threadripper" in model:
+            return CPU._calculate_power_from_cpu_load_treadripper(tdp, cpu_load)
+        else:
+            return tdp * (cpu_load / 100.0)
+
+    @staticmethod
+    def _calculate_power_from_cpu_load_treadripper(tdp, cpu_load):
+        load = cpu_load / 100.0
+
+        if load < 0.1:  # Below 10% CPU load
+            return tdp * (0.05 * load * 10)
+        elif load <= 0.3:  # 10-30% load - linear phase
+            return tdp * (0.05 + 1.8 * (load - 0.1))
+        elif load <= 0.5:  # 30-50% load - adjusted coefficients
+            # Increased base power and adjusted curve
+            base_power = 0.45  # Increased from 0.41
+            power_range = 0.50  # Increased from 0.44
+            factor = ((load - 0.3) / 0.2) ** 1.8  # Reduced power from 2.0 to 1.8
+            return tdp * (base_power + power_range * factor)
+        else:  # Above 50% - plateau phase
+            return tdp * (0.85 + 0.15 * (1 - math.exp(-(load - 0.5) * 5)))
+
+    def _get_power_from_cpu_load(self):
+        """
+        When in MODE_CPU_LOAD
+        """
+        if self._tracking_mode == "machine":
+            tdp = self._tdp
+            cpu_load = psutil.cpu_percent(interval=0.5, percpu=False)
+            power = self._calculate_power_from_cpu_load(tdp, cpu_load, self._model)
+            logger.debug(
+                f"A TDP of {self._tdp} W and a CPU load of {cpu_load:.1f}% give an estimation of {power} W for whole machine."
+            )
+        elif self._tracking_mode == "process":
+            cpu_load = (
+                self._process.cpu_percent(interval=0.5, percpu=False) / self._cpu_count
+            )
+            power = self._calculate_power_from_cpu_load(self.tdp, cpu_load, self._model)
+            logger.debug(
+                f"A TDP of {self._tdp} W and a CPU load of {cpu_load * 100:.1f}% give an estimation of {power} W for process {self._pid}."
+            )
+        else:
+            raise Exception(f"Unknown tracking_mode {self._tracking_mode}")
+        return Power.from_watts(power)
+
     def _get_power_from_cpus(self) -> Power:
         """
         Get CPU power
         :return: power in kW
         """
-        if self._mode == "constant":
+        if self._mode == MODE_CPU_LOAD:
+            power = self._get_power_from_cpu_load()
+            return power
+        elif self._mode == "constant":
             power = self._tdp * CONSUMPTION_PERCENTAGE_CONSTANT
             return Power.from_watts(power)
         if self._mode == "intel_rapl":
@@ -208,20 +269,35 @@ class CPU(BaseHardware):
         return Energy.from_energy(energy)
 
     def total_power(self) -> Power:
-        cpu_power = self._get_power_from_cpus()
-        return cpu_power
+        self._power_history.append(self._get_power_from_cpus())
+        power_history_in_W = [power.W for power in self._power_history]
+        cpu_power = sum(power_history_in_W) / len(power_history_in_W)
+        self._power_history = []
+        return Power.from_watts(cpu_power)
 
     def measure_power_and_energy(self, last_duration: float) -> Tuple[Power, Energy]:
         if self._mode == "intel_rapl":
             energy = self._get_energy_from_cpus(delay=Time(seconds=last_duration))
             power = self.total_power()
+            # Patch AMD Threadripper that count 2x the power
+            if "AMD Ryzen Threadripper" in self._model:
+                power = power / 2
+                energy = energy / 2
             return power, energy
-        # If not intel_rapl
+        # If not intel_rapl, we call the parent method from BaseHardware
+        # to compute energy from power and time
         return super().measure_power_and_energy(last_duration=last_duration)
 
     def start(self):
         if self._mode in ["intel_power_gadget", "intel_rapl", "apple_powermetrics"]:
             self._intel_interface.start()
+        if self._mode == MODE_CPU_LOAD:
+            # The first time this is called it will return a meaningless 0.0 value which you are supposed to ignore.
+            _ = self._get_power_from_cpu_load()
+
+    def monitor_power(self):
+        cpu_power = self._get_power_from_cpus()
+        self._power_history.append(cpu_power)
 
     def get_model(self):
         return self._model
@@ -233,6 +309,7 @@ class CPU(BaseHardware):
         mode: str,
         model: Optional[str] = None,
         tdp: Optional[int] = None,
+        tracking_mode: str = "machine",
     ) -> "CPU":
         if model is None:
             model = detect_cpu_model()
@@ -245,7 +322,13 @@ class CPU(BaseHardware):
             cpu._is_generic_tdp = True
             return cpu
 
-        return cls(output_dir=output_dir, mode=mode, model=model, tdp=tdp)
+        return cls(
+            output_dir=output_dir,
+            mode=mode,
+            model=model,
+            tdp=tdp,
+            tracking_mode=tracking_mode,
+        )
 
 
 @dataclass

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/codecarbon/output_methods/file.py

@@ -78,12 +78,10 @@ class FileOutput(BaseOutput):
             self.output_dir, "emissions_" + experiment_name + "_" + run_id + ".csv"
         )
         df = pd.DataFrame(columns=data[0].values.keys())
-        df = pd.concat(
-            [
-                df,
-                pd.DataFrame.from_records(
-                    [dict(data_point.values) for data_point in data]
-                ),
-            ]
+        new_df = pd.DataFrame.from_records(
+            [dict(data_point.values) for data_point in data]
         )
+        # Filter out empty or all-NA columns, to avoid warnings from Pandas
+        new_df = new_df.dropna(axis=1, how="all")
+        df = pd.concat([df, new_df], ignore_index=True)
         df.to_csv(save_task_file_path, index=False)

{codecarbon-2.8.2 → codecarbon-3.0.0rc0}/pyproject.toml

@@ -183,8 +183,8 @@ include = [
 ]
 
 [tool.bumpver]
-current_version = "2.8.2"
-version_pattern = "MAJOR.MINOR.PATCH"
+current_version = "3.0.0_rc0"
+version_pattern = "MAJOR.MINOR.PATCH[_TAGNUM]"
 
 [tool.bumpver.file_patterns]
 "codecarbon/_version.py" = [

codecarbon-2.8.2/codecarbon/_version.py

@@ -1 +0,0 @@
-__version__ = "2.8.2"