codecarbon 3.3.0__tar.gz → 3.3.1__tar.gz

{codecarbon-3.3.0 → codecarbon-3.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codecarbon
-Version: 3.3.0
+Version: 3.3.1
 Author: Mila, DataForGood, BCG GAMMA, Comet.ml, Haverford College
 License-Expression: MIT
 Project-URL: Homepage, https://codecarbon.io/

codecarbon-3.3.1/codecarbon/_version.py

@@ -0,0 +1 @@
+__version__ = "3.3.1"

{codecarbon-3.3.0 → codecarbon-3.3.1}/codecarbon/cli/auth.py

@@ -86,8 +86,15 @@ def _discover_endpoints():
 
 def _save_credentials(tokens):
     """Save OAuth tokens to the local credentials file."""
-    with open(_CREDENTIALS_FILE, "w") as f:
-        json.dump(tokens, f)
+    try:
+        with open(_CREDENTIALS_FILE, "w") as f:
+            json.dump(tokens, f)
+    except OSError as e:
+        raise ValueError(
+            f"Could not write the credentials file {_CREDENTIALS_FILE.resolve()} "
+            f"(error: {e}). Please run the command from a directory you can "
+            "write to."
+        )
 
 
 def _load_credentials():
@@ -169,6 +176,7 @@ def authorize():
     server = HTTPServer(("localhost", _REDIRECT_PORT), _CallbackHandler)
 
     print("Opening browser for authentication...")
+    print(f"If no browser opens, copy this URL into your browser:\n{uri}")
     webbrowser.open(uri)
 
     server.handle_request()

{codecarbon-3.3.0 → codecarbon-3.3.1}/codecarbon/cli/main.py

@@ -358,6 +358,34 @@ def config():
     )
 
 
+def _cli_provided(ctx, name: str) -> bool:
+    """
+    Whether the option `name` was typed on the command line (or set through its
+    environment variable) rather than left at its Typer default.
+
+    Options left at their default must not be forwarded to the tracker: doing so
+    would silently override the values coming from `.codecarbon.config` and the
+    `CODECARBON_*` environment variables, which the tracker reads itself.
+    """
+    get_source = getattr(ctx, "get_parameter_source", None)
+    if get_source is None:
+        # `monitor` called directly from Python, not through Click: every value
+        # given is explicit.
+        return True
+    source = get_source(name)
+    return source is None or source.name in ("COMMANDLINE", "ENVIRONMENT")
+
+
+def _external_config() -> dict:
+    """The configuration files and CODECARBON_* variables, as a plain dict."""
+    from codecarbon.core.config import get_hierarchical_config
+
+    try:
+        return dict(get_hierarchical_config())
+    except Exception:
+        return {}
+
+
 @codecarbon.command(
     "monitor",
     short_help="Monitor your machine's carbon emissions.",
@@ -393,15 +421,28 @@ def monitor(
 ):
     """Monitor your machine's carbon emissions."""
 
-    # Shared tracker args so monitor and run_and_monitor behave the same
+    external_conf = _external_config()
+
+    # Shared tracker args so monitor and run_and_monitor behave the same.
+    # Only the options actually given are forwarded: the others are left to the
+    # tracker, which resolves them from the configuration file and environment.
+    cli_defaults = (
+        ("measure_power_secs", measure_power_secs),
+        ("api_call_interval", api_call_interval),
+        ("log_level", log_level),
+    )
     tracker_args = {
-        "measure_power_secs": measure_power_secs,
-        "api_call_interval": api_call_interval,
-        "log_level": log_level,
+        name: value for name, value in cli_defaults if _cli_provided(ctx, name)
     }
+    for name, value in cli_defaults:
+        if name not in tracker_args and name not in external_conf:
+            # Nothing configures it: keep the defaults advertised by `--help`
+            # (and an unattended monitor quiet) instead of the tracker's own.
+            tracker_args[name] = value
+
     # Set up the tracker arguments based on mode (offline vs online) and validate required args for each mode
     if offline:
-        if not country_iso_code:
+        if not country_iso_code and not external_conf.get("country_iso_code"):
             print(
                 "ERROR: Country ISO code is required for offline mode. Add it to your configuration or provide it via the command line: `--country-iso-code FRA`",
                 file=sys.stderr,
@@ -410,8 +451,8 @@ def monitor(
 
         tracker_args = {
             **tracker_args,
-            "country_iso_code": country_iso_code,
-            "region": region,
+            **({"country_iso_code": country_iso_code} if country_iso_code else {}),
+            **({"region": region} if region else {}),
         }
     else:
         experiment_id = get_existing_exp_id()

{codecarbon-3.3.0 → codecarbon-3.3.1}/codecarbon/cli/monitor.py

@@ -3,6 +3,7 @@
 import os
 import subprocess
 import sys
+from typing import Optional
 
 import typer
 from rich import print
@@ -12,9 +13,9 @@ from typing_extensions import Annotated
 def run_and_monitor(
     ctx: typer.Context,
     log_level: Annotated[
-        str,
+        Optional[str],
         typer.Option(help="Log level (critical, error, warning, info, debug)"),
-    ] = "error",
+    ] = None,
     offline: bool = False,
     **tracker_args,
 ):
@@ -51,7 +52,11 @@ def run_and_monitor(
     from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker
     from codecarbon.external.logger import set_logger_level
 
-    set_logger_level(log_level)
+    # `log_level` is None when nothing set it: leave it to the tracker, which
+    # resolves it from the configuration file and the environment.
+    if log_level is not None:
+        set_logger_level(log_level)
+        tracker_args["log_level"] = log_level
 
     # Get the command from remaining args (strip nested subcommand / `--` leftovers)
     command = list(getattr(ctx, "args", None) or [])
@@ -67,7 +72,6 @@ def run_and_monitor(
 
     tracker_cls = OfflineEmissionsTracker if offline else EmissionsTracker
     tracker = tracker_cls(
-        log_level=log_level,
         save_to_logger=False,
         tracking_mode="process",
         **tracker_args,

{codecarbon-3.3.0 → codecarbon-3.3.1}/codecarbon/core/api_client.py

@@ -22,6 +22,11 @@ from codecarbon.core.schemas import (
 from codecarbon.external.logger import logger
 
 
+def _round_or_none(value: float | None) -> float | None:
+    """Round a coordinate, keeping None when it is unknown."""
+    return None if value is None else round(value, 1)
+
+
 def get_datetime_with_timezone():
     import arrow
 
@@ -242,8 +247,8 @@ class ApiClient:  # (AsyncClient)
                 gpu_count=self.conf.get("gpu_count"),
                 gpu_model=self.conf.get("gpu_model"),
                 # Reduce precision for Privacy
-                longitude=round(self.conf.get("longitude", 0), 1),
-                latitude=round(self.conf.get("latitude", 0), 1),
+                longitude=_round_or_none(self.conf.get("longitude")),
+                latitude=_round_or_none(self.conf.get("latitude")),
                 region=self.conf.get("region"),
                 provider=self.conf.get("provider"),
                 ram_total_size=self.conf.get("ram_total_size"),

{codecarbon-3.3.0 → codecarbon-3.3.1}/codecarbon/core/config.py

@@ -1,10 +1,13 @@
 import configparser
 import os
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from codecarbon.external.logger import logger
 
+CONFIG_FILE_NAME = ".codecarbon.config"
+CONFIG_SECTION = "codecarbon"
+
 
 def clean_env_key(k: str) -> str:
     """
@@ -36,7 +39,7 @@ def parse_env_config() -> dict:
         dict: existing relevant environment variables mapped into a dict
     """
     return {
-        "codecarbon": {
+        CONFIG_SECTION: {
             clean_env_key(k): v
             for k, v in os.environ.items()
             if k.lower().startswith("codecarbon_")
@@ -110,10 +113,81 @@ def normalize_gpu_ids(
     return None
 
 
+def read_config_file(path: str) -> dict:
+    """
+    Read the `[codecarbon]` section of a single configuration file.
+
+    A file that does not exist, or that cannot be read, is ignored, like
+    configparser.ConfigParser.read does.
+
+    Args:
+        path (str): Path of the configuration file to read.
+
+    Returns:
+        dict: The keys defined in this file only, empty if there is nothing to read.
+    """
+    config = configparser.ConfigParser()
+    config.read(path)
+    if not config.has_section(CONFIG_SECTION):
+        return {}
+    return dict(config[CONFIG_SECTION])
+
+
+def log_config_sources(sources: Dict[str, str]) -> None:
+    """
+    Log a single line telling, for every configuration key, which layer set it.
+
+    Only the key names and their source are logged, never the values, because
+    this configuration holds secrets such as `api_key` and `experiment_id`.
+
+    Args:
+        sources (dict): The key to source mapping built by
+            get_hierarchical_config_with_sources.
+    """
+    if not sources:
+        return
+    logger.info(
+        "Codecarbon configuration: "
+        + ", ".join(f"{key} from {source}" for key, source in sorted(sources.items()))
+    )
+
+
+def get_hierarchical_config_with_sources() -> Tuple[dict, Dict[str, str]]:
+    """
+    Same as get_hierarchical_config, but also tells where each key comes from.
+
+    The layers are read from the lowest to the highest priority: the global
+    config file, then the local config file, then the environment variables.
+    The last layer defining a key wins, and is the one reported as its source.
+    A source is either the path of the config file that set the key, or the
+    name of the environment variable, e.g. `CODECARBON_PROJECT_NAME`.
+
+    Returns:
+        tuple: The (config, sources) dicts, which share the same keys. `config`
+        maps a key to its value (**all values are strings**) and `sources` maps
+        a key to the layer it was read from.
+    """
+    global_path = str((Path.home() / CONFIG_FILE_NAME).expanduser().resolve())
+    local_path = str((Path.cwd() / CONFIG_FILE_NAME).expanduser().resolve())
+
+    config = {}
+    sources = {}
+
+    for path in (global_path, local_path):
+        for key, value in read_config_file(path).items():
+            config[key] = value
+            sources[key] = path
+
+    for key, value in parse_env_config()[CONFIG_SECTION].items():
+        config[key] = value
+        sources[key] = f"CODECARBON_{key.upper()}"
+
+    return config, sources
+
+
 def get_hierarchical_config():
     """
-    Get the user-defined codecarbon configuration ConfigParser dictionnary
-    (actually a configparser.SectionProxy instance).
+    Get the user-defined codecarbon configuration as a dict.
 
     ```
     >>> from codecarbon.core.config import get_hierarchical_config
@@ -121,10 +195,7 @@ def get_hierarchical_config():
     >>> print(conf)
     ```
 
-    `conf` works like a regular dict + methods getint(key) getfloat(key)
-    and getboolean(key) to automatically parse strings into those types.
-
-    All values (outputs of get(key)) are strings.
+    All values are strings.
 
     It looks for, and reads, a config file .codecarbon.config in the user's $HOME.
     It then looks for, reads, and updates the previous configuration from a config
@@ -133,29 +204,13 @@ def get_hierarchical_config():
     with `CODECARBON_` (for instance if `CODECARBON_PROJECT_NAME` is `your-project`
     then the resulting configuration key `project_name` will have value `your-project`)
 
+    The source of every resolved key is logged, so that a surprising value can be
+    traced back to the file or the environment variable that set it.
+
     Returns:
         dict: The final configuration dict parsed from global,
         local and environment configurations. **All values are strings**.
     """
-
-    config = configparser.ConfigParser()
-
-    cwd = Path.cwd()
-    home = Path.home()
-    global_path = str((home / ".codecarbon.config").expanduser().resolve())
-    local_path = str((cwd / ".codecarbon.config").expanduser().resolve())
-    if Path(global_path).exists():
-        logger.info(
-            f"Codecarbon is taking the configuration from global file: {global_path}"
-        )
-        if Path(local_path).exists():
-            logger.info(f"Some variables are overriden by the local file: {local_path}")
-    elif Path(local_path).exists():
-        logger.info(
-            f"Codecarbon is taking the configuration from the local file {local_path}"
-        )
-
-    config.read([global_path, local_path])
-    config.read_dict(parse_env_config())
-
-    return dict(config["codecarbon"])
+    config, sources = get_hierarchical_config_with_sources()
+    log_config_sources(sources)
+    return config

{codecarbon-3.3.0 → codecarbon-3.3.1}/codecarbon/core/cpu.py

@@ -17,7 +17,12 @@ from typing import TYPE_CHECKING, Dict, Optional, Tuple
 import psutil
 from rapidfuzz import fuzz, process, utils
 
-from codecarbon.core.rapl import RAPLFile
+from codecarbon.core.rapl import (
+    SEQUENTIAL_READ_TOLERANCE_KWH,
+    RAPLFile,
+    counters_match,
+    find_mirrored_counters,
+)
 from codecarbon.core.units import Time
 from codecarbon.core.util import count_cpus, detect_cpu_model
 from codecarbon.external.logger import logger
@@ -450,6 +455,10 @@ class IntelRAPL:
         self._lin_rapl_dir = rapl_dir
         self._system = sys.platform.lower()
         self._rapl_files = []
+        # Files that look like a duplicate of another counter, but whose
+        # energy deltas have not confirmed it yet. They are left out of the
+        # measurement while pending. Maps file path -> mirrored file path.
+        self._mirrored_candidates: Dict[str, str] = {}
         self.rapl_include_dram = rapl_include_dram
         self.rapl_prefer_psys = rapl_prefer_psys
         self._setup_rapl()
@@ -842,7 +851,12 @@ class IntelRAPL:
             for rapl_file in self._rapl_files:
                 rapl_file.delta(duration)
 
+            self._confirm_mirrored_files()
+
             for rapl_file in self._rapl_files:
+                if rapl_file.path in self._mirrored_candidates:
+                    # Still suspected of duplicating another counter
+                    continue
                 logger.debug(rapl_file)
                 cpu_details[rapl_file.name] = rapl_file.energy_delta.kWh
                 # We fake the name used by Power Gadget when using RAPL
@@ -874,6 +888,71 @@ class IntelRAPL:
         """
         for rapl_file in self._rapl_files:
             rapl_file.start()
+        # DRAM domains are a different measurement, never a duplicate
+        counters = [
+            (rapl_file.path, float(rapl_file.last_energy))
+            for rapl_file in self._rapl_files
+            if "dram" not in rapl_file.name.lower()
+        ]
+        self._mirrored_candidates = find_mirrored_counters(
+            counters, SEQUENTIAL_READ_TOLERANCE_KWH
+        )
+        for path, reference_path in self._mirrored_candidates.items():
+            logger.warning(
+                "\tRAPL - Energy counter at %s looks like a mirror of %s, it is "
+                "left out of the measurement to avoid double-counting",
+                path,
+                reference_path,
+            )
+
+    def _confirm_mirrored_files(self) -> None:
+        """
+        Decide the fate of the files flagged as duplicates at ``start()``.
+
+        Two independent meters could hold indistinguishable counters at the
+        single instant of the initial reading, so a candidate is only dropped
+        for good once it has also accumulated the very same energy over a
+        measurement interval. A candidate whose energy delta differs is a real
+        meter and is restored, and a candidate that has not accumulated
+        anything yet stays pending until an interval is conclusive.
+        """
+        if not self._mirrored_candidates:
+            return
+        files_by_path = {rapl_file.path: rapl_file for rapl_file in self._rapl_files}
+        pending = {}
+        confirmed = set()
+        for path, reference_path in self._mirrored_candidates.items():
+            rapl_file = files_by_path.get(path)
+            reference = files_by_path.get(reference_path)
+            if rapl_file is None or reference is None:
+                continue
+            delta = float(rapl_file.energy_delta)
+            reference_delta = float(reference.energy_delta)
+            if delta <= 0 and reference_delta <= 0:
+                # Nothing was accumulated: the interval is not conclusive
+                pending[path] = reference_path
+            elif counters_match(delta, reference_delta, SEQUENTIAL_READ_TOLERANCE_KWH):
+                confirmed.add(path)
+                logger.warning(
+                    "\tRAPL - Energy counter at %s mirrors the one at %s, it is "
+                    "dropped to avoid double-counting the CPU power (multi-die "
+                    "CPUs mirror the package counter on each die)",
+                    path,
+                    reference_path,
+                )
+            else:
+                logger.info(
+                    "\tRAPL - Energy counter at %s accumulates energy on its own, "
+                    "it is an independent meter and is measured again",
+                    path,
+                )
+        self._mirrored_candidates = pending
+        if confirmed:
+            self._rapl_files = [
+                rapl_file
+                for rapl_file in self._rapl_files
+                if rapl_file.path not in confirmed
+            ]
 
 
 class TDP:

{codecarbon-3.3.0 → codecarbon-3.3.1}/codecarbon/core/powermetrics.py

@@ -30,6 +30,26 @@ def clear_powermetrics_cache() -> None:
     is_powermetrics_available.cache_clear()
 
 
+def _kill_process(process: subprocess.Popen) -> None:
+    """
+    Kill a Powermetrics subprocess, tolerating a failure to do so.
+
+    The command runs through `sudo`, so the child may be owned by root and
+    refuse our signal. Letting that raise would replace the timeout we are
+    handling with a PermissionError, and waiting on it without a bound would
+    hang for as long as the child lives.
+    """
+    try:
+        process.kill()
+        process.wait(timeout=1)
+    except (OSError, subprocess.SubprocessError):
+        logger.debug("Could not kill the Powermetrics process.")
+    finally:
+        for stream in (process.stdout, process.stderr):
+            if stream is not None:
+                stream.close()
+
+
 def _has_powermetrics_sudo() -> bool:
     if shutil.which("sudo") is None:
         logger.debug("sudo not available, we won't use Apple PowerMetrics.")
@@ -40,7 +60,9 @@ def _has_powermetrics_sudo() -> bool:
         )
         return False
 
-    with subprocess.Popen(
+    # No context manager here: Popen.__exit__ would wait() without a timeout,
+    # which is exactly the hang we are trying to avoid when the kill fails.
+    process = subprocess.Popen(
         [
             "sudo",
             "powermetrics",
@@ -53,30 +75,31 @@ def _has_powermetrics_sudo() -> bool:
             "-o",
             "/dev/null",
         ],
+        stdin=subprocess.DEVNULL,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         text=True,
-    ) as process:
-        deadline = time.time() + 3
-        while process.poll() is None and time.time() < deadline:
-            time.sleep(0.05)
-        if process.poll() is None:
-            process.kill()
-            logger.debug("PowerMetrics sudo check timed out.")
-            return False
-        _, stderr = process.communicate()
-
-        if re.search(r"[sudo].*password", stderr):
-            logger.debug("""Not using PowerMetrics, sudo password prompt detected.
-                    If you want to enable Powermetrics please modify your sudoers file
-                    as described in :
-                    https://docs.codecarbon.io/latest/explanation/methodology/#power-usage
-                """)
-            return False
-        if process.returncode != 0:
-            raise Exception("Return code != 0")
+    )
+    deadline = time.time() + 3
+    while process.poll() is None and time.time() < deadline:
+        time.sleep(0.05)
+    if process.poll() is None:
+        logger.debug("PowerMetrics sudo check timed out.")
+        _kill_process(process)
+        return False
+    _, stderr = process.communicate()
 
-        return True
+    if re.search(r"[sudo].*password", stderr):
+        logger.debug("""Not using PowerMetrics, sudo password prompt detected.
+                If you want to enable Powermetrics please modify your sudoers file
+                as described in :
+                https://docs.codecarbon.io/latest/explanation/methodology/#power-usage
+            """)
+        return False
+    if process.returncode != 0:
+        raise Exception("Return code != 0")
+
+    return True
 
 
 class ApplePowermetrics:
@@ -130,47 +153,69 @@ class ApplePowermetrics:
         else:
             raise SystemError("Platform not supported by Powermetrics")
 
-    def _log_values(self) -> None:
+    def _log_values(self) -> bool:
         """
         Logs output from Powermetrics to a file
-        """
-        returncode = None
 
-        if self._system.startswith("darwin"):
-            # Run the powermetrics command with sudo and capture its output
-            cmd = [
-                "sudo",
-                "powermetrics",
-                "-n",
-                str(self._n_points),
-                "",
-                "--samplers",
-                "cpu_power",
-                "--format",
-                "csv",
-                "-i",
-                str(self._interval),
-                "-o",
-                self._log_file_path,
-            ]
-            returncode = subprocess.call(cmd, universal_newlines=True)
+        :return: False when the command did not run, so that the caller knows
+                 the log file was not refreshed and still holds the previous
+                 measure.
+        """
+        if not self._system.startswith("darwin"):
+            return False
 
-        else:
-            return None
+        # Run the powermetrics command with sudo and capture its output
+        cmd = [
+            "sudo",
+            "powermetrics",
+            "-n",
+            str(self._n_points),
+            "",
+            "--samplers",
+            "cpu_power",
+            "--format",
+            "csv",
+            "-i",
+            str(self._interval),
+            "-o",
+            self._log_file_path,
+        ]
+        # _n_points samples of _interval ms is the nominal runtime of the
+        # command. Double it to absorb sampler overhead and scheduling jitter,
+        # and add a floor so short configurations keep a usable margin. This is
+        # a heuristic: it only has to be loose enough that a healthy run never
+        # trips it, because its only job is to bound a hang.
+        timeout = self._n_points * self._interval / 1000 * 2 + 5
+        # DEVNULL so that sudo gets EOF and fails fast instead of blocking
+        # forever on a password prompt when it has no cached credential.
+        # No context manager: Popen.__exit__ would wait() without a timeout.
+        process = subprocess.Popen(
+            cmd, universal_newlines=True, stdin=subprocess.DEVNULL
+        )
+        try:
+            returncode = process.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            logger.warning(
+                f"Powermetrics did not complete within {timeout:g} seconds, "
+                "skipping this measure."
+            )
+            _kill_process(process)
+            return False
 
         if returncode != 0:
             logger.warning(
                 "Returncode while logging power values using "
                 + f"Powermetrics: {returncode}"
             )
-        return
+        return True
 
     def get_details(self) -> Dict:
         """
         Fetches the CPU Power Details by fetching values from a logged csv file
         in _log_values function
         """
-        self._log_values()
+        if not self._log_values():
+            return dict()
         details = dict()
         try:
             with open(self._log_file_path) as f:

{codecarbon-3.3.0 → codecarbon-3.3.1}/codecarbon/core/rapl.py

@@ -3,6 +3,54 @@ from dataclasses import dataclass, field
 from codecarbon.core.units import Energy, Power, Time
 from codecarbon.external.logger import logger
 
+# Maximum relative difference between two cumulative energy counters for them
+# to be considered two views of the same hardware counter. Multi-die CPUs
+# (e.g. AMD Ryzen Threadripper / EPYC) can expose one package domain per die
+# that all mirror the same socket-wide counter, which would multiply the
+# reported CPU power by the number of dies.
+# See https://github.com/mlco2/codecarbon/issues/1274
+MIRRORED_COUNTER_TOLERANCE = 1e-6
+
+# Unlike EMI, which snapshots all the channels of a device atomically, the
+# powercap sysfs files are read one after the other: two views of the same
+# hardware counter differ by the energy the package accrued between the two
+# reads. One joule covers several milliseconds at full load, while staying
+# far below what independent packages drift apart by over a measurement
+# interval.
+SEQUENTIAL_READ_TOLERANCE_KWH = float(Energy.from_ujoules(1_000_000))  # 1 J
+
+
+def counters_match(first, second, abs_tolerance=0.0) -> bool:
+    """Tell whether two cumulative counter values are indistinguishable."""
+    return abs(first - second) <= max(
+        MIRRORED_COUNTER_TOLERANCE * max(first, second), abs_tolerance
+    )
+
+
+def find_mirrored_counters(counters, abs_tolerance=0.0) -> dict:
+    """
+    Identify the counters that report the same underlying energy value.
+
+    ``counters`` is an ordered list of ``(key, absolute_energy)`` pairs whose
+    energies share a unit. Two independent meters are extremely unlikely to
+    hold the very same cumulative count, so equal counters point at duplicated
+    ones. A zero counter carries no information and is never flagged.
+
+    Returns a mapping of each duplicating key to the earlier key it mirrors.
+    """
+    references = []
+    mirrored = {}
+    for key, energy in counters:
+        if energy <= 0:
+            continue
+        for reference_key, reference in references:
+            if counters_match(energy, reference, abs_tolerance):
+                mirrored[key] = reference_key
+                break
+        else:
+            references.append((key, energy))
+    return mirrored
+
 
 @dataclass
 class RAPLFile: