codecarbon 3.3.0__tar.gz → 3.3.2__tar.gz

{codecarbon-3.3.0 → codecarbon-3.3.2}/PKG-INFO

@@ -1,6 +1,7 @@
 Metadata-Version: 2.4
 Name: codecarbon
-Version: 3.3.0
+Version: 3.3.2
+Summary: Track and reduce the CO2 emissions of your computing
 Author: Mila, DataForGood, BCG GAMMA, Comet.ml, Haverford College
 License-Expression: MIT
 Project-URL: Homepage, https://codecarbon.io/
@@ -9,12 +10,25 @@ Project-URL: Dashboard, http://dashboard.codecarbon.io/
 Project-URL: Documentation, https://docs.codecarbon.io/
 Project-URL: Issues, https://github.com/mlco2/codecarbon/issues
 Project-URL: Changelog, https://github.com/mlco2/codecarbon/releases
+Keywords: carbon,co2,emissions,carbon-footprint,sustainability,green-ai,green-computing,energy,energy-consumption,power-consumption,rapl,gpu,machine-learning,mlops,monitoring
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
 Classifier: Natural Language :: English
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: System :: Monitoring
+Classifier: Topic :: System :: Benchmark
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE

codecarbon-3.3.2/codecarbon/_version.py

@@ -0,0 +1 @@
+__version__ = "3.3.2"

{codecarbon-3.3.0 → codecarbon-3.3.2}/codecarbon/cli/auth.py

@@ -86,8 +86,15 @@ def _discover_endpoints():
 
 def _save_credentials(tokens):
     """Save OAuth tokens to the local credentials file."""
-    with open(_CREDENTIALS_FILE, "w") as f:
-        json.dump(tokens, f)
+    try:
+        with open(_CREDENTIALS_FILE, "w") as f:
+            json.dump(tokens, f)
+    except OSError as e:
+        raise ValueError(
+            f"Could not write the credentials file {_CREDENTIALS_FILE.resolve()} "
+            f"(error: {e}). Please run the command from a directory you can "
+            "write to."
+        )
 
 
 def _load_credentials():
@@ -169,6 +176,7 @@ def authorize():
     server = HTTPServer(("localhost", _REDIRECT_PORT), _CallbackHandler)
 
     print("Opening browser for authentication...")
+    print(f"If no browser opens, copy this URL into your browser:\n{uri}")
     webbrowser.open(uri)
 
     server.handle_request()

{codecarbon-3.3.0 → codecarbon-3.3.2}/codecarbon/cli/main.py

@@ -358,6 +358,34 @@ def config():
     )
 
 
+def _cli_provided(ctx, name: str) -> bool:
+    """
+    Whether the option `name` was typed on the command line (or set through its
+    environment variable) rather than left at its Typer default.
+
+    Options left at their default must not be forwarded to the tracker: doing so
+    would silently override the values coming from `.codecarbon.config` and the
+    `CODECARBON_*` environment variables, which the tracker reads itself.
+    """
+    get_source = getattr(ctx, "get_parameter_source", None)
+    if get_source is None:
+        # `monitor` called directly from Python, not through Click: every value
+        # given is explicit.
+        return True
+    source = get_source(name)
+    return source is None or source.name in ("COMMANDLINE", "ENVIRONMENT")
+
+
+def _external_config() -> dict:
+    """The configuration files and CODECARBON_* variables, as a plain dict."""
+    from codecarbon.core.config import get_hierarchical_config
+
+    try:
+        return dict(get_hierarchical_config())
+    except Exception:
+        return {}
+
+
 @codecarbon.command(
     "monitor",
     short_help="Monitor your machine's carbon emissions.",
@@ -365,6 +393,10 @@ def config():
 )
 def monitor(
     ctx: typer.Context,
+    project_name: Annotated[
+        str,
+        typer.Option(help="Project name for the current experiment."),
+    ] = None,
     measure_power_secs: Annotated[
         int,
         typer.Option(help="Interval between two measures."),
@@ -393,15 +425,29 @@ def monitor(
 ):
     """Monitor your machine's carbon emissions."""
 
-    # Shared tracker args so monitor and run_and_monitor behave the same
+    external_conf = _external_config()
+
+    # Shared tracker args so monitor and run_and_monitor behave the same.
+    # Only the options actually given are forwarded: the others are left to the
+    # tracker, which resolves them from the configuration file and environment.
+    cli_defaults = (
+        ("project_name", project_name),
+        ("measure_power_secs", measure_power_secs),
+        ("api_call_interval", api_call_interval),
+        ("log_level", log_level),
+    )
     tracker_args = {
-        "measure_power_secs": measure_power_secs,
-        "api_call_interval": api_call_interval,
-        "log_level": log_level,
+        name: value for name, value in cli_defaults if _cli_provided(ctx, name)
     }
+    for name, value in cli_defaults:
+        if name not in tracker_args and name not in external_conf:
+            # Nothing configures it: keep the defaults advertised by `--help`
+            # (and an unattended monitor quiet) instead of the tracker's own.
+            tracker_args[name] = value
+
     # Set up the tracker arguments based on mode (offline vs online) and validate required args for each mode
     if offline:
-        if not country_iso_code:
+        if not country_iso_code and not external_conf.get("country_iso_code"):
             print(
                 "ERROR: Country ISO code is required for offline mode. Add it to your configuration or provide it via the command line: `--country-iso-code FRA`",
                 file=sys.stderr,
@@ -410,8 +456,8 @@ def monitor(
 
         tracker_args = {
             **tracker_args,
-            "country_iso_code": country_iso_code,
-            "region": region,
+            **({"country_iso_code": country_iso_code} if country_iso_code else {}),
+            **({"region": region} if region else {}),
         }
     else:
         experiment_id = get_existing_exp_id()

{codecarbon-3.3.0 → codecarbon-3.3.2}/codecarbon/cli/monitor.py

@@ -3,6 +3,7 @@
 import os
 import subprocess
 import sys
+from typing import Optional
 
 import typer
 from rich import print
@@ -12,9 +13,9 @@ from typing_extensions import Annotated
 def run_and_monitor(
     ctx: typer.Context,
     log_level: Annotated[
-        str,
+        Optional[str],
         typer.Option(help="Log level (critical, error, warning, info, debug)"),
-    ] = "error",
+    ] = None,
     offline: bool = False,
     **tracker_args,
 ):
@@ -51,7 +52,11 @@ def run_and_monitor(
     from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker
     from codecarbon.external.logger import set_logger_level
 
-    set_logger_level(log_level)
+    # `log_level` is None when nothing set it: leave it to the tracker, which
+    # resolves it from the configuration file and the environment.
+    if log_level is not None:
+        set_logger_level(log_level)
+        tracker_args["log_level"] = log_level
 
     # Get the command from remaining args (strip nested subcommand / `--` leftovers)
     command = list(getattr(ctx, "args", None) or [])
@@ -67,7 +72,6 @@ def run_and_monitor(
 
     tracker_cls = OfflineEmissionsTracker if offline else EmissionsTracker
     tracker = tracker_cls(
-        log_level=log_level,
         save_to_logger=False,
         tracking_mode="process",
         **tracker_args,
@@ -116,13 +120,12 @@ def run_and_monitor(
         else:
             print("   Emissions: N/A")
 
-        # Show where the data was saved
-        if hasattr(tracker, "_conf") and "output_file" in tracker._conf:
-            output_path = tracker._conf["output_file"]
-            # Make it absolute if it's relative
-            if not os.path.isabs(output_path):
-                output_path = os.path.abspath(output_path)
-            print(f"   Saved to: {output_path}")
+        # Show where the data was saved, asking the output handlers themselves
+        # so the path stays right whatever `output_dir` / `output_file` are.
+        for handler in getattr(tracker, "_output_handlers", []):
+            save_file_path = getattr(handler, "save_file_path", None)
+            if save_file_path:
+                print(f"   Saved to: {os.path.abspath(save_file_path)}")
 
         print("   ⚠️  Note: Tracked the command process and its children")
         print("=" * 60)

{codecarbon-3.3.0 → codecarbon-3.3.2}/codecarbon/core/api_client.py

@@ -22,6 +22,11 @@ from codecarbon.core.schemas import (
 from codecarbon.external.logger import logger
 
 
+def _round_or_none(value: float | None) -> float | None:
+    """Round a coordinate, keeping None when it is unknown."""
+    return None if value is None else round(value, 1)
+
+
 def get_datetime_with_timezone():
     import arrow
 
@@ -242,8 +247,8 @@ class ApiClient:  # (AsyncClient)
                 gpu_count=self.conf.get("gpu_count"),
                 gpu_model=self.conf.get("gpu_model"),
                 # Reduce precision for Privacy
-                longitude=round(self.conf.get("longitude", 0), 1),
-                latitude=round(self.conf.get("latitude", 0), 1),
+                longitude=_round_or_none(self.conf.get("longitude")),
+                latitude=_round_or_none(self.conf.get("latitude")),
                 region=self.conf.get("region"),
                 provider=self.conf.get("provider"),
                 ram_total_size=self.conf.get("ram_total_size"),

{codecarbon-3.3.0 → codecarbon-3.3.2}/codecarbon/core/config.py

@@ -1,10 +1,13 @@
 import configparser
 import os
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from codecarbon.external.logger import logger
 
+CONFIG_FILE_NAME = ".codecarbon.config"
+CONFIG_SECTION = "codecarbon"
+
 
 def clean_env_key(k: str) -> str:
     """
@@ -36,7 +39,7 @@ def parse_env_config() -> dict:
         dict: existing relevant environment variables mapped into a dict
     """
     return {
-        "codecarbon": {
+        CONFIG_SECTION: {
             clean_env_key(k): v
             for k, v in os.environ.items()
             if k.lower().startswith("codecarbon_")
@@ -110,10 +113,81 @@ def normalize_gpu_ids(
     return None
 
 
+def read_config_file(path: str) -> dict:
+    """
+    Read the `[codecarbon]` section of a single configuration file.
+
+    A file that does not exist, or that cannot be read, is ignored, like
+    configparser.ConfigParser.read does.
+
+    Args:
+        path (str): Path of the configuration file to read.
+
+    Returns:
+        dict: The keys defined in this file only, empty if there is nothing to read.
+    """
+    config = configparser.ConfigParser()
+    config.read(path)
+    if not config.has_section(CONFIG_SECTION):
+        return {}
+    return dict(config[CONFIG_SECTION])
+
+
+def log_config_sources(sources: Dict[str, str]) -> None:
+    """
+    Log a single line telling, for every configuration key, which layer set it.
+
+    Only the key names and their source are logged, never the values, because
+    this configuration holds secrets such as `api_key` and `experiment_id`.
+
+    Args:
+        sources (dict): The key to source mapping built by
+            get_hierarchical_config_with_sources.
+    """
+    if not sources:
+        return
+    logger.info(
+        "Codecarbon configuration: "
+        + ", ".join(f"{key} from {source}" for key, source in sorted(sources.items()))
+    )
+
+
+def get_hierarchical_config_with_sources() -> Tuple[dict, Dict[str, str]]:
+    """
+    Same as get_hierarchical_config, but also tells where each key comes from.
+
+    The layers are read from the lowest to the highest priority: the global
+    config file, then the local config file, then the environment variables.
+    The last layer defining a key wins, and is the one reported as its source.
+    A source is either the path of the config file that set the key, or the
+    name of the environment variable, e.g. `CODECARBON_PROJECT_NAME`.
+
+    Returns:
+        tuple: The (config, sources) dicts, which share the same keys. `config`
+        maps a key to its value (**all values are strings**) and `sources` maps
+        a key to the layer it was read from.
+    """
+    global_path = str((Path.home() / CONFIG_FILE_NAME).expanduser().resolve())
+    local_path = str((Path.cwd() / CONFIG_FILE_NAME).expanduser().resolve())
+
+    config = {}
+    sources = {}
+
+    for path in (global_path, local_path):
+        for key, value in read_config_file(path).items():
+            config[key] = value
+            sources[key] = path
+
+    for key, value in parse_env_config()[CONFIG_SECTION].items():
+        config[key] = value
+        sources[key] = f"CODECARBON_{key.upper()}"
+
+    return config, sources
+
+
 def get_hierarchical_config():
     """
-    Get the user-defined codecarbon configuration ConfigParser dictionnary
-    (actually a configparser.SectionProxy instance).
+    Get the user-defined codecarbon configuration as a dict.
 
     ```
     >>> from codecarbon.core.config import get_hierarchical_config
@@ -121,10 +195,7 @@ def get_hierarchical_config():
     >>> print(conf)
     ```
 
-    `conf` works like a regular dict + methods getint(key) getfloat(key)
-    and getboolean(key) to automatically parse strings into those types.
-
-    All values (outputs of get(key)) are strings.
+    All values are strings.
 
     It looks for, and reads, a config file .codecarbon.config in the user's $HOME.
     It then looks for, reads, and updates the previous configuration from a config
@@ -133,29 +204,13 @@ def get_hierarchical_config():
     with `CODECARBON_` (for instance if `CODECARBON_PROJECT_NAME` is `your-project`
     then the resulting configuration key `project_name` will have value `your-project`)
 
+    The source of every resolved key is logged, so that a surprising value can be
+    traced back to the file or the environment variable that set it.
+
     Returns:
         dict: The final configuration dict parsed from global,
         local and environment configurations. **All values are strings**.
     """
-
-    config = configparser.ConfigParser()
-
-    cwd = Path.cwd()
-    home = Path.home()
-    global_path = str((home / ".codecarbon.config").expanduser().resolve())
-    local_path = str((cwd / ".codecarbon.config").expanduser().resolve())
-    if Path(global_path).exists():
-        logger.info(
-            f"Codecarbon is taking the configuration from global file: {global_path}"
-        )
-        if Path(local_path).exists():
-            logger.info(f"Some variables are overriden by the local file: {local_path}")
-    elif Path(local_path).exists():
-        logger.info(
-            f"Codecarbon is taking the configuration from the local file {local_path}"
-        )
-
-    config.read([global_path, local_path])
-    config.read_dict(parse_env_config())
-
-    return dict(config["codecarbon"])
+    config, sources = get_hierarchical_config_with_sources()
+    log_config_sources(sources)
+    return config

{codecarbon-3.3.0 → codecarbon-3.3.2}/codecarbon/core/cpu.py

@@ -17,7 +17,12 @@ from typing import TYPE_CHECKING, Dict, Optional, Tuple
 import psutil
 from rapidfuzz import fuzz, process, utils
 
-from codecarbon.core.rapl import RAPLFile
+from codecarbon.core.rapl import (
+    SEQUENTIAL_READ_TOLERANCE_KWH,
+    RAPLFile,
+    counters_match,
+    find_mirrored_counters,
+)
 from codecarbon.core.units import Time
 from codecarbon.core.util import count_cpus, detect_cpu_model
 from codecarbon.external.logger import logger
@@ -28,6 +33,13 @@ if TYPE_CHECKING:
 # default W value per core for a CPU if no model is found in the ref csv
 DEFAULT_POWER_PER_CORE = 4
 
+_TROUBLESHOOTING_URL = "https://docs.codecarbon.io/latest/how-to/troubleshooting/"
+RAPL_PERMISSION_HELP = (
+    "You can grant read permission with: sudo chmod -R a+r /sys/class/powercap/* "
+    "(this does not persist across reboots, see "
+    f"{_TROUBLESHOOTING_URL}#rapl-permission-denied for a permanent setup)"
+)
+
 
 @lru_cache(maxsize=1)
 def is_powergadget_available() -> bool:
@@ -175,18 +187,16 @@ def _create_warn_function():
         nonlocal already_warned
         if not already_warned:
             logger.warning(
-                "\tRAPL - Permission denied reading RAPL file %s. "
-                "You can grant read permission with: "
-                "sudo chmod -R a+r /sys/class/powercap/*",
+                "\tRAPL - Permission denied reading RAPL file %s. %s",
                 energy_path,
+                RAPL_PERMISSION_HELP,
             )
             already_warned = True
         else:
             logger.debug(
-                "\tRAPL - Permission denied reading RAPL file %s. "
-                "You can grant read permission with: "
-                "sudo chmod -R a+r /sys/class/powercap/*",
+                "\tRAPL - Permission denied reading RAPL file %s. %s",
                 energy_path,
+                RAPL_PERMISSION_HELP,
             )
 
     return warn_permission_denied
@@ -248,6 +258,24 @@ def is_psutil_available():
         return False
 
 
+def is_raspberry() -> bool:
+    """
+    Check if the Raspberry Pi PMIC power readings are available.
+
+    `vcgencmd` ships with every Raspberry Pi OS, but only the Pi 5 answers
+    `pmic_read_adc`: older boards fall back to the TDP of their CPU.
+    """
+    if not os.path.exists("/usr/bin/vcgencmd"):
+        return False
+    try:
+        res = subprocess.run(
+            ["vcgencmd", "pmic_read_adc"], capture_output=True, timeout=5
+        )
+    except (OSError, subprocess.SubprocessError):
+        return False
+    return res.returncode == 0 and b"volt(" in res.stdout
+
+
 class IntelPowerGadget:
     """
     A class to interface with Intel Power Gadget for monitoring CPU power consumption on Windows and (non-Apple Silicon) macOS.
@@ -450,6 +478,10 @@ class IntelRAPL:
         self._lin_rapl_dir = rapl_dir
         self._system = sys.platform.lower()
         self._rapl_files = []
+        # Files that look like a duplicate of another counter, but whose
+        # energy deltas have not confirmed it yet. They are left out of the
+        # measurement while pending. Maps file path -> mirrored file path.
+        self._mirrored_candidates: Dict[str, str] = {}
         self.rapl_include_dram = rapl_include_dram
         self.rapl_prefer_psys = rapl_prefer_psys
         self._setup_rapl()
@@ -480,7 +512,7 @@ class IntelRAPL:
         """Scan subdirectories of a RAPL entry for domain directories."""
         subdirs = []
         try:
-            for sub in os.listdir(entry_path):
+            for sub in sorted(os.listdir(entry_path)):
                 sub_path = os.path.join(entry_path, sub)
                 if ":" in sub and os.path.isdir(sub_path):
                     if os.path.exists(os.path.join(sub_path, "energy_uj")):
@@ -499,7 +531,7 @@ class IntelRAPL:
         """Scan a base directory for intel-rapl entries."""
         domain_dirs = []
         try:
-            for entry in os.listdir(base):
+            for entry in sorted(os.listdir(base)):
                 if not entry.startswith("intel-rapl"):
                     continue
                 entry_path = os.path.join(base, entry)
@@ -528,7 +560,7 @@ class IntelRAPL:
         if domain_dirs:
             return domain_dirs
         try:
-            for item in os.listdir(self._lin_rapl_dir):
+            for item in sorted(os.listdir(self._lin_rapl_dir)):
                 if ":" in item:
                     path = os.path.join(self._lin_rapl_dir, item)
                     if os.path.isdir(path) and os.path.exists(
@@ -583,8 +615,7 @@ class IntelRAPL:
             return True, is_required_main
         except PermissionError:
             msg = f"\tRAPL - Permission denied reading RAPL file {rapl_file}."
-            suggestion = "You can grant read permission with: sudo chmod -R a+r /sys/class/powercap/*"
-            logger.warning("%s %s; skipping.", msg, suggestion)
+            logger.warning("%s %s; skipping.", msg, RAPL_PERMISSION_HELP)
             return False, False
         except Exception as e:
             logger.debug(
@@ -842,7 +873,12 @@ class IntelRAPL:
             for rapl_file in self._rapl_files:
                 rapl_file.delta(duration)
 
+            self._confirm_mirrored_files()
+
             for rapl_file in self._rapl_files:
+                if rapl_file.path in self._mirrored_candidates:
+                    # Still suspected of duplicating another counter
+                    continue
                 logger.debug(rapl_file)
                 cpu_details[rapl_file.name] = rapl_file.energy_delta.kWh
                 # We fake the name used by Power Gadget when using RAPL
@@ -874,6 +910,75 @@ class IntelRAPL:
         """
         for rapl_file in self._rapl_files:
             rapl_file.start()
+        # DRAM domains are a different measurement, never a duplicate
+        counters = [
+            (rapl_file.path, float(rapl_file.last_energy))
+            for rapl_file in self._rapl_files
+            if "dram" not in rapl_file.name.lower()
+            and rapl_file.last_energy is not None
+        ]
+        self._mirrored_candidates = find_mirrored_counters(
+            counters, SEQUENTIAL_READ_TOLERANCE_KWH
+        )
+        for path, reference_path in self._mirrored_candidates.items():
+            logger.warning(
+                "\tRAPL - Energy counter at %s looks like a mirror of %s, it is "
+                "left out of the measurement to avoid double-counting",
+                path,
+                reference_path,
+            )
+
+    def _confirm_mirrored_files(self) -> None:
+        """
+        Decide the fate of the files flagged as duplicates at ``start()``.
+
+        Two independent meters could hold indistinguishable counters at the
+        single instant of the initial reading, so a candidate is only dropped
+        for good once it has also accumulated the very same energy over a
+        measurement interval. A candidate whose energy delta differs is a real
+        meter and is restored, and a candidate that has not accumulated
+        anything yet stays pending until an interval is conclusive.
+        """
+        if not self._mirrored_candidates:
+            return
+        files_by_path = {rapl_file.path: rapl_file for rapl_file in self._rapl_files}
+        pending = {}
+        confirmed = set()
+        for path, reference_path in self._mirrored_candidates.items():
+            rapl_file = files_by_path.get(path)
+            reference = files_by_path.get(reference_path)
+            if rapl_file is None or reference is None:
+                continue
+            delta = float(rapl_file.energy_delta)
+            reference_delta = float(reference.energy_delta)
+            if rapl_file.sample_skipped or reference.sample_skipped:
+                # A skipped sample reports no energy: the interval is not conclusive
+                pending[path] = reference_path
+            elif delta <= 0 and reference_delta <= 0:
+                # Nothing was accumulated: the interval is not conclusive
+                pending[path] = reference_path
+            elif counters_match(delta, reference_delta, SEQUENTIAL_READ_TOLERANCE_KWH):
+                confirmed.add(path)
+                logger.warning(
+                    "\tRAPL - Energy counter at %s mirrors the one at %s, it is "
+                    "dropped to avoid double-counting the CPU power (multi-die "
+                    "CPUs mirror the package counter on each die)",
+                    path,
+                    reference_path,
+                )
+            else:
+                logger.info(
+                    "\tRAPL - Energy counter at %s accumulates energy on its own, "
+                    "it is an independent meter and is measured again",
+                    path,
+                )
+        self._mirrored_candidates = pending
+        if confirmed:
+            self._rapl_files = [
+                rapl_file
+                for rapl_file in self._rapl_files
+                if rapl_file.path not in confirmed
+            ]
 
 
 class TDP:
@@ -1009,9 +1114,10 @@ class TDP:
                 )
                 return cpu_model_detected, power
             logger.warning(
-                "We saw that you have a %s but we don't know it."
-                + " Please contact us.",
+                "We saw that you have a %s but we don't know it. "
+                "Please help us add it, see %s#unknown-cpu-model",
                 cpu_model_detected,
+                _TROUBLESHOOTING_URL,
             )
             if is_psutil_available():
                 # Count thread of the CPU