codecarbon 2.8.0__tar.gz → 2.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. {codecarbon-2.8.0 → codecarbon-2.8.2}/PKG-INFO +3 -2
  2. codecarbon-2.8.2/codecarbon/_version.py +1 -0
  3. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/cli/main.py +21 -12
  4. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/api_client.py +8 -9
  5. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/config.py +12 -0
  6. codecarbon-2.8.2/codecarbon/core/resource_tracker.py +147 -0
  7. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/emissions_tracker.py +8 -139
  8. codecarbon-2.8.2/codecarbon/lock.py +68 -0
  9. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/http.py +8 -2
  10. {codecarbon-2.8.0 → codecarbon-2.8.2}/pyproject.toml +1 -1
  11. codecarbon-2.8.0/codecarbon/_version.py +0 -1
  12. codecarbon-2.8.0/codecarbon/lock.py +0 -61
  13. {codecarbon-2.8.0 → codecarbon-2.8.2}/.gitignore +0 -0
  14. {codecarbon-2.8.0 → codecarbon-2.8.2}/LICENSE +0 -0
  15. {codecarbon-2.8.0 → codecarbon-2.8.2}/README.md +0 -0
  16. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/__init__.py +0 -0
  17. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/cli/__init__.py +0 -0
  18. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/cli/cli_utils.py +0 -0
  19. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/__init__.py +0 -0
  20. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/cloud.py +0 -0
  21. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/co2_signal.py +0 -0
  22. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/cpu.py +0 -0
  23. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/emissions.py +0 -0
  24. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/gpu.py +0 -0
  25. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/measure.py +0 -0
  26. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/powermetrics.py +0 -0
  27. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/rapl.py +0 -0
  28. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/schemas.py +0 -0
  29. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/units.py +0 -0
  30. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/core/util.py +0 -0
  31. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/canada_provinces.geojson +0 -0
  32. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/cloud/impact.csv +0 -0
  33. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/hardware/cpu_power.csv +0 -0
  34. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/2016/canada_energy_mix.json +0 -0
  35. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/2016/global_energy_mix-old.json +0 -0
  36. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/2016/usa_emissions.json +0 -0
  37. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/2020/01_get_world_carbon_intensity.ipynb +0 -0
  38. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/2020/02_convert_csv_to_json.ipynb +0 -0
  39. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/2020/03_add_eu_data.ipynb +0 -0
  40. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/2020/eu-carbon-intensity-electricity.csv +0 -0
  41. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/2023-07-07-22-40-48.png +0 -0
  42. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/carbon_intensity_per_source.json +0 -0
  43. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/global_energy_mix.json +0 -0
  44. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/our_world_in_data.ipynb +0 -0
  45. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/data/private_infra/world_energy_mix.csv +0 -0
  46. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/external/__init__.py +0 -0
  47. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/external/geography.py +0 -0
  48. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/external/hardware.py +0 -0
  49. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/external/logger.py +0 -0
  50. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/external/scheduler.py +0 -0
  51. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/external/task.py +0 -0
  52. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/input.py +0 -0
  53. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output.py +0 -0
  54. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/__init__.py +0 -0
  55. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/base_output.py +0 -0
  56. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/emissions_data.py +0 -0
  57. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/file.py +0 -0
  58. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/logger.py +0 -0
  59. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/metrics/__init__.py +0 -0
  60. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/metrics/logfire.py +0 -0
  61. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/metrics/metric_docs.py +0 -0
  62. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/output_methods/metrics/prometheus.py +0 -0
  63. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/__init__.py +0 -0
  64. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/assets/__init__.py +0 -0
  65. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/assets/car_icon.png +0 -0
  66. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/assets/house_icon.png +0 -0
  67. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/assets/tv_icon.png +0 -0
  68. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/carbonboard.py +0 -0
  69. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/carbonboard_on_api.py +0 -0
  70. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/components.py +0 -0
  71. {codecarbon-2.8.0 → codecarbon-2.8.2}/codecarbon/viz/data.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.3
1
+ Metadata-Version: 2.4
2
2
  Name: codecarbon
3
- Version: 2.8.0
3
+ Version: 2.8.2
4
4
  Project-URL: Homepage, https://codecarbon.io/
5
5
  Project-URL: Repository, https://github.com/mlco2/codecarbon
6
6
  Project-URL: Dashboard, http://dashboard.codecarbon.io/
@@ -8,6 +8,7 @@ Project-URL: Documentation, https://mlco2.github.io/codecarbon/
8
8
  Project-URL: Issues, https://github.com/mlco2/codecarbon/issues
9
9
  Project-URL: Changelog, https://github.com/mlco2/codecarbon/releases
10
10
  Author: Mila, DataForGood, BCG GAMMA, Comet.ml, Haverford College
11
+ License-File: LICENSE
11
12
  Classifier: License :: OSI Approved :: MIT License
12
13
  Classifier: Natural Language :: English
13
14
  Classifier: Programming Language :: Python :: 3.7
@@ -0,0 +1 @@
1
+ __version__ = "2.8.2"
@@ -25,12 +25,13 @@ from codecarbon.core.schemas import ExperimentCreate, OrganizationCreate, Projec
25
25
  from codecarbon.emissions_tracker import EmissionsTracker
26
26
 
27
27
  AUTH_CLIENT_ID = os.environ.get(
28
- "AUTH_CLIENT_ID", "pkqh9CiOkp4MkPqRqM_k8Xc3mwBRpojS3RayIk1i5Pg"
28
+ "AUTH_CLIENT_ID",
29
+ "jsUPWIcUECQFE_ouanUuVhXx52TTjEVcVNNtNGeyAtU",
29
30
  )
30
31
  AUTH_SERVER_URL = os.environ.get(
31
- "AUTH_SERVER_URL", "https://auth.codecarbon.io/codecarbon-dev"
32
+ "AUTH_SERVER_URL", "https://auth.codecarbon.io/codecarbon"
32
33
  )
33
- API_URL = os.environ.get("API_URL", "https://dash-dev.cleverapps.io/api")
34
+ API_URL = os.environ.get("API_URL", "https://dashboard.codecarbon.io/api")
34
35
 
35
36
  DEFAULT_PROJECT_ID = "e60afa92-17b7-4720-91a0-1ae91e409ba1"
36
37
  DEFAULT_ORGANIzATION_ID = "e60afa92-17b7-4720-91a0-1ae91e409ba1"
@@ -62,6 +63,7 @@ def show_config(path: Path = Path("./.codecarbon.config")) -> None:
62
63
  d = get_config(path)
63
64
  api_endpoint = get_api_endpoint(path)
64
65
  api = ApiClient(endpoint_url=api_endpoint)
66
+ api.set_access_token(_get_access_token())
65
67
  print("Current configuration : \n")
66
68
  print("Config file content : ")
67
69
  print(d)
@@ -99,7 +101,6 @@ def show_config(path: Path = Path("./.codecarbon.config")) -> None:
99
101
 
100
102
  fief = Fief(AUTH_SERVER_URL, AUTH_CLIENT_ID)
101
103
  fief_auth = FiefAuth(fief, "./credentials.json")
102
- print("FIEF", AUTH_SERVER_URL, AUTH_CLIENT_ID)
103
104
 
104
105
 
105
106
  def _get_access_token():
@@ -126,10 +127,7 @@ def login():
126
127
  fief_auth.authorize()
127
128
 
128
129
 
129
- @codecarbon.command("get-token", short_help="Get project token")
130
- def get_token(project_id: str):
131
- # api = ApiClient(endpoint_url=API_URL) # TODO: get endpoint from config
132
- # api.set_access_token(_get_access_token())
130
+ def get_api_key(project_id: str):
133
131
  req = requests.post(
134
132
  f"{API_URL}/projects/{project_id}/api-tokens",
135
133
  json={
@@ -139,7 +137,16 @@ def get_token(project_id: str):
139
137
  },
140
138
  headers={"Authorization": f"Bearer {_get_access_token()}"},
141
139
  )
142
- print("Your token: " + req.json()["token"])
140
+ api_key = req.json()["token"]
141
+ return api_key
142
+
143
+
144
+ @codecarbon.command("get-token", short_help="Get project token")
145
+ def get_token(project_id: str):
146
+ # api = ApiClient(endpoint_url=API_URL) # TODO: get endpoint from config
147
+ # api.set_access_token(_get_access_token())
148
+ token = get_api_key(project_id)
149
+ print("Your token: " + token)
143
150
  print("Add it to the api_key field in your configuration file")
144
151
 
145
152
 
@@ -290,6 +297,8 @@ def config():
290
297
  experiment = [e for e in experiments if e["name"] == experiment][0]
291
298
 
292
299
  overwrite_local_config("experiment_id", experiment["id"], path=file_path)
300
+ api_key = get_api_key(project_id)
301
+ overwrite_local_config("api_key", api_key, path=file_path)
293
302
  show_config(file_path)
294
303
  print(
295
304
  "Consult [link=https://mlco2.github.io/codecarbon/usage.html#configuration]configuration documentation[/link] for more configuration options"
@@ -316,14 +325,14 @@ def monitor(
316
325
  api (Annotated[bool, typer.Option, optional): Choose to call Code Carbon API or not. Defaults to True.
317
326
  """
318
327
  experiment_id = get_existing_local_exp_id()
319
- if api and experiment_id is None:
320
- print("ERROR: No experiment id, call 'codecarbon init' first.", err=True)
328
+ if api:
329
+ if experiment_id is None:
330
+ print("ERROR: No experiment id, call 'codecarbon config' first.", err=True)
321
331
  print("CodeCarbon is going in an infinite loop to monitor this machine.")
322
332
  with EmissionsTracker(
323
333
  measure_power_secs=measure_power_secs,
324
334
  api_call_interval=api_call_interval,
325
335
  save_to_api=api,
326
- access_token=_get_access_token(),
327
336
  ) as tracker:
328
337
  # Infinite loop
329
338
  while True:
@@ -67,24 +67,23 @@ class ApiClient: # (AsyncClient)
67
67
  stacklevel=2,
68
68
  )
69
69
 
70
- def set_access_token(self, token: str):
71
- """This method sets the access token to be used for the API. For now it is not used.
72
-
73
- Args:
74
- token (str): access token to be used for the API
75
- """
76
- self.access_token = token
77
-
78
70
  def _get_headers(self):
79
71
  headers = {"Content-Type": "application/json"}
80
72
  if self.api_key:
81
73
  print(type(self.api_key))
82
74
  # set the x-api-token header
83
75
  headers["x-api-token"] = self.api_key
84
- if self.access_token:
76
+ elif self.access_token:
85
77
  headers["Authorization"] = f"Bearer {self.access_token}"
86
78
  return headers
87
79
 
80
+ def set_access_token(self, token: str):
81
+ """This method sets the access token to be used for the API.
82
+ Args:
83
+ token (str): access token to be used for the API
84
+ """
85
+ self.access_token = token
86
+
88
87
  def get_list_organizations(self):
89
88
  """
90
89
  List all organizations
@@ -3,6 +3,8 @@ import os
3
3
  from pathlib import Path
4
4
  from typing import List
5
5
 
6
+ from codecarbon.external.logger import logger
7
+
6
8
 
7
9
  def clean_env_key(k: str) -> str:
8
10
  """
@@ -96,6 +98,16 @@ def get_hierarchical_config():
96
98
  home = Path.home()
97
99
  global_path = str((home / ".codecarbon.config").expanduser().resolve())
98
100
  local_path = str((cwd / ".codecarbon.config").expanduser().resolve())
101
+ if Path(global_path).exists():
102
+ logger.info(
103
+ f"Codecarbon is taking the configuration from global file: {global_path}"
104
+ )
105
+ if Path(local_path).exists():
106
+ logger.info(f"Some variables are overriden by the local file: {local_path}")
107
+ elif Path(local_path).exists():
108
+ logger.info(
109
+ f"Codecarbon is taking the configuration from the local file {local_path}"
110
+ )
99
111
 
100
112
  config.read([global_path, local_path])
101
113
  config.read_dict(parse_env_config())
@@ -0,0 +1,147 @@
1
+ from collections import Counter
2
+ from typing import List, Union
3
+
4
+ from codecarbon.core import cpu, gpu, powermetrics
5
+ from codecarbon.core.config import parse_gpu_ids
6
+ from codecarbon.core.util import detect_cpu_model, is_linux_os, is_mac_os, is_windows_os
7
+ from codecarbon.external.hardware import CPU, GPU, RAM, AppleSiliconChip
8
+ from codecarbon.external.logger import logger
9
+
10
+
11
+ class ResourceTracker:
12
+ cpu_tracker = gpu_tracker = ram_tracker = "Unspecified"
13
+
14
+ def __init__(self, tracker):
15
+ self.tracker = tracker
16
+
17
+ def set_RAM_tracking(self):
18
+ logger.info("[setup] RAM Tracking...")
19
+ self.ram_tracker = "3 Watts for 8 GB ratio constant"
20
+ ram = RAM(tracking_mode=self.tracker._tracking_mode)
21
+ self.tracker._conf["ram_total_size"] = ram.machine_memory_GB
22
+ self.tracker._hardware: List[Union[RAM, CPU, GPU, AppleSiliconChip]] = [ram]
23
+
24
+ def set_CPU_tracking(self):
25
+ logger.info("[setup] CPU Tracking...")
26
+ if cpu.is_powergadget_available() and self.tracker._default_cpu_power is None:
27
+ logger.info("Tracking Intel CPU via Power Gadget")
28
+ self.cpu_tracker = "Power Gadget"
29
+ hardware = CPU.from_utils(self.tracker._output_dir, "intel_power_gadget")
30
+ self.tracker._hardware.append(hardware)
31
+ self.tracker._conf["cpu_model"] = hardware.get_model()
32
+ elif cpu.is_rapl_available():
33
+ logger.info("Tracking Intel CPU via RAPL interface")
34
+ self.cpu_tracker = "RAPL"
35
+ hardware = CPU.from_utils(self.tracker._output_dir, "intel_rapl")
36
+ self.tracker._hardware.append(hardware)
37
+ self.tracker._conf["cpu_model"] = hardware.get_model()
38
+ # change code to check if powermetrics needs to be installed or just sudo setup
39
+ elif (
40
+ powermetrics.is_powermetrics_available()
41
+ and self.tracker._default_cpu_power is None
42
+ ):
43
+ logger.info("Tracking Apple CPU and GPU via PowerMetrics")
44
+ self.gpu_tracker = "PowerMetrics"
45
+ self.cpu_tracker = "PowerMetrics"
46
+ hardware_cpu = AppleSiliconChip.from_utils(
47
+ self.tracker._output_dir, chip_part="CPU"
48
+ )
49
+ self.tracker._hardware.append(hardware_cpu)
50
+ self.tracker._conf["cpu_model"] = hardware_cpu.get_model()
51
+
52
+ hardware_gpu = AppleSiliconChip.from_utils(
53
+ self.tracker._output_dir, chip_part="GPU"
54
+ )
55
+ self.tracker._hardware.append(hardware_gpu)
56
+
57
+ self.tracker._conf["gpu_model"] = hardware_gpu.get_model()
58
+ self.tracker._conf["gpu_count"] = 1
59
+ else:
60
+ # Explain what to install to increase accuracy
61
+ cpu_tracking_install_instructions = ""
62
+ if is_mac_os():
63
+ if (
64
+ "M1" in detect_cpu_model()
65
+ or "M2" in detect_cpu_model()
66
+ or "M3" in detect_cpu_model()
67
+ ):
68
+ cpu_tracking_install_instructions = ""
69
+ cpu_tracking_install_instructions = "Mac OS and ARM processor detected: Please enable PowerMetrics sudo to measure CPU"
70
+ else:
71
+ cpu_tracking_install_instructions = "Mac OS detected: Please install Intel Power Gadget or enable PowerMetrics sudo to measure CPU"
72
+ elif is_windows_os():
73
+ cpu_tracking_install_instructions = "Windows OS detected: Please install Intel Power Gadget to measure CPU"
74
+ elif is_linux_os():
75
+ cpu_tracking_install_instructions = "Linux OS detected: Please ensure RAPL files exist at \\sys\\class\\powercap\\intel-rapl to measure CPU"
76
+ logger.warning(
77
+ f"No CPU tracking mode found. Falling back on CPU constant mode. \n {cpu_tracking_install_instructions}\n"
78
+ )
79
+ self.cpu_tracker = "TDP constant"
80
+ tdp = cpu.TDP()
81
+ power = tdp.tdp
82
+ model = tdp.model
83
+ if (power is None) and self.tracker._default_cpu_power:
84
+ # We haven't been able to calculate CPU power but user has input a default one. We use it
85
+ user_input_power = self.tracker._default_cpu_power
86
+ logger.debug(f"Using user input TDP: {user_input_power} W")
87
+ self.cpu_tracker = "User Input TDP constant"
88
+ power = user_input_power
89
+ logger.info(f"CPU Model on constant consumption mode: {model}")
90
+ self.tracker._conf["cpu_model"] = model
91
+ if tdp:
92
+ hardware = CPU.from_utils(
93
+ self.tracker._output_dir, "constant", model, power
94
+ )
95
+ self.tracker._hardware.append(hardware)
96
+ else:
97
+ logger.warning(
98
+ "Failed to match CPU TDP constant. "
99
+ + "Falling back on a global constant."
100
+ )
101
+ self.cpu_tracker = "global constant"
102
+ hardware = CPU.from_utils(self.tracker._output_dir, "constant")
103
+ self.tracker._hardware.append(hardware)
104
+
105
+ def set_GPU_tracking(self):
106
+ logger.info("[setup] GPU Tracking...")
107
+ if self.tracker._gpu_ids:
108
+ # If _gpu_ids is a string or a list of int, parse it to a list of ints
109
+ if isinstance(self.tracker._gpu_ids, str) or (
110
+ isinstance(self.tracker._gpu_ids, list)
111
+ and all(isinstance(gpu_id, int) for gpu_id in self.tracker._gpu_ids)
112
+ ):
113
+ self.tracker._gpu_ids: List[int] = parse_gpu_ids(self.tracker._gpu_ids)
114
+ self.tracker._conf["gpu_ids"] = self.tracker._gpu_ids
115
+ self.tracker._conf["gpu_count"] = len(self.tracker._gpu_ids)
116
+ else:
117
+ logger.warning(
118
+ "Invalid gpu_ids format. Expected a string or a list of ints."
119
+ )
120
+ if gpu.is_gpu_details_available():
121
+ logger.info("Tracking Nvidia GPU via pynvml")
122
+ gpu_devices = GPU.from_utils(self.tracker._gpu_ids)
123
+ self.tracker._hardware.append(gpu_devices)
124
+ gpu_names = [n["name"] for n in gpu_devices.devices.get_gpu_static_info()]
125
+ gpu_names_dict = Counter(gpu_names)
126
+ self.tracker._conf["gpu_model"] = "".join(
127
+ [f"{i} x {name}" for name, i in gpu_names_dict.items()]
128
+ )
129
+ if self.tracker._conf.get("gpu_count") is None:
130
+ self.tracker._conf["gpu_count"] = len(
131
+ gpu_devices.devices.get_gpu_static_info()
132
+ )
133
+ else:
134
+ logger.info("No GPU found.")
135
+
136
+ def set_CPU_GPU_ram_tracking(self):
137
+ self.set_RAM_tracking()
138
+ self.set_CPU_tracking()
139
+ self.set_GPU_tracking()
140
+
141
+ logger.debug(
142
+ f"""The below tracking methods have been set up:
143
+ RAM Tracking Method: {self.ram_tracker}
144
+ CPU Tracking Method: {self.cpu_tracker}
145
+ GPU Tracking Method: {self.gpu_tracker}
146
+ """
147
+ )
@@ -9,24 +9,16 @@ import platform
9
9
  import time
10
10
  import uuid
11
11
  from abc import ABC, abstractmethod
12
- from collections import Counter
13
12
  from datetime import datetime
14
13
  from functools import wraps
15
14
  from typing import Any, Callable, Dict, List, Optional, Union
16
15
 
17
16
  from codecarbon._version import __version__
18
- from codecarbon.core import cpu, gpu, powermetrics
19
- from codecarbon.core.config import get_hierarchical_config, parse_gpu_ids
17
+ from codecarbon.core.config import get_hierarchical_config
20
18
  from codecarbon.core.emissions import Emissions
19
+ from codecarbon.core.resource_tracker import ResourceTracker
21
20
  from codecarbon.core.units import Energy, Power, Time
22
- from codecarbon.core.util import (
23
- count_cpus,
24
- detect_cpu_model,
25
- is_linux_os,
26
- is_mac_os,
27
- is_windows_os,
28
- suppress,
29
- )
21
+ from codecarbon.core.util import count_cpus, suppress
30
22
  from codecarbon.external.geography import CloudMetadata, GeoMetadata
31
23
  from codecarbon.external.hardware import CPU, GPU, RAM, AppleSiliconChip
32
24
  from codecarbon.external.logger import logger, set_logger_format, set_logger_level
@@ -157,7 +149,6 @@ class BaseEmissionsTracker(ABC):
157
149
  api_call_interval: Optional[int] = _sentinel,
158
150
  api_endpoint: Optional[str] = _sentinel,
159
151
  api_key: Optional[str] = _sentinel,
160
- access_token: Optional[str] = _sentinel,
161
152
  output_dir: Optional[str] = _sentinel,
162
153
  output_file: Optional[str] = _sentinel,
163
154
  save_to_file: Optional[bool] = _sentinel,
@@ -312,7 +303,9 @@ class BaseEmissionsTracker(ABC):
312
303
  self._tasks: Dict[str, Task] = {}
313
304
  self._active_task: Optional[str] = None
314
305
 
315
- self.set_CPU_GPU_ram_tracking()
306
+ # Tracking mode detection
307
+ ressource_tracker = ResourceTracker(self)
308
+ ressource_tracker.set_CPU_GPU_ram_tracking()
316
309
 
317
310
  self._conf["hardware"] = list(map(lambda x: x.description(), self._hardware))
318
311
 
@@ -355,132 +348,9 @@ class BaseEmissionsTracker(ABC):
355
348
  self._emissions: Emissions = Emissions(
356
349
  self._data_source, self._co2_signal_api_token
357
350
  )
358
- self._init_output_methods(api_key=self._api_key, access_token=access_token)
359
-
360
- def set_CPU_GPU_ram_tracking(self):
361
- cpu_tracker = gpu_tracker = ram_tracker = "Unspecified"
362
- if self._gpu_ids:
363
- # If _gpu_ids is a string or a list of int, parse it to a list of ints
364
- if isinstance(self._gpu_ids, str) or (
365
- isinstance(self._gpu_ids, list)
366
- and all(isinstance(gpu_id, int) for gpu_id in self._gpu_ids)
367
- ):
368
- self._gpu_ids: List[int] = parse_gpu_ids(self._gpu_ids)
369
- self._conf["gpu_ids"] = self._gpu_ids
370
- self._conf["gpu_count"] = len(self._gpu_ids)
371
- else:
372
- logger.warning(
373
- "Invalid gpu_ids format. Expected a string or a list of ints."
374
- )
375
-
376
- logger.info("[setup] RAM Tracking...")
377
- ram_tracker = "3 Watts for 8 GB ratio constant"
378
- ram = RAM(tracking_mode=self._tracking_mode)
379
- self._conf["ram_total_size"] = ram.machine_memory_GB
380
- self._hardware: List[Union[RAM, CPU, GPU, AppleSiliconChip]] = [ram]
381
-
382
- # Hardware detection
383
- logger.info("[setup] GPU Tracking...")
384
- if gpu.is_gpu_details_available():
385
- logger.info("Tracking Nvidia GPU via pynvml")
386
- gpu_tracker = "pynvml"
387
- gpu_devices = GPU.from_utils(self._gpu_ids)
388
- self._hardware.append(gpu_devices)
389
- gpu_names = [n["name"] for n in gpu_devices.devices.get_gpu_static_info()]
390
- gpu_names_dict = Counter(gpu_names)
391
- self._conf["gpu_model"] = "".join(
392
- [f"{i} x {name}" for name, i in gpu_names_dict.items()]
393
- )
394
- if self._conf.get("gpu_count") is None:
395
- self._conf["gpu_count"] = len(gpu_devices.devices.get_gpu_static_info())
396
- else:
397
- logger.info("No GPU found.")
398
-
399
- logger.info("[setup] CPU Tracking...")
400
- if cpu.is_powergadget_available() and self._default_cpu_power is None:
401
- logger.info("Tracking Intel CPU via Power Gadget")
402
- cpu_tracker = "Power Gadget"
403
- hardware = CPU.from_utils(self._output_dir, "intel_power_gadget")
404
- self._hardware.append(hardware)
405
- self._conf["cpu_model"] = hardware.get_model()
406
- elif cpu.is_rapl_available():
407
- logger.info("Tracking Intel CPU via RAPL interface")
408
- cpu_tracker = "RAPL"
409
- hardware = CPU.from_utils(self._output_dir, "intel_rapl")
410
- self._hardware.append(hardware)
411
- self._conf["cpu_model"] = hardware.get_model()
412
- # change code to check if powermetrics needs to be installed or just sudo setup
413
- elif (
414
- powermetrics.is_powermetrics_available() and self._default_cpu_power is None
415
- ):
416
- logger.info("Tracking Apple CPU and GPU via PowerMetrics")
417
- gpu_tracker = "PowerMetrics"
418
- cpu_tracker = "PowerMetrics"
419
- hardware_cpu = AppleSiliconChip.from_utils(
420
- self._output_dir, chip_part="CPU"
421
- )
422
- self._hardware.append(hardware_cpu)
423
- self._conf["cpu_model"] = hardware_cpu.get_model()
424
-
425
- hardware_gpu = AppleSiliconChip.from_utils(
426
- self._output_dir, chip_part="GPU"
427
- )
428
- self._hardware.append(hardware_gpu)
429
-
430
- self._conf["gpu_model"] = hardware_gpu.get_model()
431
- self._conf["gpu_count"] = 1
432
- else:
433
- # Explain what to install to increase accuracy
434
- cpu_tracking_install_instructions = ""
435
- if is_mac_os():
436
- if (
437
- "M1" in detect_cpu_model()
438
- or "M2" in detect_cpu_model()
439
- or "M3" in detect_cpu_model()
440
- ):
441
- cpu_tracking_install_instructions = ""
442
- cpu_tracking_install_instructions = "Mac OS and ARM processor detected: Please enable PowerMetrics sudo to measure CPU"
443
- else:
444
- cpu_tracking_install_instructions = "Mac OS detected: Please install Intel Power Gadget or enable PowerMetrics sudo to measure CPU"
445
- elif is_windows_os():
446
- cpu_tracking_install_instructions = "Windows OS detected: Please install Intel Power Gadget to measure CPU"
447
- elif is_linux_os():
448
- cpu_tracking_install_instructions = "Linux OS detected: Please ensure RAPL files exist at \\sys\\class\\powercap\\intel-rapl to measure CPU"
449
- logger.warning(
450
- f"No CPU tracking mode found. Falling back on CPU constant mode. \n {cpu_tracking_install_instructions}\n"
451
- )
452
- cpu_tracker = "TDP constant"
453
- tdp = cpu.TDP()
454
- power = tdp.tdp
455
- model = tdp.model
456
- if (power is None) and self._default_cpu_power:
457
- # We haven't been able to calculate CPU power but user has input a default one. We use it
458
- user_input_power = self._default_cpu_power
459
- logger.debug(f"Using user input TDP: {user_input_power} W")
460
- cpu_tracker = "User Input TDP constant"
461
- power = user_input_power
462
- logger.info(f"CPU Model on constant consumption mode: {model}")
463
- self._conf["cpu_model"] = model
464
- if tdp:
465
- hardware = CPU.from_utils(self._output_dir, "constant", model, power)
466
- self._hardware.append(hardware)
467
- else:
468
- logger.warning(
469
- "Failed to match CPU TDP constant. "
470
- + "Falling back on a global constant."
471
- )
472
- cpu_tracker = "global constant"
473
- hardware = CPU.from_utils(self._output_dir, "constant")
474
- self._hardware.append(hardware)
475
- logger.debug(
476
- f"""The below tracking methods have been set up:
477
- RAM Tracking Method: {ram_tracker}
478
- CPU Tracking Method: {cpu_tracker}
479
- GPU Tracking Method: {gpu_tracker}
480
- """
481
- )
351
+ self._init_output_methods(api_key=self._api_key)
482
352
 
483
- def _init_output_methods(self, *, api_key: str = None, access_token: str = None):
353
+ def _init_output_methods(self, *, api_key: str = None):
484
354
  """
485
355
  Prepare the different output methods
486
356
  """
@@ -504,7 +374,6 @@ class BaseEmissionsTracker(ABC):
504
374
  endpoint_url=self._api_endpoint,
505
375
  experiment_id=self._experiment_id,
506
376
  api_key=api_key,
507
- access_token=access_token,
508
377
  conf=self._conf,
509
378
  )
510
379
  self.run_id = cc_api__out.run_id
@@ -0,0 +1,68 @@
1
+ """
2
+ Ensures that only one instance of codecarbon is running at a time.
3
+ It creates a lock file in /tmp/.codecarbon.lock and removes it on exit.
4
+ If the lock file already exists, it exits the program.
5
+ """
6
+
7
+ import atexit
8
+ import errno
9
+ import os
10
+ import signal
11
+ import tempfile
12
+ import threading
13
+
14
+ from codecarbon.external.logger import logger
15
+
16
+ # We use tempfile.gettempdir() to get the system's temporary directory (linux: /tmp, windows: C:\Users\username\AppData\Local\Temp)
17
+ LOCKFILE = os.path.join(tempfile.gettempdir(), ".codecarbon.lock")
18
+
19
+
20
+ class Lock:
21
+ """A lock to ensure only one instance of codecarbon is running."""
22
+
23
+ def __init__(self):
24
+ self._has_created_lock = False
25
+ self.lockfile_path = LOCKFILE
26
+ atexit.register(
27
+ self.release
28
+ ) # Ensure release() is called on unexpected exit of the user's python code
29
+ # If there is more than one thread add a lock
30
+ self._thread_lock = threading.Lock()
31
+ # If the current thread is the main thread, register signal handlers
32
+ if threading.current_thread() is threading.main_thread():
33
+ # Register signal handlers to ensure lock release on interruption
34
+ signal.signal(signal.SIGINT, self._handle_exit) # Ctrl+C
35
+ signal.signal(signal.SIGTERM, self._handle_exit) # Termination signal
36
+
37
+ def _handle_exit(self, signum, frame):
38
+ """Ensures the lock file is removed when the script is interrupted."""
39
+ logger.debug(f"Signal {signum} received. Releasing lock and exiting.")
40
+ self.release()
41
+ os._exit(1) # Exit immediately to prevent further execution
42
+
43
+ def acquire(self):
44
+ """Creates a lock file and ensures it's the only instance running."""
45
+ with self._thread_lock:
46
+ # Attempt to create the lock file
47
+ try:
48
+ with open(LOCKFILE, "x") as _:
49
+ logger.debug(f"Lock file created. Path: {LOCKFILE}")
50
+ self._has_created_lock = True
51
+ except FileExistsError:
52
+ logger.debug(
53
+ f"Lock file {LOCKFILE} already exists. This usually means another instance of codecarbon is running. You can safely delete it if you want or use allow_multiple_runs parameter to always bypass it."
54
+ )
55
+ raise
56
+
57
+ def release(self):
58
+ """Removes the lock file on exit."""
59
+ with self._thread_lock:
60
+ logger.debug("Removing the lock")
61
+ try:
62
+ # Remove the lock file only if it was created by this instance
63
+ if self._has_created_lock:
64
+ os.remove(LOCKFILE)
65
+ except OSError as e:
66
+ logger.debug(f"Error: {e}")
67
+ if e.errno != errno.ENOENT:
68
+ raise
@@ -46,19 +46,25 @@ class CodeCarbonAPIOutput(BaseOutput):
46
46
  experiment_id: str,
47
47
  api_key: str,
48
48
  conf,
49
- access_token: str = None,
50
49
  ):
51
50
  self.endpoint_url: str = endpoint_url
52
51
  self.api = ApiClient(
53
52
  experiment_id=experiment_id,
54
53
  endpoint_url=endpoint_url,
55
54
  api_key=api_key,
56
- access_token=access_token,
57
55
  conf=conf,
58
56
  )
59
57
  self.run_id = self.api.run_id
60
58
 
59
+ def live_out(self, total: EmissionsData, delta: EmissionsData):
60
+ # Called at regular intervals
61
+ try:
62
+ self.api.add_emission(dataclasses.asdict(delta))
63
+ except Exception as e:
64
+ logger.error(e, exc_info=True)
65
+
61
66
  def out(self, total: EmissionsData, delta: EmissionsData):
67
+ # Called on exit
62
68
  try:
63
69
  self.api.add_emission(dataclasses.asdict(delta))
64
70
  except Exception as e:
@@ -183,7 +183,7 @@ include = [
183
183
  ]
184
184
 
185
185
  [tool.bumpver]
186
- current_version = "2.8.0"
186
+ current_version = "2.8.2"
187
187
  version_pattern = "MAJOR.MINOR.PATCH"
188
188
 
189
189
  [tool.bumpver.file_patterns]
@@ -1 +0,0 @@
1
- __version__ = "2.8.0"
@@ -1,61 +0,0 @@
1
- """
2
- Ensures that only one instance of codecarbon is running at a time.
3
- It creates a lock file in /tmp/.codecarbon.lock and removes it on exit.
4
- If the lock file already exists, it exits the program.
5
- """
6
-
7
- import atexit
8
- import errno
9
- import os
10
- import signal
11
- import tempfile
12
-
13
- from codecarbon.external.logger import logger
14
-
15
- # We use tempfile.gettempdir() to get the system's temporary directory (linux: /tmp, windows: C:\Users\username\AppData\Local\Temp)
16
- LOCKFILE = os.path.join(tempfile.gettempdir(), ".codecarbon.lock")
17
-
18
-
19
- class Lock:
20
- """A lock to ensure only one instance of codecarbon is running."""
21
-
22
- def __init__(self):
23
- self._has_created_lock = False
24
- self.lockfile_path = LOCKFILE
25
- atexit.register(
26
- self.release
27
- ) # Ensure release() is called on unexpected exit of the user's python code
28
- # Register signal handlers to ensure lock release on interruption
29
- signal.signal(signal.SIGINT, self._handle_exit) # Ctrl+C
30
- signal.signal(signal.SIGTERM, self._handle_exit) # Termination signal
31
-
32
- def _handle_exit(self, signum, frame):
33
- """Ensures the lock file is removed when the script is interrupted."""
34
- logger.debug(f"Signal {signum} received. Releasing lock and exiting.")
35
- self.release()
36
- os._exit(1) # Exit immediately to prevent further execution
37
-
38
- def acquire(self):
39
- """Creates a lock file and ensures it's the only instance running."""
40
- # Attempt to create the lock file
41
- try:
42
- with open(LOCKFILE, "x") as _:
43
- logger.debug(f"Lock file created. Path: {LOCKFILE}")
44
- self._has_created_lock = True
45
- except FileExistsError:
46
- logger.debug(
47
- f"Lock file {LOCKFILE} already exists. This usually means another instance of codecarbon is running. You can safely delete it if you want or use allow_multiple_runs parameter to always bypass it."
48
- )
49
- raise
50
-
51
- def release(self):
52
- """Removes the lock file on exit."""
53
- logger.debug("Removing the lock")
54
- try:
55
- # Remove the lock file only if it was created by this instance
56
- if self._has_created_lock:
57
- os.remove(LOCKFILE)
58
- except OSError as e:
59
- logger.debug(f"Error: {e}")
60
- if e.errno != errno.ENOENT:
61
- raise
File without changes
File without changes
File without changes