ml-goodput-measurement 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,198 @@
1
+ Metadata-Version: 2.1
2
+ Name: ml_goodput_measurement
3
+ Version: 0.0.1
4
+ Summary: Package to retrieve Goodput of jobs running on Cloud TPU.
5
+ Keywords:
6
+ Author-email: Cloud TPU Team <cloud-tpu-eng@google.com>
7
+ Requires-Python: >=3.8
8
+ Description-Content-Type: text/markdown
9
+ Classifier: Programming Language :: Python :: 3.8
10
+ Classifier: Programming Language :: Python :: 3.9
11
+ Classifier: Programming Language :: Python :: 3.10
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Requires-Dist: google-cloud-logging>=3.5.0
14
+ Project-URL: Bug Tracker, https://github.com/google/cloud_tpu_goodput/issues
15
+ Project-URL: Homepage, https://github.com/google/cloud_tpu_goodput
16
+
17
+ <!--
18
+ Copyright 2023 Google LLC
19
+
20
+ Licensed under the Apache License, Version 2.0 (the "License");
21
+ you may not use this file except in compliance with the License.
22
+ You may obtain a copy of the License at
23
+
24
+ https://www.apache.org/licenses/LICENSE-2.0
25
+
26
+ Unless required by applicable law or agreed to in writing, software
27
+ distributed under the License is distributed on an "AS IS" BASIS,
28
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
29
+ See the License for the specific language governing permissions and
30
+ limitations under the License.
31
+ -->
32
+ # ML Goodput Measurement
33
+
34
+ ## Overview
35
+
36
+ ML Goodput Measurement is a library intended to be used with Cloud TPU to log the
37
+ necessary information and query a job's Goodput. It can be pip installed to
38
+ import its modules, and retrieve information about a training job's overall
39
+ productive Goodput. The package exposes API interfaces to log useful
40
+ information from the user application and query Goodput for the job run, gain
41
+ insight into the productivity of ML workloads and utilization of compute
42
+ resources.
43
+
44
+ ## Components
45
+
46
+
47
+ The ML Goodput Measurement library consists of two main components:
48
+ the `GoodputRecorder` and the `GoodputCalculator`. The `GoodputRecorder`
49
+ exposes APIs to the client to export key timestamps while a training job makes
50
+ progress, namely APIs that allow logging of productive step time and total job
51
+ run time. The library will serialize and store this data in Google Cloud
52
+ Logging. The `GoodputCalculator` exposes APIs to compute Goodput based on the
53
+ recorded data. Cloud Logging handles its internal operations asynchronously.
54
+ The recommended way to compute Goodput is to run an analysis program separate
55
+ from the training application, either on a CPU instance or on the users'
56
+ development machine.
57
+
58
+ ## Installation
59
+
60
+ To install the ML Goodput Measurement package, run the following command on TPU VM:
61
+
62
+ ```bash
63
+ pip install ml-goodput-measurement
64
+ ```
65
+
66
+ ## Usage
67
+
68
+ The usage of this package requires the setup of a Google Cloud project with
69
+ billing enabled to properly use Google Cloud Logging. If you don't have a Google
70
+ Cloud project, or if you don't have billing enabled for your Google Cloud
71
+ project, then do the following:
72
+
73
+ 1. In the Google Cloud console, on the project selector page,
74
+ [select or create a Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects).
75
+
76
+ 2. Make sure that billing is enabled for your Google Cloud project. Instructions can be found [here](https://cloud.google.com/billing/docs/how-to/verify-billing-enabled#console)
77
+
78
+ To run your training on Cloud TPU, set up the Cloud TPU environment by following
79
+ instructions [here](https://cloud.google.com/tpu/docs/setup-gcp-account).
80
+
81
+ To learn more about Google Cloud Logging, visit this [page](https://cloud.google.com/logging/docs).
82
+
83
+
84
+ ### Import
85
+
86
+ To use this package, import the `goodput` module:
87
+
88
+
89
+ ```python
90
+ from ml_goodput_measurement import goodput
91
+ ```
92
+
93
+ ### Define the name of the Google Cloud Logging logger bucket
94
+
95
+ Create a run-specific logger bucket where Cloud Logging entries can be written and read from.
96
+
97
+ For example:
98
+
99
+ ```python
100
+ goodput_logger_name = f'goodput_{config.run_name}'
101
+ ```
102
+
103
+ ### Create a `GoodputRecorder` object
104
+
105
+ Next, create a recorder object with the following parameters:
106
+
107
+ 1. `job_name`: The full run name of the job.
108
+ 2. `logger_name`: The name of the Cloud Logging logger object (created in the previous step).
109
+ 3. `logging_enabled`: Whether or not this process has Cloud Logging enabled.
110
+
111
+
112
+
113
+ > **_NOTE:_** For a multi-worker setup, please ensure that only one worker
114
+ writes the logs to avoid the duplication. In JAX, for example, the check
115
+ could be `if jax.process_index() == 0`
116
+
117
+
118
+ > **_NOTE:_** `logging_enabled` defaults to `False` and Goodput computations cannot be completed if no logs are ever written.
119
+
120
+ For example:
121
+
122
+
123
+ ```python
124
+ goodput_recorder = goodput.GoodputRecorder(job_name=config.run_name, logger_name=goodput_logger_name, logging_enabled=(jax.process_index() == 0))
125
+ ```
126
+
127
+
128
+ ### Record Data with `GoodputRecorder`
129
+
130
+ #### Record Job Start and End Time
131
+
132
+ Use the recorder object to record the job's overall start and end time.
133
+
134
+ For example:
135
+
136
+ ```python
137
+ def main(argv: Sequence[str]) -> None:
138
+ # Initialize configs…
139
+ goodput_recorder.record_job_start_time(datetime.datetime.now())
140
+ # TPU Initialization and device scanning…
141
+ # Set up other things for the main training loop…
142
+ # Main training loop
143
+ train_loop(config)
144
+ goodput_recorder.record_job_end_time(datetime.datetime.now())
145
+ ```
146
+
147
+
148
+ #### Record Step Time
149
+
150
+ Use the recorder object to record a step's start time using `record_step_start_time(step_count)`:
151
+
152
+ For example:
153
+
154
+ ```python
155
+ def train_loop(config, state=None):
156
+ # Set up mesh, model, state, checkpoint manager…
157
+
158
+ # Initialize functional train arguments and model parameters…
159
+
160
+ # Define the compilation
161
+
162
+ for step in np.arange(start_step, config.steps):
163
+ goodput_recorder.record_step_start_time(step)
164
+ # Training step…
165
+
166
+ return state
167
+ ```
168
+
169
+ ### Retrieve Goodput with `GoodputCalculator`
170
+
171
+ In order to retrieve the Goodput of a job run, all you need to do is instantiate
172
+ a `GoodputCalculator` object with the job's run name and the Cloud Logging
173
+ logger name used to record data for that job run. Then call the `get_job_goodput`
174
+ API to get the computed Goodput for the job run.
175
+
176
+ It is recommended to make the `get_job_goodput` calls for a job run from an
177
+ instance that runs elsewhere from your training machine.
178
+
179
+
180
+ #### Create a `GoodputCalculator` object
181
+
182
+ Create the calculator object:
183
+
184
+ ```python
185
+ goodput_logger_name = f'goodput_{config.run_name}' # You can choose your own logger name.
186
+ goodput_calculator = goodput.GoodputCalculator(job_name=config.run_name, logger_name=goodput_logger_name)
187
+ ```
188
+
189
+ #### Retrieve Goodput
190
+
191
+ Finally, call the `get_job_goodput` API to retrieve Goodput for the entire job run.
192
+
193
+ ```python
194
+ total_goodput = goodput_calculator.get_job_goodput()
195
+ print(f"Total job goodput: {total_goodput:.2f}%")
196
+ ```
197
+
198
+
@@ -0,0 +1,181 @@
1
+ <!--
2
+ Copyright 2023 Google LLC
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License");
5
+ you may not use this file except in compliance with the License.
6
+ You may obtain a copy of the License at
7
+
8
+ https://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ See the License for the specific language governing permissions and
14
+ limitations under the License.
15
+ -->
16
+ # ML Goodput Measurement
17
+
18
+ ## Overview
19
+
20
+ ML Goodput Measurement is a library intended to be used with Cloud TPU to log the
21
+ necessary information and query a job's Goodput. It can be pip installed to
22
+ import its modules, and retrieve information about a training job's overall
23
+ productive Goodput. The package exposes API interfaces to log useful
24
+ information from the user application and query Goodput for the job run, gain
25
+ insight into the productivity of ML workloads and utilization of compute
26
+ resources.
27
+
28
+ ## Components
29
+
30
+
31
+ The ML Goodput Measurement library consists of two main components:
32
+ the `GoodputRecorder` and the `GoodputCalculator`. The `GoodputRecorder`
33
+ exposes APIs to the client to export key timestamps while a training job makes
34
+ progress, namely APIs that allow logging of productive step time and total job
35
+ run time. The library will serialize and store this data in Google Cloud
36
+ Logging. The `GoodputCalculator` exposes APIs to compute Goodput based on the
37
+ recorded data. Cloud Logging handles its internal operations asynchronously.
38
+ The recommended way to compute Goodput is to run an analysis program separate
39
+ from the training application, either on a CPU instance or on the users'
40
+ development machine.
41
+
42
+ ## Installation
43
+
44
+ To install the ML Goodput Measurement package, run the following command on TPU VM:
45
+
46
+ ```bash
47
+ pip install ml-goodput-measurement
48
+ ```
49
+
50
+ ## Usage
51
+
52
+ The usage of this package requires the setup of a Google Cloud project with
53
+ billing enabled to properly use Google Cloud Logging. If you don't have a Google
54
+ Cloud project, or if you don't have billing enabled for your Google Cloud
55
+ project, then do the following:
56
+
57
+ 1. In the Google Cloud console, on the project selector page,
58
+ [select or create a Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects).
59
+
60
+ 2. Make sure that billing is enabled for your Google Cloud project. Instructions can be found [here](https://cloud.google.com/billing/docs/how-to/verify-billing-enabled#console)
61
+
62
+ To run your training on Cloud TPU, set up the Cloud TPU environment by following
63
+ instructions [here](https://cloud.google.com/tpu/docs/setup-gcp-account).
64
+
65
+ To learn more about Google Cloud Logging, visit this [page](https://cloud.google.com/logging/docs).
66
+
67
+
68
+ ### Import
69
+
70
+ To use this package, import the `goodput` module:
71
+
72
+
73
+ ```python
74
+ from ml_goodput_measurement import goodput
75
+ ```
76
+
77
+ ### Define the name of the Google Cloud Logging logger bucket
78
+
79
+ Create a run-specific logger bucket where Cloud Logging entries can be written and read from.
80
+
81
+ For example:
82
+
83
+ ```python
84
+ goodput_logger_name = f'goodput_{config.run_name}'
85
+ ```
86
+
87
+ ### Create a `GoodputRecorder` object
88
+
89
+ Next, create a recorder object with the following parameters:
90
+
91
+ 1. `job_name`: The full run name of the job.
92
+ 2. `logger_name`: The name of the Cloud Logging logger object (created in the previous step).
93
+ 3. `logging_enabled`: Whether or not this process has Cloud Logging enabled.
94
+
95
+
96
+
97
+ > **_NOTE:_** For a multi-worker setup, please ensure that only one worker
98
+ writes the logs to avoid the duplication. In JAX, for example, the check
99
+ could be `if jax.process_index() == 0`
100
+
101
+
102
+ > **_NOTE:_** `logging_enabled` defaults to `False` and Goodput computations cannot be completed if no logs are ever written.
103
+
104
+ For example:
105
+
106
+
107
+ ```python
108
+ goodput_recorder = goodput.GoodputRecorder(job_name=config.run_name, logger_name=goodput_logger_name, logging_enabled=(jax.process_index() == 0))
109
+ ```
110
+
111
+
112
+ ### Record Data with `GoodputRecorder`
113
+
114
+ #### Record Job Start and End Time
115
+
116
+ Use the recorder object to record the job's overall start and end time.
117
+
118
+ For example:
119
+
120
+ ```python
121
+ def main(argv: Sequence[str]) -> None:
122
+ # Initialize configs…
123
+ goodput_recorder.record_job_start_time(datetime.datetime.now())
124
+ # TPU Initialization and device scanning…
125
+ # Set up other things for the main training loop…
126
+ # Main training loop
127
+ train_loop(config)
128
+ goodput_recorder.record_job_end_time(datetime.datetime.now())
129
+ ```
130
+
131
+
132
+ #### Record Step Time
133
+
134
+ Use the recorder object to record a step's start time using `record_step_start_time(step_count)`:
135
+
136
+ For example:
137
+
138
+ ```python
139
+ def train_loop(config, state=None):
140
+ # Set up mesh, model, state, checkpoint manager…
141
+
142
+ # Initialize functional train arguments and model parameters…
143
+
144
+ # Define the compilation
145
+
146
+ for step in np.arange(start_step, config.steps):
147
+ goodput_recorder.record_step_start_time(step)
148
+ # Training step…
149
+
150
+ return state
151
+ ```
152
+
153
+ ### Retrieve Goodput with `GoodputCalculator`
154
+
155
+ In order to retrieve the Goodput of a job run, all you need to do is instantiate
156
+ a `GoodputCalculator` object with the job's run name and the Cloud Logging
157
+ logger name used to record data for that job run. Then call the `get_job_goodput`
158
+ API to get the computed Goodput for the job run.
159
+
160
+ It is recommended to make the `get_job_goodput` calls for a job run from an
161
+ instance that runs elsewhere from your training machine.
162
+
163
+
164
+ #### Create a `GoodputCalculator` object
165
+
166
+ Create the calculator object:
167
+
168
+ ```python
169
+ goodput_logger_name = f'goodput_{config.run_name}' # You can choose your own logger name.
170
+ goodput_calculator = goodput.GoodputCalculator(job_name=config.run_name, logger_name=goodput_logger_name)
171
+ ```
172
+
173
+ #### Retrieve Goodput
174
+
175
+ Finally, call the `get_job_goodput` API to retrieve Goodput for the entire job run.
176
+
177
+ ```python
178
+ total_goodput = goodput_calculator.get_job_goodput()
179
+ print(f"Total job goodput: {total_goodput:.2f}%")
180
+ ```
181
+
@@ -0,0 +1,16 @@
1
+
2
+ # Copyright 2024 Google LLC
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # https://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from cloud_tpu_goodput.ml_goodput_measurement.src import goodput
@@ -0,0 +1,405 @@
1
+ """Goodput package API implementations.
2
+
3
+ This file contains all the methods exposed through the cloud_tpu_goodput library
4
+ for users to log necessary information to compute Goodput, and to query the
5
+ computed Goodput.
6
+ """
7
+
8
+ import datetime
9
+ import logging
10
+ from typing import Any, Optional
11
+ import google.cloud.logging
12
+
13
+ _JOB_NAME = 'job_name'
14
+ _STEP_COUNT = 'step_count'
15
+ _STEP_START_TIME = 'step_start_time'
16
+ _JOB_START_TIME = 'job_start_time'
17
+ _JOB_END_TIME = 'job_end_time'
18
+
19
+
20
+ class _CloudLogger:
21
+ """A helper class for reading and writing to Cloud Logging.
22
+
23
+ Attributes:
24
+ job_name: Name of a specific job.
25
+ logger: The Cloud Logging logger object.
26
+ """
27
+
28
+ def __init__(self, job_name: str, log_name: str):
29
+ """_CloudLogger constructor.
30
+
31
+ Args:
32
+ job_name: Name of the job the _CloudLogger is for.
33
+ log_name: Name of the log being written.
34
+ """
35
+ self.job_name = job_name
36
+ logging_client = google.cloud.logging.Client()
37
+ self.logger = logging_client.logger(log_name)
38
+
39
+ def _filter_entries_for_job(self, entries) -> list[Any]:
40
+ """Filters entries for the specific job.
41
+
42
+ Args:
43
+ entries: Cloud Logging entries from user-specified logger.
44
+
45
+ Returns:
46
+ Filtered entries for the specific job.
47
+ """
48
+ filtered_entries = []
49
+ for entry in entries:
50
+ payload = entry.payload
51
+ if _JOB_NAME not in payload:
52
+ continue
53
+ if payload[_JOB_NAME] != self.job_name:
54
+ continue
55
+ filtered_entries.append(payload)
56
+
57
+ return filtered_entries
58
+
59
+ def write_cloud_logging_entry(self, entry) -> None:
60
+ """Writes an entry to the Cloud Logging logger at INFO level.
61
+
62
+ Args:
63
+ entry: JSON-serializable structured log dictionary.
64
+ """
65
+ if entry is None:
66
+ return
67
+ if entry[_JOB_NAME] == self.job_name:
68
+ self.logger.log_struct(
69
+ entry,
70
+ severity='INFO',
71
+ )
72
+
73
+ def read_cloud_logging_entries(self):
74
+ return self._filter_entries_for_job(
75
+ self.logger.list_entries(order_by='timestamp asc')
76
+ )
77
+
78
+
79
+ class GoodputRecorder:
80
+ """The Goodput recorder class, responsible for recording Goodput metrics from the user application.
81
+
82
+ Attributes:
83
+ job_name: Name of the job the GoodputRecorder is for.
84
+ """
85
+
86
+ def __init__(
87
+ self,
88
+ job_name: str,
89
+ logger_name: str,
90
+ logging_enabled=False,
91
+ logger: Optional[_CloudLogger] = None,
92
+ ):
93
+ """GoodputRecorder constructor.
94
+
95
+ Args:
96
+ job_name: Name of the job the GoodputRecorder is for.
97
+ logger_name: The name of the Cloud Logging logger object that the
98
+ application wants logs to be written to and read from.
99
+ logging_enabled: A boolean value to indicate whether the current process
100
+ should send logs to Cloud Logging or not. The application should set
101
+ this value to True if the Recorder is being called from TPU worker 0 and
102
+ the application's configurations request Goodput logging.
103
+ logger: Should never be passed directly by the user.
104
+ """
105
+ self.job_name = job_name
106
+ # If logging is disabled for this process, do not create a _cloud_logger
107
+ # object and exit early if any record record_* API is called.
108
+ if not logging_enabled:
109
+ self._cloud_logger = None
110
+ logging.info('Logging is disabled for this process.')
111
+ return
112
+
113
+ if logger is not None:
114
+ self._cloud_logger = logger
115
+ else:
116
+ self._cloud_logger = _CloudLogger(job_name, logger_name)
117
+
118
+ def record_step_start_time(
119
+ self, step: int, start_time: Optional[datetime.datetime] = None
120
+ ):
121
+ """Main recorder function to log an individual step's start time.
122
+
123
+ Args:
124
+ step: The count of the step that timing information is recorded for.
125
+ start_time: Optional backfill start time of the training step. If
126
+ provided, it has to be in UTC time.
127
+ """
128
+ if self._cloud_logger is None:
129
+ return
130
+ if start_time is None:
131
+ start_time = datetime.datetime.utcnow()
132
+
133
+ self._cloud_logger.write_cloud_logging_entry({
134
+ _JOB_NAME: self.job_name,
135
+ _STEP_COUNT: int(step),
136
+ _STEP_START_TIME: start_time.timestamp(),
137
+ })
138
+
139
+ def record_checkpoint_progress(self, step, checkpoint_start_time):
140
+ """Main recorder function to log information on a successful checkpoint.
141
+
142
+ This method is intended to log the progress for a checkpoint (last step
143
+ included in the checkpoint) and when the checkpoint starts. This information
144
+ will be retrieved in the future to determine whether training progress from
145
+ a completed step contributes to Goodput or wasted progress Badput.
146
+
147
+ Args:
148
+ step: The step count of the last step included in the saved checkpoint.
149
+ checkpoint_start_time: Timestamp at which the checkpoint containing
150
+ progress upto "step" starts to save.
151
+ """
152
+ pass
153
+
154
+ def record_job_start_time(
155
+ self, start_time: Optional[datetime.datetime] = None
156
+ ):
157
+ """Main recorder function to log a job's start time.
158
+
159
+ Args:
160
+ start_time: Optional backfill start time of the job. If provided, it has
161
+ to be in UTC time.
162
+ """
163
+ if self._cloud_logger is None:
164
+ return
165
+ if start_time is None:
166
+ start_time = datetime.datetime.utcnow()
167
+
168
+ self._cloud_logger.write_cloud_logging_entry({
169
+ _JOB_NAME: self.job_name,
170
+ _JOB_START_TIME: start_time.timestamp(),
171
+ })
172
+
173
+ def record_job_end_time(self, end_time: Optional[datetime.datetime] = None):
174
+ """Main recorder function to log a job's end time.
175
+
176
+ Args:
177
+ end_time: Optional backfull end time of the job. If provided, it has to be
178
+ in UTC time.
179
+ """
180
+ if self._cloud_logger is None:
181
+ return
182
+ if end_time is None:
183
+ end_time = datetime.datetime.utcnow()
184
+
185
+ self._cloud_logger.write_cloud_logging_entry({
186
+ _JOB_NAME: self.job_name,
187
+ _JOB_END_TIME: end_time.timestamp(),
188
+ })
189
+
190
+ def record_tpu_init_start_time(self, start_time):
191
+ """Main recorder function to log the start time for TPU initialization.
192
+
193
+ Note: TPU initialization may include the time spent in completing
194
+ jax.devices() which is responsible for device scanning and Slice Builder
195
+ initialization.
196
+
197
+ Args:
198
+ start_time: Start time of TPU initialization.
199
+ """
200
+ pass
201
+
202
+ def record_tpu_init_end_time(self, end_time):
203
+ """Main recorder function to log the end time for TPU initialization.
204
+
205
+ Args:
206
+ end_time: End time of TPU initialization.
207
+ """
208
+ pass
209
+
210
+ def record_training_preparation_start_time(self, start_time):
211
+ """Main recorder function to log the start time of training preparation before starting a training loop.
212
+
213
+ Note: Training preparation may include the time spent in creation of
214
+ checkpoint managers, checkpoint loading, running mesh and model optimizers
215
+ etc.
216
+
217
+ Args:
218
+ start_time: Start time of training preparation.
219
+ """
220
+ pass
221
+
222
+ def record_training_preparation_end_time(self, end_time):
223
+ """Main recorder function to log the end time of training preparation before starting a training loop.
224
+
225
+ Args:
226
+ end_time: End time of training preparation.
227
+ """
228
+ pass
229
+
230
+ def record_data_loading_start_time(self, start_time):
231
+ """Main recorder function to log the start time of training's data loading.
232
+
233
+ Args:
234
+ start_time: Start time of data loading.
235
+ """
236
+ pass
237
+
238
+ def record_data_loading_end_time(self, end_time):
239
+ """Main recorder function to log the end time of training's data loading.
240
+
241
+ Args:
242
+ end_time: End time of data loading.
243
+ """
244
+ pass
245
+
246
+
247
+ class GoodputCalculator:
248
+ """The Goodput calculator class, responsible for querying necessary information and computing Goodput metrics to return to the user application.
249
+
250
+ Attributes:
251
+ job_name: Name of the job the GoodputCalculator is for.
252
+ """
253
+
254
+ def __init__(
255
+ self,
256
+ job_name: str,
257
+ logger_name: str,
258
+ logger: Optional[_CloudLogger] = None,
259
+ ):
260
+ """GoodputCalculator constructor.
261
+
262
+ Args:
263
+ job_name: Name of the job the GoodputCalculator is for.
264
+ logger_name: Name of the log being written.
265
+ logger: Should never be passed directly by the user.
266
+ """
267
+ self.job_name = job_name
268
+ if logger is not None:
269
+ self._cloud_logger = logger
270
+ else:
271
+ self._cloud_logger = _CloudLogger(job_name, logger_name)
272
+
273
+ def _get_total_productive_training_time(self, entries: list[Any]) -> float:
274
+ """Helper function to compute the total productive training time.
275
+
276
+ Args:
277
+ entries: Cloud Logging entries from user-specified logger for a specific
278
+ job.
279
+
280
+ Returns:
281
+ The job's total productive training time.
282
+
283
+ """
284
+ # Build a deserialized dictionary from cloud logging entries to store step
285
+ # start times. The dictionary maps from step count to start time and will be
286
+ # used to each step's productive time by looking for its completion in the
287
+ # next step's start.
288
+ # Note in the instance where progress is lost due to a disruption and the
289
+ # last successful checkpoint did not include all the steps, the last set of
290
+ # records of the step information will be kept and the previous set will be
291
+ # overwritten by design so as to correct for the the previously computed
292
+ # additional time that was counted as productive but lost due to a
293
+ # disruption.
294
+ step_start_data = {}
295
+ curr_last_step = -1
296
+ job_end_time = None
297
+ for payload in entries:
298
+ if _STEP_START_TIME in payload:
299
+ curr_step = payload[_STEP_COUNT]
300
+ step_start_data[curr_step] = payload[_STEP_START_TIME]
301
+ for step in range(curr_step + 1, curr_last_step + 1):
302
+ if step in step_start_data:
303
+ step_start_data.pop(step)
304
+ curr_last_step = curr_step
305
+
306
+ if _JOB_END_TIME in payload:
307
+ # Locate the last instance of job's end time if the job has completed.
308
+ job_end_time = payload[_JOB_END_TIME]
309
+
310
+ # Largest step count that has been recorded. For the last step, we check if
311
+ # the step completed by looking for the corresponding run's end time, since
312
+ # there are are no more steps to check for its completion or compute step
313
+ # time.
314
+ productive_training_time = 0.0
315
+
316
+ for step_count, step_start_time in step_start_data.items():
317
+ if step_count < curr_last_step:
318
+ productive_training_time += (
319
+ step_start_data[step_count + 1] - step_start_time
320
+ )
321
+ elif job_end_time is not None:
322
+ productive_training_time += job_end_time - step_start_time
323
+ else:
324
+ productive_training_time += (
325
+ datetime.datetime.utcnow().timestamp() - step_start_time
326
+ )
327
+
328
+ return productive_training_time
329
+
330
+ def _get_total_job_time(self, entries: list[Any]) -> float:
331
+ """Helper function to compute the total job runtime.
332
+
333
+ Args:
334
+ entries: Cloud Logging entries from user-specified logger for a specific
335
+ job.
336
+
337
+ Returns:
338
+ The job's total runtime.
339
+ """
340
+ # De-serealize job start and end times from cloud logging entries. These
341
+ # will be used to compute total runtime of the job.
342
+ job_start_time = None
343
+ job_end_time = None
344
+ for payload in entries:
345
+ # Locate the earliest timestamp recorded for the job's start.
346
+ if _JOB_START_TIME in payload and job_start_time is None:
347
+ job_start_time = payload[_JOB_START_TIME]
348
+ # Locate the latest timestamp recorded for the job's end.
349
+ if _JOB_END_TIME in payload:
350
+ job_end_time = payload[_JOB_END_TIME]
351
+
352
+ if job_end_time is not None:
353
+ return job_end_time - job_start_time
354
+
355
+ # If the job did not complete, use current time to compute total job time.
356
+ if job_start_time is not None:
357
+ return datetime.datetime.utcnow().timestamp() - job_start_time
358
+
359
+ return 0.0
360
+
361
+ def get_job_goodput(self):
362
+ """Method to get the cumulative Goodput of the job computed until now.
363
+
364
+ If the application is interested in retrieving the overall Goodput of the
365
+ job throughout its lifetime, this method provides the singular Goodput
366
+ computation for the entire job.
367
+
368
+ Returns:
369
+ Goodput percentage of the entire job.
370
+
371
+ Raises:
372
+ ValueError if computed total job time is zero. In this case, Goodput
373
+ cannot be computed.
374
+ """
375
+ entries = self._cloud_logger.read_cloud_logging_entries()
376
+ total_job_time = self._get_total_job_time(entries)
377
+ # No calculations can be made if total job time is zero. This can happen if
378
+ # logs for the job are not present, sent to an invalid location or contain
379
+ # bad data. Raise a ValueError if this happens.
380
+ if total_job_time == 0.0:
381
+ raise ValueError(
382
+ 'Total job time is zero, Goodput cannot be calculated. Please fix the'
383
+ ' logging entries.'
384
+ )
385
+ productive_training_time = self._get_total_productive_training_time(entries)
386
+
387
+ return (float(productive_training_time) / total_job_time) * 100
388
+
389
+ def get_job_goodput_interval(self, interval_start, interval_end):
390
+ """Method to get the Goodput of the job within an interval window.
391
+
392
+ If the application is interested in retrieving the Goodput of the job within
393
+ a specific window of time, this method provides the singular Goodput
394
+ computation between the start and end of this window.
395
+
396
+ Args:
397
+ interval_start: The start time of the window for which Goodput is to be
398
+ computed.
399
+ interval_end: The end time of the window for which Goodput is to be
400
+ computed.
401
+
402
+ Returns:
403
+ Goodput percentage of the job within specified time window.
404
+ """
405
+ pass
@@ -0,0 +1,189 @@
1
+ """Goodput tests to validate Recorder, Calculator and Logger classes."""
2
+
3
+ import datetime
4
+ import time
5
+
6
+ from cloud_tpu_goodput.ml_goodput_measurement.src import goodput
7
+
8
+ from google3.testing.pybase import googletest
9
+
10
+ # Fake job timeline information for test purposes.
11
+ _TEST_JOB_START_TIME = datetime.datetime(
12
+ year=2024, month=1, day=1, hour=1, minute=0, second=0, microsecond=0
13
+ )
14
+ _TEST_PROGRAM_STARTUP_TIME = datetime.timedelta(seconds=5)
15
+ _TEST_STEP_START_TIME = _TEST_JOB_START_TIME + _TEST_PROGRAM_STARTUP_TIME
16
+ _TEST_TOTAL_STEPS = 5
17
+ _TEST_STEP_TIME = datetime.timedelta(seconds=3)
18
+ _TEST_JOB_END_TIME = _TEST_STEP_START_TIME + _TEST_STEP_TIME * _TEST_TOTAL_STEPS
19
+
20
+
21
+ class MockCloudLogger:
22
+
23
+ def __init__(self, job_name, logger_name):
24
+ self.job_name = job_name
25
+ self.logger_name = logger_name
26
+ self.entries = []
27
+
28
+ def write_cloud_logging_entry(self, entry):
29
+ self.entries.append(entry)
30
+
31
+ def read_cloud_logging_entries(self):
32
+ return self.entries
33
+
34
+
35
+ class GoodputTest(googletest.TestCase):
36
+
37
+ def setUp(self):
38
+ super().setUp()
39
+ self.job_name = 'test-run'
40
+ self.logger_name = 'test-log'
41
+ self.mock_cloud_logger = MockCloudLogger(self.job_name, self.logger_name)
42
+ self.goodput_recorder = goodput.GoodputRecorder(
43
+ self.job_name,
44
+ self.logger_name,
45
+ True,
46
+ self.mock_cloud_logger,
47
+ )
48
+ self.goodput_calculator = goodput.GoodputCalculator(
49
+ self.job_name, self.logger_name, self.mock_cloud_logger
50
+ )
51
+
52
+ def _mock_sample_program(self):
53
+ # Record job start time of the job: use a fake timestamp
54
+ self.goodput_recorder.record_job_start_time(_TEST_JOB_START_TIME)
55
+
56
+ # Mock _TEST_TOTAL_STEPS steps of training
57
+ step_start_time = _TEST_STEP_START_TIME
58
+ for step in range(_TEST_TOTAL_STEPS):
59
+ # Record step time
60
+ self.goodput_recorder.record_step_start_time(step, step_start_time)
61
+ step_start_time += _TEST_STEP_TIME
62
+
63
+ # Record job end time
64
+ self.goodput_recorder.record_job_end_time(_TEST_JOB_END_TIME)
65
+
66
+ def test_goodput_recorder(self):
67
+ """Test function to validate goodput recorder and logger."""
68
+ # Emulate job run timeline.
69
+ self._mock_sample_program()
70
+
71
+ # Ensure read returns the right number of entries.
72
+ validate_entries = self.mock_cloud_logger.read_cloud_logging_entries()
73
+ # There should be one entry for each of the 5 steps, one job start
74
+ # and one job end entry.
75
+ self.assertLen(validate_entries, _TEST_TOTAL_STEPS + 2)
76
+ # Ensure payload contains the expected information.
77
+ for entry_payload in validate_entries:
78
+ self.assertIn(goodput._JOB_NAME, entry_payload)
79
+ self.assertEqual(entry_payload[goodput._JOB_NAME], self.job_name)
80
+ if goodput._JOB_START_TIME in entry_payload:
81
+ self.assertEqual(
82
+ entry_payload[goodput._JOB_START_TIME],
83
+ _TEST_JOB_START_TIME.timestamp(),
84
+ )
85
+ if goodput._JOB_END_TIME in entry_payload:
86
+ self.assertEqual(
87
+ entry_payload[goodput._JOB_END_TIME],
88
+ _TEST_JOB_END_TIME.timestamp(),
89
+ )
90
+ if goodput._STEP_START_TIME in entry_payload:
91
+ step_count = entry_payload[goodput._STEP_COUNT]
92
+ expected_start_start_time = (
93
+ _TEST_STEP_START_TIME + _TEST_STEP_TIME * step_count
94
+ )
95
+ self.assertEqual(
96
+ entry_payload[goodput._STEP_START_TIME],
97
+ expected_start_start_time.timestamp(),
98
+ )
99
+
100
+ def test_goodput_calculator(self):
101
+ """Test function to validate goodput calculator."""
102
+ # Emulate job run timeline.
103
+ self._mock_sample_program()
104
+ # Get the computed Goodput from the library and compare with expected
105
+ # result.
106
+ computed_goodput = self.goodput_calculator.get_job_goodput()
107
+ expected_goodput = (
108
+ (_TEST_STEP_TIME * _TEST_TOTAL_STEPS)
109
+ / (_TEST_JOB_END_TIME - _TEST_JOB_START_TIME)
110
+ * 100
111
+ )
112
+ self.assertEqual(computed_goodput, expected_goodput)
113
+
114
+
115
+ class GoodputDisruptionTest(googletest.TestCase):
116
+
117
+ def setUp(self):
118
+ super().setUp()
119
+ self.job_name = 'test-run'
120
+ self.logger_name = 'test-log'
121
+ self.mock_cloud_logger = MockCloudLogger(self.job_name, self.logger_name)
122
+ self.goodput_recorder = goodput.GoodputRecorder(
123
+ self.job_name,
124
+ self.logger_name,
125
+ True,
126
+ self.mock_cloud_logger,
127
+ )
128
+ self.goodput_calculator = goodput.GoodputCalculator(
129
+ self.job_name, self.logger_name, self.mock_cloud_logger
130
+ )
131
+
132
+ def test_goodput_calculator(self):
133
+ """Test function to validate goodput calculator."""
134
+ # It is not ideal to use non-deterministic timestamps in unit tests, but
135
+ # testing this complex scenario using deterministic timestamps is not
136
+ # straightforward.
137
+ # TODO(xfgu): Refactor this test.
138
+ job_start_time = datetime.datetime.utcnow()
139
+ self.goodput_recorder.record_job_start_time(job_start_time)
140
+
141
+ # Mock _TEST_TOTAL_STEPS steps of training
142
+ step_start_time = job_start_time + _TEST_PROGRAM_STARTUP_TIME
143
+ for step in range(_TEST_TOTAL_STEPS):
144
+ # Record step time
145
+ self.goodput_recorder.record_step_start_time(step, step_start_time)
146
+ step_start_time += _TEST_STEP_TIME
147
+
148
+ # Simulate a 30-second disruption.
149
+ disruption_time = datetime.timedelta(seconds=30)
150
+ job_start_time = step_start_time + disruption_time
151
+ self.goodput_recorder.record_job_start_time(job_start_time)
152
+ step_start_time = job_start_time + _TEST_PROGRAM_STARTUP_TIME
153
+
154
+ steps_before_query = _TEST_TOTAL_STEPS - 2
155
+ for step in range(steps_before_query):
156
+ self.goodput_recorder.record_step_start_time(step, step_start_time)
157
+ step_start_time += _TEST_STEP_TIME
158
+
159
+ # Get the computed Goodput from the library and compare with expected
160
+ # result.
161
+
162
+ # The time from when the job first started to when the last step start was
163
+ # logged.
164
+ total_time = (
165
+ _TEST_PROGRAM_STARTUP_TIME
166
+ + _TEST_STEP_TIME * _TEST_TOTAL_STEPS
167
+ + disruption_time
168
+ + _TEST_PROGRAM_STARTUP_TIME
169
+ + (steps_before_query - 1) * _TEST_STEP_TIME
170
+ )
171
+ seconds_before_query = 2
172
+ query_time = total_time.total_seconds() + seconds_before_query
173
+
174
+ time.sleep(query_time)
175
+ computed_goodput = self.goodput_calculator.get_job_goodput()
176
+ expected_goodput = (
177
+ (
178
+ (steps_before_query - 1) * _TEST_STEP_TIME.total_seconds()
179
+ + seconds_before_query
180
+ )
181
+ / query_time
182
+ * 100
183
+ )
184
+
185
+ self.assertAlmostEqual(computed_goodput, expected_goodput, delta=0.1)
186
+
187
+
188
+ if __name__ == '__main__':
189
+ googletest.main()
@@ -0,0 +1,52 @@
1
+ # Copyright 2024 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ [project]
16
+ name = "ml_goodput_measurement"
17
+ version = "0.0.1"
18
+ authors = [
19
+ { name="Cloud TPU Team", email="cloud-tpu-eng@google.com" },
20
+ ]
21
+ description = "Package to retrieve Goodput of jobs running on Cloud TPU."
22
+ readme = "README.md"
23
+ requires-python = ">=3.8"
24
+ license = {text = "Apache-2.0"}
25
+ classifiers = [
26
+ "Programming Language :: Python :: 3.8",
27
+ "Programming Language :: Python :: 3.9",
28
+ "Programming Language :: Python :: 3.10",
29
+ "Programming Language :: Python :: 3.11",
30
+ ]
31
+ keywords = []
32
+
33
+ # pip dependencies installed with `pip install -e .`
34
+ dependencies = [
35
+ "google-cloud-logging>=3.5.0"
36
+ ]
37
+
38
+ [project.urls]
39
+ "Homepage" = "https://github.com/google/cloud_tpu_goodput"
40
+ "Bug Tracker" = "https://github.com/google/cloud_tpu_goodput/issues"
41
+
42
+ [build-system]
43
+ # Build system specify which backend is used to build/install the project
44
+ requires = ["flit_core >=3.8,<4"]
45
+ build-backend = "flit_core.buildapi"
46
+
47
+ [tool.flit.sdist]
48
+ # Flit specific options (files to exclude from the PyPI package)
49
+ exclude = [
50
+ # Do not release tests files on PyPI
51
+ "tests/*_test.py",
52
+ ]