ml-goodput-measurement 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_goodput_measurement-0.0.1/PKG-INFO +198 -0
- ml_goodput_measurement-0.0.1/README.md +181 -0
- ml_goodput_measurement-0.0.1/ml_goodput_measurement/__init__.py +16 -0
- ml_goodput_measurement-0.0.1/ml_goodput_measurement/src/goodput.py +405 -0
- ml_goodput_measurement-0.0.1/ml_goodput_measurement/tests/goodput_test.py +189 -0
- ml_goodput_measurement-0.0.1/pyproject.toml +52 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: ml_goodput_measurement
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Package to retrieve Goodput of jobs running on Cloud TPU.
|
|
5
|
+
Keywords:
|
|
6
|
+
Author-email: Cloud TPU Team <cloud-tpu-eng@google.com>
|
|
7
|
+
Requires-Python: >=3.8
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Requires-Dist: google-cloud-logging>=3.5.0
|
|
14
|
+
Project-URL: Bug Tracker, https://github.com/google/cloud_tpu_goodput/issues
|
|
15
|
+
Project-URL: Homepage, https://github.com/google/cloud_tpu_goodput
|
|
16
|
+
|
|
17
|
+
<!--
|
|
18
|
+
Copyright 2023 Google LLC
|
|
19
|
+
|
|
20
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
21
|
+
you may not use this file except in compliance with the License.
|
|
22
|
+
You may obtain a copy of the License at
|
|
23
|
+
|
|
24
|
+
https://www.apache.org/licenses/LICENSE-2.0
|
|
25
|
+
|
|
26
|
+
Unless required by applicable law or agreed to in writing, software
|
|
27
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
28
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
29
|
+
See the License for the specific language governing permissions and
|
|
30
|
+
limitations under the License.
|
|
31
|
+
-->
|
|
32
|
+
# ML Goodput Measurement
|
|
33
|
+
|
|
34
|
+
## Overview
|
|
35
|
+
|
|
36
|
+
ML Goodput Measurement is a library intended to be used with Cloud TPU to log the
|
|
37
|
+
necessary information and query a job's Goodput. It can be pip installed to
|
|
38
|
+
import its modules, and retrieve information about a training job's overall
|
|
39
|
+
productive Goodput. The package exposes API interfaces to log useful
|
|
40
|
+
information from the user application and query Goodput for the job run, gain
|
|
41
|
+
insight into the productivity of ML workloads and utilization of compute
|
|
42
|
+
resources.
|
|
43
|
+
|
|
44
|
+
## Components
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
The ML Goodput Measurement library consists of two main components:
|
|
48
|
+
the `GoodputRecorder` and the `GoodputCalculator`. The `GoodputRecorder`
|
|
49
|
+
exposes APIs to the client to export key timestamps while a training job makes
|
|
50
|
+
progress, namely APIs that allow logging of productive step time and total job
|
|
51
|
+
run time. The library will serialize and store this data in Google Cloud
|
|
52
|
+
Logging. The `GoodputCalculator` exposes APIs to compute Goodput based on the
|
|
53
|
+
recorded data. Cloud Logging handles its internal operations asynchronously.
|
|
54
|
+
The recommended way to compute Goodput is to run an analysis program separate
|
|
55
|
+
from the training application, either on a CPU instance or on the users'
|
|
56
|
+
development machine.
|
|
57
|
+
|
|
58
|
+
## Installation
|
|
59
|
+
|
|
60
|
+
To install the ML Goodput Measurement package, run the following command on TPU VM:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install ml-goodput-measurement
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Usage
|
|
67
|
+
|
|
68
|
+
The usage of this package requires the setup of a Google Cloud project with
|
|
69
|
+
billing enabled to properly use Google Cloud Logging. If you don't have a Google
|
|
70
|
+
Cloud project, or if you don't have billing enabled for your Google Cloud
|
|
71
|
+
project, then do the following:
|
|
72
|
+
|
|
73
|
+
1. In the Google Cloud console, on the project selector page,
|
|
74
|
+
[select or create a Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects).
|
|
75
|
+
|
|
76
|
+
2. Make sure that billing is enabled for your Google Cloud project. Instructions can be found [here](https://cloud.google.com/billing/docs/how-to/verify-billing-enabled#console)
|
|
77
|
+
|
|
78
|
+
To run your training on Cloud TPU, set up the Cloud TPU environment by following
|
|
79
|
+
instructions [here](https://cloud.google.com/tpu/docs/setup-gcp-account).
|
|
80
|
+
|
|
81
|
+
To learn more about Google Cloud Logging, visit this [page](https://cloud.google.com/logging/docs).
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
### Import
|
|
85
|
+
|
|
86
|
+
To use this package, import the `goodput` module:
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from ml_goodput_measurement import goodput
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Define the name of the Google Cloud Logging logger bucket
|
|
94
|
+
|
|
95
|
+
Create a run-specific logger bucket where Cloud Logging entries can be written and read from.
|
|
96
|
+
|
|
97
|
+
For example:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
goodput_logger_name = f'goodput_{config.run_name}'
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Create a `GoodputRecorder` object
|
|
104
|
+
|
|
105
|
+
Next, create a recorder object with the following parameters:
|
|
106
|
+
|
|
107
|
+
1. `job_name`: The full run name of the job.
|
|
108
|
+
2. `logger_name`: The name of the Cloud Logging logger object (created in the previous step).
|
|
109
|
+
3. `logging_enabled`: Whether or not this process has Cloud Logging enabled.
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
> **_NOTE:_** For a multi-worker setup, please ensure that only one worker
|
|
114
|
+
writes the logs to avoid the duplication. In JAX, for example, the check
|
|
115
|
+
could be `if jax.process_index() == 0`
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
> **_NOTE:_** `logging_enabled` defaults to `False` and Goodput computations cannot be completed if no logs are ever written.
|
|
119
|
+
|
|
120
|
+
For example:
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
goodput_recorder = goodput.GoodputRecorder(job_name=config.run_name, logger_name=goodput_logger_name, logging_enabled=(jax.process_index() == 0))
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
### Record Data with `GoodputRecorder`
|
|
129
|
+
|
|
130
|
+
#### Record Job Start and End Time
|
|
131
|
+
|
|
132
|
+
Use the recorder object to record the job's overall start and end time.
|
|
133
|
+
|
|
134
|
+
For example:
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
def main(argv: Sequence[str]) -> None:
|
|
138
|
+
# Initialize configs…
|
|
139
|
+
goodput_recorder.record_job_start_time(datetime.datetime.now())
|
|
140
|
+
# TPU Initialization and device scanning…
|
|
141
|
+
# Set up other things for the main training loop…
|
|
142
|
+
# Main training loop
|
|
143
|
+
train_loop(config)
|
|
144
|
+
goodput_recorder.record_job_end_time(datetime.datetime.now())
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
#### Record Step Time
|
|
149
|
+
|
|
150
|
+
Use the recorder object to record a step's start time using `record_step_start_time(step_count)`:
|
|
151
|
+
|
|
152
|
+
For example:
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
def train_loop(config, state=None):
|
|
156
|
+
# Set up mesh, model, state, checkpoint manager…
|
|
157
|
+
|
|
158
|
+
# Initialize functional train arguments and model parameters…
|
|
159
|
+
|
|
160
|
+
# Define the compilation
|
|
161
|
+
|
|
162
|
+
for step in np.arange(start_step, config.steps):
|
|
163
|
+
goodput_recorder.record_step_start_time(step)
|
|
164
|
+
# Training step…
|
|
165
|
+
|
|
166
|
+
return state
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Retrieve Goodput with `GoodputCalculator`
|
|
170
|
+
|
|
171
|
+
In order to retrieve the Goodput of a job run, all you need to do is instantiate
|
|
172
|
+
a `GoodputCalculator` object with the job's run name and the Cloud Logging
|
|
173
|
+
logger name used to record data for that job run. Then call the `get_job_goodput`
|
|
174
|
+
API to get the computed Goodput for the job run.
|
|
175
|
+
|
|
176
|
+
It is recommended to make the `get_job_goodput` calls for a job run from an
|
|
177
|
+
instance that runs elsewhere from your training machine.
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
#### Create a `GoodputCalculator` object
|
|
181
|
+
|
|
182
|
+
Create the calculator object:
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
goodput_logger_name = f'goodput_{config.run_name}' # You can choose your own logger name.
|
|
186
|
+
goodput_calculator = goodput.GoodputCalculator(job_name=config.run_name, logger_name=goodput_logger_name)
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
#### Retrieve Goodput
|
|
190
|
+
|
|
191
|
+
Finally, call the `get_job_goodput` API to retrieve Goodput for the entire job run.
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
total_goodput = goodput_calculator.get_job_goodput()
|
|
195
|
+
print(f"Total job goodput: {total_goodput:.2f}%")
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
Copyright 2023 Google LLC
|
|
3
|
+
|
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
you may not use this file except in compliance with the License.
|
|
6
|
+
You may obtain a copy of the License at
|
|
7
|
+
|
|
8
|
+
https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
See the License for the specific language governing permissions and
|
|
14
|
+
limitations under the License.
|
|
15
|
+
-->
|
|
16
|
+
# ML Goodput Measurement
|
|
17
|
+
|
|
18
|
+
## Overview
|
|
19
|
+
|
|
20
|
+
ML Goodput Measurement is a library intended to be used with Cloud TPU to log the
|
|
21
|
+
necessary information and query a job's Goodput. It can be pip installed to
|
|
22
|
+
import its modules, and retrieve information about a training job's overall
|
|
23
|
+
productive Goodput. The package exposes API interfaces to log useful
|
|
24
|
+
information from the user application and query Goodput for the job run, gain
|
|
25
|
+
insight into the productivity of ML workloads and utilization of compute
|
|
26
|
+
resources.
|
|
27
|
+
|
|
28
|
+
## Components
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
The ML Goodput Measurement library consists of two main components:
|
|
32
|
+
the `GoodputRecorder` and the `GoodputCalculator`. The `GoodputRecorder`
|
|
33
|
+
exposes APIs to the client to export key timestamps while a training job makes
|
|
34
|
+
progress, namely APIs that allow logging of productive step time and total job
|
|
35
|
+
run time. The library will serialize and store this data in Google Cloud
|
|
36
|
+
Logging. The `GoodputCalculator` exposes APIs to compute Goodput based on the
|
|
37
|
+
recorded data. Cloud Logging handles its internal operations asynchronously.
|
|
38
|
+
The recommended way to compute Goodput is to run an analysis program separate
|
|
39
|
+
from the training application, either on a CPU instance or on the users'
|
|
40
|
+
development machine.
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
To install the ML Goodput Measurement package, run the following command on TPU VM:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install ml-goodput-measurement
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Usage
|
|
51
|
+
|
|
52
|
+
The usage of this package requires the setup of a Google Cloud project with
|
|
53
|
+
billing enabled to properly use Google Cloud Logging. If you don't have a Google
|
|
54
|
+
Cloud project, or if you don't have billing enabled for your Google Cloud
|
|
55
|
+
project, then do the following:
|
|
56
|
+
|
|
57
|
+
1. In the Google Cloud console, on the project selector page,
|
|
58
|
+
[select or create a Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects).
|
|
59
|
+
|
|
60
|
+
2. Make sure that billing is enabled for your Google Cloud project. Instructions can be found [here](https://cloud.google.com/billing/docs/how-to/verify-billing-enabled#console)
|
|
61
|
+
|
|
62
|
+
To run your training on Cloud TPU, set up the Cloud TPU environment by following
|
|
63
|
+
instructions [here](https://cloud.google.com/tpu/docs/setup-gcp-account).
|
|
64
|
+
|
|
65
|
+
To learn more about Google Cloud Logging, visit this [page](https://cloud.google.com/logging/docs).
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
### Import
|
|
69
|
+
|
|
70
|
+
To use this package, import the `goodput` module:
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from ml_goodput_measurement import goodput
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Define the name of the Google Cloud Logging logger bucket
|
|
78
|
+
|
|
79
|
+
Create a run-specific logger bucket where Cloud Logging entries can be written and read from.
|
|
80
|
+
|
|
81
|
+
For example:
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
goodput_logger_name = f'goodput_{config.run_name}'
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### Create a `GoodputRecorder` object
|
|
88
|
+
|
|
89
|
+
Next, create a recorder object with the following parameters:
|
|
90
|
+
|
|
91
|
+
1. `job_name`: The full run name of the job.
|
|
92
|
+
2. `logger_name`: The name of the Cloud Logging logger object (created in the previous step).
|
|
93
|
+
3. `logging_enabled`: Whether or not this process has Cloud Logging enabled.
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
> **_NOTE:_** For a multi-worker setup, please ensure that only one worker
|
|
98
|
+
writes the logs to avoid the duplication. In JAX, for example, the check
|
|
99
|
+
could be `if jax.process_index() == 0`
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
> **_NOTE:_** `logging_enabled` defaults to `False` and Goodput computations cannot be completed if no logs are ever written.
|
|
103
|
+
|
|
104
|
+
For example:
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
goodput_recorder = goodput.GoodputRecorder(job_name=config.run_name, logger_name=goodput_logger_name, logging_enabled=(jax.process_index() == 0))
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
### Record Data with `GoodputRecorder`
|
|
113
|
+
|
|
114
|
+
#### Record Job Start and End Time
|
|
115
|
+
|
|
116
|
+
Use the recorder object to record the job's overall start and end time.
|
|
117
|
+
|
|
118
|
+
For example:
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
def main(argv: Sequence[str]) -> None:
|
|
122
|
+
# Initialize configs…
|
|
123
|
+
goodput_recorder.record_job_start_time(datetime.datetime.now())
|
|
124
|
+
# TPU Initialization and device scanning…
|
|
125
|
+
# Set up other things for the main training loop…
|
|
126
|
+
# Main training loop
|
|
127
|
+
train_loop(config)
|
|
128
|
+
goodput_recorder.record_job_end_time(datetime.datetime.now())
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
#### Record Step Time
|
|
133
|
+
|
|
134
|
+
Use the recorder object to record a step's start time using `record_step_start_time(step_count)`:
|
|
135
|
+
|
|
136
|
+
For example:
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
def train_loop(config, state=None):
|
|
140
|
+
# Set up mesh, model, state, checkpoint manager…
|
|
141
|
+
|
|
142
|
+
# Initialize functional train arguments and model parameters…
|
|
143
|
+
|
|
144
|
+
# Define the compilation
|
|
145
|
+
|
|
146
|
+
for step in np.arange(start_step, config.steps):
|
|
147
|
+
goodput_recorder.record_step_start_time(step)
|
|
148
|
+
# Training step…
|
|
149
|
+
|
|
150
|
+
return state
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### Retrieve Goodput with `GoodputCalculator`
|
|
154
|
+
|
|
155
|
+
In order to retrieve the Goodput of a job run, all you need to do is instantiate
|
|
156
|
+
a `GoodputCalculator` object with the job's run name and the Cloud Logging
|
|
157
|
+
logger name used to record data for that job run. Then call the `get_job_goodput`
|
|
158
|
+
API to get the computed Goodput for the job run.
|
|
159
|
+
|
|
160
|
+
It is recommended to make the `get_job_goodput` calls for a job run from an
|
|
161
|
+
instance that runs elsewhere from your training machine.
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
#### Create a `GoodputCalculator` object
|
|
165
|
+
|
|
166
|
+
Create the calculator object:
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
goodput_logger_name = f'goodput_{config.run_name}' # You can choose your own logger name.
|
|
170
|
+
goodput_calculator = goodput.GoodputCalculator(job_name=config.run_name, logger_name=goodput_logger_name)
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
#### Retrieve Goodput
|
|
174
|
+
|
|
175
|
+
Finally, call the `get_job_goodput` API to retrieve Goodput for the entire job run.
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
total_goodput = goodput_calculator.get_job_goodput()
|
|
179
|
+
print(f"Total job goodput: {total_goodput:.2f}%")
|
|
180
|
+
```
|
|
181
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
|
|
2
|
+
# Copyright 2024 Google LLC
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
from cloud_tpu_goodput.ml_goodput_measurement.src import goodput
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
"""Goodput package API implementations.
|
|
2
|
+
|
|
3
|
+
This file contains all the methods exposed through the cloud_tpu_goodput library
|
|
4
|
+
for users to log necessary information to compute Goodput, and to query the
|
|
5
|
+
computed Goodput.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import datetime
|
|
9
|
+
import logging
|
|
10
|
+
from typing import Any, Optional
|
|
11
|
+
import google.cloud.logging
|
|
12
|
+
|
|
13
|
+
_JOB_NAME = 'job_name'
|
|
14
|
+
_STEP_COUNT = 'step_count'
|
|
15
|
+
_STEP_START_TIME = 'step_start_time'
|
|
16
|
+
_JOB_START_TIME = 'job_start_time'
|
|
17
|
+
_JOB_END_TIME = 'job_end_time'
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _CloudLogger:
|
|
21
|
+
"""A helper class for reading and writing to Cloud Logging.
|
|
22
|
+
|
|
23
|
+
Attributes:
|
|
24
|
+
job_name: Name of a specific job.
|
|
25
|
+
logger: The Cloud Logging logger object.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(self, job_name: str, log_name: str):
|
|
29
|
+
"""_CloudLogger constructor.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
job_name: Name of the job the _CloudLogger is for.
|
|
33
|
+
log_name: Name of the log being written.
|
|
34
|
+
"""
|
|
35
|
+
self.job_name = job_name
|
|
36
|
+
logging_client = google.cloud.logging.Client()
|
|
37
|
+
self.logger = logging_client.logger(log_name)
|
|
38
|
+
|
|
39
|
+
def _filter_entries_for_job(self, entries) -> list[Any]:
|
|
40
|
+
"""Filters entries for the specific job.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
entries: Cloud Logging entries from user-specified logger.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
Filtered entries for the specific job.
|
|
47
|
+
"""
|
|
48
|
+
filtered_entries = []
|
|
49
|
+
for entry in entries:
|
|
50
|
+
payload = entry.payload
|
|
51
|
+
if _JOB_NAME not in payload:
|
|
52
|
+
continue
|
|
53
|
+
if payload[_JOB_NAME] != self.job_name:
|
|
54
|
+
continue
|
|
55
|
+
filtered_entries.append(payload)
|
|
56
|
+
|
|
57
|
+
return filtered_entries
|
|
58
|
+
|
|
59
|
+
def write_cloud_logging_entry(self, entry) -> None:
|
|
60
|
+
"""Writes an entry to the Cloud Logging logger at INFO level.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
entry: JSON-serializable structured log dictionary.
|
|
64
|
+
"""
|
|
65
|
+
if entry is None:
|
|
66
|
+
return
|
|
67
|
+
if entry[_JOB_NAME] == self.job_name:
|
|
68
|
+
self.logger.log_struct(
|
|
69
|
+
entry,
|
|
70
|
+
severity='INFO',
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def read_cloud_logging_entries(self):
|
|
74
|
+
return self._filter_entries_for_job(
|
|
75
|
+
self.logger.list_entries(order_by='timestamp asc')
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class GoodputRecorder:
|
|
80
|
+
"""The Goodput recorder class, responsible for recording Goodput metrics from the user application.
|
|
81
|
+
|
|
82
|
+
Attributes:
|
|
83
|
+
job_name: Name of the job the GoodputRecorder is for.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def __init__(
|
|
87
|
+
self,
|
|
88
|
+
job_name: str,
|
|
89
|
+
logger_name: str,
|
|
90
|
+
logging_enabled=False,
|
|
91
|
+
logger: Optional[_CloudLogger] = None,
|
|
92
|
+
):
|
|
93
|
+
"""GoodputRecorder constructor.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
job_name: Name of the job the GoodputRecorder is for.
|
|
97
|
+
logger_name: The name of the Cloud Logging logger object that the
|
|
98
|
+
application wants logs to be written to and read from.
|
|
99
|
+
logging_enabled: A boolean value to indicate whether the current process
|
|
100
|
+
should send logs to Cloud Logging or not. The application should set
|
|
101
|
+
this value to True if the Recorder is being called from TPU worker 0 and
|
|
102
|
+
the application's configurations request Goodput logging.
|
|
103
|
+
logger: Should never be passed directly by the user.
|
|
104
|
+
"""
|
|
105
|
+
self.job_name = job_name
|
|
106
|
+
# If logging is disabled for this process, do not create a _cloud_logger
|
|
107
|
+
# object and exit early if any record record_* API is called.
|
|
108
|
+
if not logging_enabled:
|
|
109
|
+
self._cloud_logger = None
|
|
110
|
+
logging.info('Logging is disabled for this process.')
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
if logger is not None:
|
|
114
|
+
self._cloud_logger = logger
|
|
115
|
+
else:
|
|
116
|
+
self._cloud_logger = _CloudLogger(job_name, logger_name)
|
|
117
|
+
|
|
118
|
+
def record_step_start_time(
|
|
119
|
+
self, step: int, start_time: Optional[datetime.datetime] = None
|
|
120
|
+
):
|
|
121
|
+
"""Main recorder function to log an individual step's start time.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
step: The count of the step that timing information is recorded for.
|
|
125
|
+
start_time: Optional backfill start time of the training step. If
|
|
126
|
+
provided, it has to be in UTC time.
|
|
127
|
+
"""
|
|
128
|
+
if self._cloud_logger is None:
|
|
129
|
+
return
|
|
130
|
+
if start_time is None:
|
|
131
|
+
start_time = datetime.datetime.utcnow()
|
|
132
|
+
|
|
133
|
+
self._cloud_logger.write_cloud_logging_entry({
|
|
134
|
+
_JOB_NAME: self.job_name,
|
|
135
|
+
_STEP_COUNT: int(step),
|
|
136
|
+
_STEP_START_TIME: start_time.timestamp(),
|
|
137
|
+
})
|
|
138
|
+
|
|
139
|
+
def record_checkpoint_progress(self, step, checkpoint_start_time):
|
|
140
|
+
"""Main recorder function to log information on a successful checkpoint.
|
|
141
|
+
|
|
142
|
+
This method is intended to log the progress for a checkpoint (last step
|
|
143
|
+
included in the checkpoint) and when the checkpoint starts. This information
|
|
144
|
+
will be retrieved in the future to determine whether training progress from
|
|
145
|
+
a completed step contributes to Goodput or wasted progress Badput.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
step: The step count of the last step included in the saved checkpoint.
|
|
149
|
+
checkpoint_start_time: Timestamp at which the checkpoint containing
|
|
150
|
+
progress upto "step" starts to save.
|
|
151
|
+
"""
|
|
152
|
+
pass
|
|
153
|
+
|
|
154
|
+
def record_job_start_time(
|
|
155
|
+
self, start_time: Optional[datetime.datetime] = None
|
|
156
|
+
):
|
|
157
|
+
"""Main recorder function to log a job's start time.
|
|
158
|
+
|
|
159
|
+
Args:
|
|
160
|
+
start_time: Optional backfill start time of the job. If provided, it has
|
|
161
|
+
to be in UTC time.
|
|
162
|
+
"""
|
|
163
|
+
if self._cloud_logger is None:
|
|
164
|
+
return
|
|
165
|
+
if start_time is None:
|
|
166
|
+
start_time = datetime.datetime.utcnow()
|
|
167
|
+
|
|
168
|
+
self._cloud_logger.write_cloud_logging_entry({
|
|
169
|
+
_JOB_NAME: self.job_name,
|
|
170
|
+
_JOB_START_TIME: start_time.timestamp(),
|
|
171
|
+
})
|
|
172
|
+
|
|
173
|
+
def record_job_end_time(self, end_time: Optional[datetime.datetime] = None):
|
|
174
|
+
"""Main recorder function to log a job's end time.
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
end_time: Optional backfull end time of the job. If provided, it has to be
|
|
178
|
+
in UTC time.
|
|
179
|
+
"""
|
|
180
|
+
if self._cloud_logger is None:
|
|
181
|
+
return
|
|
182
|
+
if end_time is None:
|
|
183
|
+
end_time = datetime.datetime.utcnow()
|
|
184
|
+
|
|
185
|
+
self._cloud_logger.write_cloud_logging_entry({
|
|
186
|
+
_JOB_NAME: self.job_name,
|
|
187
|
+
_JOB_END_TIME: end_time.timestamp(),
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
def record_tpu_init_start_time(self, start_time):
|
|
191
|
+
"""Main recorder function to log the start time for TPU initialization.
|
|
192
|
+
|
|
193
|
+
Note: TPU initialization may include the time spent in completing
|
|
194
|
+
jax.devices() which is responsible for device scanning and Slice Builder
|
|
195
|
+
initialization.
|
|
196
|
+
|
|
197
|
+
Args:
|
|
198
|
+
start_time: Start time of TPU initialization.
|
|
199
|
+
"""
|
|
200
|
+
pass
|
|
201
|
+
|
|
202
|
+
def record_tpu_init_end_time(self, end_time):
|
|
203
|
+
"""Main recorder function to log the end time for TPU initialization.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
end_time: End time of TPU initialization.
|
|
207
|
+
"""
|
|
208
|
+
pass
|
|
209
|
+
|
|
210
|
+
def record_training_preparation_start_time(self, start_time):
|
|
211
|
+
"""Main recorder function to log the start time of training preparation before starting a training loop.
|
|
212
|
+
|
|
213
|
+
Note: Training preparation may include the time spent in creation of
|
|
214
|
+
checkpoint managers, checkpoint loading, running mesh and model optimizers
|
|
215
|
+
etc.
|
|
216
|
+
|
|
217
|
+
Args:
|
|
218
|
+
start_time: Start time of training preparation.
|
|
219
|
+
"""
|
|
220
|
+
pass
|
|
221
|
+
|
|
222
|
+
def record_training_preparation_end_time(self, end_time):
|
|
223
|
+
"""Main recorder function to log the end time of training preparation before starting a training loop.
|
|
224
|
+
|
|
225
|
+
Args:
|
|
226
|
+
end_time: End time of training preparation.
|
|
227
|
+
"""
|
|
228
|
+
pass
|
|
229
|
+
|
|
230
|
+
def record_data_loading_start_time(self, start_time):
|
|
231
|
+
"""Main recorder function to log the start time of training's data loading.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
start_time: Start time of data loading.
|
|
235
|
+
"""
|
|
236
|
+
pass
|
|
237
|
+
|
|
238
|
+
def record_data_loading_end_time(self, end_time):
|
|
239
|
+
"""Main recorder function to log the end time of training's data loading.
|
|
240
|
+
|
|
241
|
+
Args:
|
|
242
|
+
end_time: End time of data loading.
|
|
243
|
+
"""
|
|
244
|
+
pass
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class GoodputCalculator:
|
|
248
|
+
"""The Goodput calculator class, responsible for querying necessary information and computing Goodput metrics to return to the user application.
|
|
249
|
+
|
|
250
|
+
Attributes:
|
|
251
|
+
job_name: Name of the job the GoodputCalculator is for.
|
|
252
|
+
"""
|
|
253
|
+
|
|
254
|
+
def __init__(
|
|
255
|
+
self,
|
|
256
|
+
job_name: str,
|
|
257
|
+
logger_name: str,
|
|
258
|
+
logger: Optional[_CloudLogger] = None,
|
|
259
|
+
):
|
|
260
|
+
"""GoodputCalculator constructor.
|
|
261
|
+
|
|
262
|
+
Args:
|
|
263
|
+
job_name: Name of the job the GoodputCalculator is for.
|
|
264
|
+
logger_name: Name of the log being written.
|
|
265
|
+
logger: Should never be passed directly by the user.
|
|
266
|
+
"""
|
|
267
|
+
self.job_name = job_name
|
|
268
|
+
if logger is not None:
|
|
269
|
+
self._cloud_logger = logger
|
|
270
|
+
else:
|
|
271
|
+
self._cloud_logger = _CloudLogger(job_name, logger_name)
|
|
272
|
+
|
|
273
|
+
def _get_total_productive_training_time(self, entries: list[Any]) -> float:
|
|
274
|
+
"""Helper function to compute the total productive training time.
|
|
275
|
+
|
|
276
|
+
Args:
|
|
277
|
+
entries: Cloud Logging entries from user-specified logger for a specific
|
|
278
|
+
job.
|
|
279
|
+
|
|
280
|
+
Returns:
|
|
281
|
+
The job's total productive training time.
|
|
282
|
+
|
|
283
|
+
"""
|
|
284
|
+
# Build a deserialized dictionary from cloud logging entries to store step
|
|
285
|
+
# start times. The dictionary maps from step count to start time and will be
|
|
286
|
+
# used to each step's productive time by looking for its completion in the
|
|
287
|
+
# next step's start.
|
|
288
|
+
# Note in the instance where progress is lost due to a disruption and the
|
|
289
|
+
# last successful checkpoint did not include all the steps, the last set of
|
|
290
|
+
# records of the step information will be kept and the previous set will be
|
|
291
|
+
# overwritten by design so as to correct for the the previously computed
|
|
292
|
+
# additional time that was counted as productive but lost due to a
|
|
293
|
+
# disruption.
|
|
294
|
+
step_start_data = {}
|
|
295
|
+
curr_last_step = -1
|
|
296
|
+
job_end_time = None
|
|
297
|
+
for payload in entries:
|
|
298
|
+
if _STEP_START_TIME in payload:
|
|
299
|
+
curr_step = payload[_STEP_COUNT]
|
|
300
|
+
step_start_data[curr_step] = payload[_STEP_START_TIME]
|
|
301
|
+
for step in range(curr_step + 1, curr_last_step + 1):
|
|
302
|
+
if step in step_start_data:
|
|
303
|
+
step_start_data.pop(step)
|
|
304
|
+
curr_last_step = curr_step
|
|
305
|
+
|
|
306
|
+
if _JOB_END_TIME in payload:
|
|
307
|
+
# Locate the last instance of job's end time if the job has completed.
|
|
308
|
+
job_end_time = payload[_JOB_END_TIME]
|
|
309
|
+
|
|
310
|
+
# Largest step count that has been recorded. For the last step, we check if
|
|
311
|
+
# the step completed by looking for the corresponding run's end time, since
|
|
312
|
+
# there are are no more steps to check for its completion or compute step
|
|
313
|
+
# time.
|
|
314
|
+
productive_training_time = 0.0
|
|
315
|
+
|
|
316
|
+
for step_count, step_start_time in step_start_data.items():
|
|
317
|
+
if step_count < curr_last_step:
|
|
318
|
+
productive_training_time += (
|
|
319
|
+
step_start_data[step_count + 1] - step_start_time
|
|
320
|
+
)
|
|
321
|
+
elif job_end_time is not None:
|
|
322
|
+
productive_training_time += job_end_time - step_start_time
|
|
323
|
+
else:
|
|
324
|
+
productive_training_time += (
|
|
325
|
+
datetime.datetime.utcnow().timestamp() - step_start_time
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
return productive_training_time
|
|
329
|
+
|
|
330
|
+
def _get_total_job_time(self, entries: list[Any]) -> float:
|
|
331
|
+
"""Helper function to compute the total job runtime.
|
|
332
|
+
|
|
333
|
+
Args:
|
|
334
|
+
entries: Cloud Logging entries from user-specified logger for a specific
|
|
335
|
+
job.
|
|
336
|
+
|
|
337
|
+
Returns:
|
|
338
|
+
The job's total runtime.
|
|
339
|
+
"""
|
|
340
|
+
# De-serealize job start and end times from cloud logging entries. These
|
|
341
|
+
# will be used to compute total runtime of the job.
|
|
342
|
+
job_start_time = None
|
|
343
|
+
job_end_time = None
|
|
344
|
+
for payload in entries:
|
|
345
|
+
# Locate the earliest timestamp recorded for the job's start.
|
|
346
|
+
if _JOB_START_TIME in payload and job_start_time is None:
|
|
347
|
+
job_start_time = payload[_JOB_START_TIME]
|
|
348
|
+
# Locate the latest timestamp recorded for the job's end.
|
|
349
|
+
if _JOB_END_TIME in payload:
|
|
350
|
+
job_end_time = payload[_JOB_END_TIME]
|
|
351
|
+
|
|
352
|
+
if job_end_time is not None:
|
|
353
|
+
return job_end_time - job_start_time
|
|
354
|
+
|
|
355
|
+
# If the job did not complete, use current time to compute total job time.
|
|
356
|
+
if job_start_time is not None:
|
|
357
|
+
return datetime.datetime.utcnow().timestamp() - job_start_time
|
|
358
|
+
|
|
359
|
+
return 0.0
|
|
360
|
+
|
|
361
|
+
def get_job_goodput(self):
|
|
362
|
+
"""Method to get the cumulative Goodput of the job computed until now.
|
|
363
|
+
|
|
364
|
+
If the application is interested in retrieving the overall Goodput of the
|
|
365
|
+
job throughout its lifetime, this method provides the singular Goodput
|
|
366
|
+
computation for the entire job.
|
|
367
|
+
|
|
368
|
+
Returns:
|
|
369
|
+
Goodput percentage of the entire job.
|
|
370
|
+
|
|
371
|
+
Raises:
|
|
372
|
+
ValueError if computed total job time is zero. In this case, Goodput
|
|
373
|
+
cannot be computed.
|
|
374
|
+
"""
|
|
375
|
+
entries = self._cloud_logger.read_cloud_logging_entries()
|
|
376
|
+
total_job_time = self._get_total_job_time(entries)
|
|
377
|
+
# No calculations can be made if total job time is zero. This can happen if
|
|
378
|
+
# logs for the job are not present, sent to an invalid location or contain
|
|
379
|
+
# bad data. Raise a ValueError if this happens.
|
|
380
|
+
if total_job_time == 0.0:
|
|
381
|
+
raise ValueError(
|
|
382
|
+
'Total job time is zero, Goodput cannot be calculated. Please fix the'
|
|
383
|
+
' logging entries.'
|
|
384
|
+
)
|
|
385
|
+
productive_training_time = self._get_total_productive_training_time(entries)
|
|
386
|
+
|
|
387
|
+
return (float(productive_training_time) / total_job_time) * 100
|
|
388
|
+
|
|
389
|
+
def get_job_goodput_interval(self, interval_start, interval_end):
|
|
390
|
+
"""Method to get the Goodput of the job within an interval window.
|
|
391
|
+
|
|
392
|
+
If the application is interested in retrieving the Goodput of the job within
|
|
393
|
+
a specific window of time, this method provides the singular Goodput
|
|
394
|
+
computation between the start and end of this window.
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
interval_start: The start time of the window for which Goodput is to be
|
|
398
|
+
computed.
|
|
399
|
+
interval_end: The end time of the window for which Goodput is to be
|
|
400
|
+
computed.
|
|
401
|
+
|
|
402
|
+
Returns:
|
|
403
|
+
Goodput percentage of the job within specified time window.
|
|
404
|
+
"""
|
|
405
|
+
pass
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Goodput tests to validate Recorder, Calculator and Logger classes."""
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
from cloud_tpu_goodput.ml_goodput_measurement.src import goodput
|
|
7
|
+
|
|
8
|
+
from google3.testing.pybase import googletest
|
|
9
|
+
|
|
10
|
+
# Fake job timeline information for test purposes.
|
|
11
|
+
_TEST_JOB_START_TIME = datetime.datetime(
|
|
12
|
+
year=2024, month=1, day=1, hour=1, minute=0, second=0, microsecond=0
|
|
13
|
+
)
|
|
14
|
+
_TEST_PROGRAM_STARTUP_TIME = datetime.timedelta(seconds=5)
|
|
15
|
+
_TEST_STEP_START_TIME = _TEST_JOB_START_TIME + _TEST_PROGRAM_STARTUP_TIME
|
|
16
|
+
_TEST_TOTAL_STEPS = 5
|
|
17
|
+
_TEST_STEP_TIME = datetime.timedelta(seconds=3)
|
|
18
|
+
_TEST_JOB_END_TIME = _TEST_STEP_START_TIME + _TEST_STEP_TIME * _TEST_TOTAL_STEPS
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class MockCloudLogger:
|
|
22
|
+
|
|
23
|
+
def __init__(self, job_name, logger_name):
|
|
24
|
+
self.job_name = job_name
|
|
25
|
+
self.logger_name = logger_name
|
|
26
|
+
self.entries = []
|
|
27
|
+
|
|
28
|
+
def write_cloud_logging_entry(self, entry):
|
|
29
|
+
self.entries.append(entry)
|
|
30
|
+
|
|
31
|
+
def read_cloud_logging_entries(self):
|
|
32
|
+
return self.entries
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class GoodputTest(googletest.TestCase):
|
|
36
|
+
|
|
37
|
+
def setUp(self):
|
|
38
|
+
super().setUp()
|
|
39
|
+
self.job_name = 'test-run'
|
|
40
|
+
self.logger_name = 'test-log'
|
|
41
|
+
self.mock_cloud_logger = MockCloudLogger(self.job_name, self.logger_name)
|
|
42
|
+
self.goodput_recorder = goodput.GoodputRecorder(
|
|
43
|
+
self.job_name,
|
|
44
|
+
self.logger_name,
|
|
45
|
+
True,
|
|
46
|
+
self.mock_cloud_logger,
|
|
47
|
+
)
|
|
48
|
+
self.goodput_calculator = goodput.GoodputCalculator(
|
|
49
|
+
self.job_name, self.logger_name, self.mock_cloud_logger
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def _mock_sample_program(self):
|
|
53
|
+
# Record job start time of the job: use a fake timestamp
|
|
54
|
+
self.goodput_recorder.record_job_start_time(_TEST_JOB_START_TIME)
|
|
55
|
+
|
|
56
|
+
# Mock _TEST_TOTAL_STEPS steps of training
|
|
57
|
+
step_start_time = _TEST_STEP_START_TIME
|
|
58
|
+
for step in range(_TEST_TOTAL_STEPS):
|
|
59
|
+
# Record step time
|
|
60
|
+
self.goodput_recorder.record_step_start_time(step, step_start_time)
|
|
61
|
+
step_start_time += _TEST_STEP_TIME
|
|
62
|
+
|
|
63
|
+
# Record job end time
|
|
64
|
+
self.goodput_recorder.record_job_end_time(_TEST_JOB_END_TIME)
|
|
65
|
+
|
|
66
|
+
def test_goodput_recorder(self):
|
|
67
|
+
"""Test function to validate goodput recorder and logger."""
|
|
68
|
+
# Emulate job run timeline.
|
|
69
|
+
self._mock_sample_program()
|
|
70
|
+
|
|
71
|
+
# Ensure read returns the right number of entries.
|
|
72
|
+
validate_entries = self.mock_cloud_logger.read_cloud_logging_entries()
|
|
73
|
+
# There should be one entry for each of the 5 steps, one job start
|
|
74
|
+
# and one job end entry.
|
|
75
|
+
self.assertLen(validate_entries, _TEST_TOTAL_STEPS + 2)
|
|
76
|
+
# Ensure payload contains the expected information.
|
|
77
|
+
for entry_payload in validate_entries:
|
|
78
|
+
self.assertIn(goodput._JOB_NAME, entry_payload)
|
|
79
|
+
self.assertEqual(entry_payload[goodput._JOB_NAME], self.job_name)
|
|
80
|
+
if goodput._JOB_START_TIME in entry_payload:
|
|
81
|
+
self.assertEqual(
|
|
82
|
+
entry_payload[goodput._JOB_START_TIME],
|
|
83
|
+
_TEST_JOB_START_TIME.timestamp(),
|
|
84
|
+
)
|
|
85
|
+
if goodput._JOB_END_TIME in entry_payload:
|
|
86
|
+
self.assertEqual(
|
|
87
|
+
entry_payload[goodput._JOB_END_TIME],
|
|
88
|
+
_TEST_JOB_END_TIME.timestamp(),
|
|
89
|
+
)
|
|
90
|
+
if goodput._STEP_START_TIME in entry_payload:
|
|
91
|
+
step_count = entry_payload[goodput._STEP_COUNT]
|
|
92
|
+
expected_start_start_time = (
|
|
93
|
+
_TEST_STEP_START_TIME + _TEST_STEP_TIME * step_count
|
|
94
|
+
)
|
|
95
|
+
self.assertEqual(
|
|
96
|
+
entry_payload[goodput._STEP_START_TIME],
|
|
97
|
+
expected_start_start_time.timestamp(),
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
def test_goodput_calculator(self):
|
|
101
|
+
"""Test function to validate goodput calculator."""
|
|
102
|
+
# Emulate job run timeline.
|
|
103
|
+
self._mock_sample_program()
|
|
104
|
+
# Get the computed Goodput from the library and compare with expected
|
|
105
|
+
# result.
|
|
106
|
+
computed_goodput = self.goodput_calculator.get_job_goodput()
|
|
107
|
+
expected_goodput = (
|
|
108
|
+
(_TEST_STEP_TIME * _TEST_TOTAL_STEPS)
|
|
109
|
+
/ (_TEST_JOB_END_TIME - _TEST_JOB_START_TIME)
|
|
110
|
+
* 100
|
|
111
|
+
)
|
|
112
|
+
self.assertEqual(computed_goodput, expected_goodput)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class GoodputDisruptionTest(googletest.TestCase):
|
|
116
|
+
|
|
117
|
+
def setUp(self):
|
|
118
|
+
super().setUp()
|
|
119
|
+
self.job_name = 'test-run'
|
|
120
|
+
self.logger_name = 'test-log'
|
|
121
|
+
self.mock_cloud_logger = MockCloudLogger(self.job_name, self.logger_name)
|
|
122
|
+
self.goodput_recorder = goodput.GoodputRecorder(
|
|
123
|
+
self.job_name,
|
|
124
|
+
self.logger_name,
|
|
125
|
+
True,
|
|
126
|
+
self.mock_cloud_logger,
|
|
127
|
+
)
|
|
128
|
+
self.goodput_calculator = goodput.GoodputCalculator(
|
|
129
|
+
self.job_name, self.logger_name, self.mock_cloud_logger
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def test_goodput_calculator(self):
|
|
133
|
+
"""Test function to validate goodput calculator."""
|
|
134
|
+
# It is not ideal to use non-deterministic timestamps in unit tests, but
|
|
135
|
+
# testing this complex scenario using deterministic timestamps is not
|
|
136
|
+
# straightforward.
|
|
137
|
+
# TODO(xfgu): Refactor this test.
|
|
138
|
+
job_start_time = datetime.datetime.utcnow()
|
|
139
|
+
self.goodput_recorder.record_job_start_time(job_start_time)
|
|
140
|
+
|
|
141
|
+
# Mock _TEST_TOTAL_STEPS steps of training
|
|
142
|
+
step_start_time = job_start_time + _TEST_PROGRAM_STARTUP_TIME
|
|
143
|
+
for step in range(_TEST_TOTAL_STEPS):
|
|
144
|
+
# Record step time
|
|
145
|
+
self.goodput_recorder.record_step_start_time(step, step_start_time)
|
|
146
|
+
step_start_time += _TEST_STEP_TIME
|
|
147
|
+
|
|
148
|
+
# Simulate a 30-second disruption.
|
|
149
|
+
disruption_time = datetime.timedelta(seconds=30)
|
|
150
|
+
job_start_time = step_start_time + disruption_time
|
|
151
|
+
self.goodput_recorder.record_job_start_time(job_start_time)
|
|
152
|
+
step_start_time = job_start_time + _TEST_PROGRAM_STARTUP_TIME
|
|
153
|
+
|
|
154
|
+
steps_before_query = _TEST_TOTAL_STEPS - 2
|
|
155
|
+
for step in range(steps_before_query):
|
|
156
|
+
self.goodput_recorder.record_step_start_time(step, step_start_time)
|
|
157
|
+
step_start_time += _TEST_STEP_TIME
|
|
158
|
+
|
|
159
|
+
# Get the computed Goodput from the library and compare with expected
|
|
160
|
+
# result.
|
|
161
|
+
|
|
162
|
+
# The time from when the job first started to when the last step start was
|
|
163
|
+
# logged.
|
|
164
|
+
total_time = (
|
|
165
|
+
_TEST_PROGRAM_STARTUP_TIME
|
|
166
|
+
+ _TEST_STEP_TIME * _TEST_TOTAL_STEPS
|
|
167
|
+
+ disruption_time
|
|
168
|
+
+ _TEST_PROGRAM_STARTUP_TIME
|
|
169
|
+
+ (steps_before_query - 1) * _TEST_STEP_TIME
|
|
170
|
+
)
|
|
171
|
+
seconds_before_query = 2
|
|
172
|
+
query_time = total_time.total_seconds() + seconds_before_query
|
|
173
|
+
|
|
174
|
+
time.sleep(query_time)
|
|
175
|
+
computed_goodput = self.goodput_calculator.get_job_goodput()
|
|
176
|
+
expected_goodput = (
|
|
177
|
+
(
|
|
178
|
+
(steps_before_query - 1) * _TEST_STEP_TIME.total_seconds()
|
|
179
|
+
+ seconds_before_query
|
|
180
|
+
)
|
|
181
|
+
/ query_time
|
|
182
|
+
* 100
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
self.assertAlmostEqual(computed_goodput, expected_goodput, delta=0.1)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
if __name__ == '__main__':
|
|
189
|
+
googletest.main()
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Copyright 2024 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
[project]
|
|
16
|
+
name = "ml_goodput_measurement"
|
|
17
|
+
version = "0.0.1"
|
|
18
|
+
authors = [
|
|
19
|
+
{ name="Cloud TPU Team", email="cloud-tpu-eng@google.com" },
|
|
20
|
+
]
|
|
21
|
+
description = "Package to retrieve Goodput of jobs running on Cloud TPU."
|
|
22
|
+
readme = "README.md"
|
|
23
|
+
requires-python = ">=3.8"
|
|
24
|
+
license = {text = "Apache-2.0"}
|
|
25
|
+
classifiers = [
|
|
26
|
+
"Programming Language :: Python :: 3.8",
|
|
27
|
+
"Programming Language :: Python :: 3.9",
|
|
28
|
+
"Programming Language :: Python :: 3.10",
|
|
29
|
+
"Programming Language :: Python :: 3.11",
|
|
30
|
+
]
|
|
31
|
+
keywords = []
|
|
32
|
+
|
|
33
|
+
# pip dependencies installed with `pip install -e .`
|
|
34
|
+
dependencies = [
|
|
35
|
+
"google-cloud-logging>=3.5.0"
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[project.urls]
|
|
39
|
+
"Homepage" = "https://github.com/google/cloud_tpu_goodput"
|
|
40
|
+
"Bug Tracker" = "https://github.com/google/cloud_tpu_goodput/issues"
|
|
41
|
+
|
|
42
|
+
[build-system]
|
|
43
|
+
# Build system specify which backend is used to build/install the project
|
|
44
|
+
requires = ["flit_core >=3.8,<4"]
|
|
45
|
+
build-backend = "flit_core.buildapi"
|
|
46
|
+
|
|
47
|
+
[tool.flit.sdist]
|
|
48
|
+
# Flit specific options (files to exclude from the PyPI package)
|
|
49
|
+
exclude = [
|
|
50
|
+
# Do not release tests files on PyPI
|
|
51
|
+
"tests/*_test.py",
|
|
52
|
+
]
|