colabutilscs4337 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- colabutilscs4337-0.1.0/.gitignore +17 -0
- colabutilscs4337-0.1.0/CHANGELOG.md +18 -0
- colabutilscs4337-0.1.0/LICENSE +21 -0
- colabutilscs4337-0.1.0/PKG-INFO +241 -0
- colabutilscs4337-0.1.0/README.md +202 -0
- colabutilscs4337-0.1.0/pyproject.toml +72 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/__init__.py +45 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/_colab.py +165 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/_compat.py +74 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/_install.py +234 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/_tested.json +20 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/_version.py +24 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/drive/__init__.py +45 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/drive/_config.py +113 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/drive/_runs.py +213 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/env/__init__.py +20 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/env/_check.py +98 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/env/_runtime.py +71 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/env/_secrets.py +164 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/models/__init__.py +5 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/models/_load.py +146 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/tools/__init__.py +14 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/tools/_netron.py +100 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/tools/_summary.py +48 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/viz/__init__.py +35 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/viz/_cam.py +190 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/viz/_data.py +381 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/viz/_hooks.py +88 -0
- colabutilscs4337-0.1.0/src/colabutilscs4337/viz/_plot.py +131 -0
- colabutilscs4337-0.1.0/tests/conftest.py +60 -0
- colabutilscs4337-0.1.0/tests/test_colab.py +58 -0
- colabutilscs4337-0.1.0/tests/test_data_viz.py +112 -0
- colabutilscs4337-0.1.0/tests/test_drive.py +102 -0
- colabutilscs4337-0.1.0/tests/test_env.py +38 -0
- colabutilscs4337-0.1.0/tests/test_install.py +113 -0
- colabutilscs4337-0.1.0/tests/test_models.py +110 -0
- colabutilscs4337-0.1.0/tests/test_secrets.py +107 -0
- colabutilscs4337-0.1.0/tests/test_tools.py +82 -0
- colabutilscs4337-0.1.0/tests/test_viz.py +137 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
Each release lists the Google Colab runtime it was tested against
|
|
4
|
+
(from [googlecolab/backend-info](https://github.com/googlecolab/backend-info)).
|
|
5
|
+
|
|
6
|
+
## 0.1.0 (2026-10-12)
|
|
7
|
+
|
|
8
|
+
Tested against: Colab GPU runtime, Python 3.13.15, torch 2.11.0+cu130, torchvision 0.26.0+cu130, fastai 2.8.12.
|
|
9
|
+
|
|
10
|
+
- `env`: runtime info, notebook name detection, version check against the tested Colab runtime.
|
|
11
|
+
- `env.load_secrets()` / `env.get_secret()`: API keys from the environment, Colab Secrets, a `.env` file, or a prompt.
|
|
12
|
+
- `viz.show_images()`, `viz.show_by_class()`, `viz.tensor_to_image()`: look at datasets, loaders, and batches.
|
|
13
|
+
- `drive`: Drive mounting, timestamped run folders, persistent config (`data_dir`, `runs_dir`).
|
|
14
|
+
- `tools`: torchinfo summary wrapper, Netron viewer inside Colab.
|
|
15
|
+
- `install()`: locked installer that refuses to change packages Colab already provides.
|
|
16
|
+
- `models`: `load_model()` for fastai exports/checkpoints and PyTorch models/state dicts.
|
|
17
|
+
- `viz`: Grad-CAM, layer activations, feature-map plotting.
|
|
18
|
+
- Sphinx documentation (user guide and API reference), published to GitHub Pages.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 CS4337 Course Staff
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: colabutilscs4337
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Google Colab utilities for CS4337: Drive run folders, model inspection, and CNN visualization (Grad-CAM, activations).
|
|
5
|
+
Project-URL: Homepage, https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337
|
|
6
|
+
Project-URL: Documentation, https://txstcodeplayground.github.io/ColabUtilsCS4337/
|
|
7
|
+
Project-URL: Repository, https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337
|
|
8
|
+
Project-URL: Issues, https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337/blob/main/CHANGELOG.md
|
|
10
|
+
Author: CS4337 Course Staff
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: colab,computer-vision,education,fastai,grad-cam,pytorch
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Framework :: Jupyter
|
|
16
|
+
Classifier: Intended Audience :: Education
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Image Recognition
|
|
23
|
+
Requires-Python: >=3.12
|
|
24
|
+
Provides-Extra: all
|
|
25
|
+
Requires-Dist: grad-cam>=1.5; extra == 'all'
|
|
26
|
+
Requires-Dist: netron>=7.0; extra == 'all'
|
|
27
|
+
Requires-Dist: torchinfo>=1.8; extra == 'all'
|
|
28
|
+
Provides-Extra: cam
|
|
29
|
+
Requires-Dist: grad-cam>=1.5; extra == 'cam'
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
33
|
+
Provides-Extra: fastai
|
|
34
|
+
Requires-Dist: fastai>=2.7; extra == 'fastai'
|
|
35
|
+
Provides-Extra: tools
|
|
36
|
+
Requires-Dist: netron>=7.0; extra == 'tools'
|
|
37
|
+
Requires-Dist: torchinfo>=1.8; extra == 'tools'
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
|
|
40
|
+
# colabutilscs4337
|
|
41
|
+
|
|
42
|
+
[](https://txstcodeplayground.github.io/ColabUtilsCS4337/)
|
|
43
|
+
[](https://pypi.org/project/colabutilscs4337/)
|
|
44
|
+
[](https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337/actions/workflows/ci.yml)
|
|
45
|
+
|
|
46
|
+
Google Colab utilities for CS4337 (computer vision with PyTorch and fastai).
|
|
47
|
+
|
|
48
|
+
**Documentation:** <https://txstcodeplayground.github.io/ColabUtilsCS4337/> |
|
|
49
|
+
**PyPI:** <https://pypi.org/project/colabutilscs4337/> |
|
|
50
|
+
**GitHub:** <https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337>
|
|
51
|
+
|
|
52
|
+
The documentation has a user guide with examples for every feature and the full API
|
|
53
|
+
reference. This README is a short overview.
|
|
54
|
+
|
|
55
|
+
- **`env`**: detect Colab, get the notebook name, load secrets (API keys) into the environment, check package versions against the Colab runtime this release was tested on.
|
|
56
|
+
- **`drive`**: mount Google Drive, create run folders like `MyDrive/runs/<notebook>_<YYYYmmdd-HHMMSS>_<slug>/`, keep default paths (`data_dir`, `runs_dir`) across sessions.
|
|
57
|
+
- **`tools`**: `torchinfo` summaries and the Netron graph viewer inside Colab.
|
|
58
|
+
- **`models`**: `load_model()` for fastai exports/checkpoints and PyTorch models/state dicts.
|
|
59
|
+
- **`viz`**: image grids for datasets and batches, Grad-CAM, layer activations, and feature-map plots.
|
|
60
|
+
|
|
61
|
+
## Quick start (first cell of every notebook)
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
%pip install -q --upgrade colabutilscs4337
|
|
65
|
+
import colabutilscs4337 as cu
|
|
66
|
+
cu.install("all") # adds torchinfo, netron, grad-cam; never changes Colab's torch/fastai/numpy
|
|
67
|
+
cu.env.check() # compares installed versions with the tested Colab runtime
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
If `%pip install --upgrade` replaced a version you had already imported, use **Runtime > Restart session**.
|
|
71
|
+
|
|
72
|
+
## Why `cu.install()` instead of `pip install colabutilscs4337[all]`?
|
|
73
|
+
|
|
74
|
+
Colab ships torch, torchvision, fastai, numpy, and matplotlib built for its GPU runtime.
|
|
75
|
+
This package never declares them as dependencies, so installing it cannot replace them.
|
|
76
|
+
`cu.install()` goes further for the optional extras: it pins every installed package
|
|
77
|
+
to its current version, dry-runs pip, and refuses if anything already installed would
|
|
78
|
+
change. Use `cu.install("cam", dry=True)` to preview.
|
|
79
|
+
|
|
80
|
+
Plain pip works too:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
pip list --format=freeze | grep -viE '^(colabutilscs4337|torchinfo|netron|grad-cam|ttach)==' > /tmp/c.txt
|
|
84
|
+
pip install -c /tmp/c.txt --upgrade "colabutilscs4337[all]"
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Secrets (API keys and tokens)
|
|
88
|
+
|
|
89
|
+
Never paste keys into a notebook. Store them once, then load them by name:
|
|
90
|
+
|
|
91
|
+
1. In Colab, click the **key icon** in the left sidebar, choose **Add new secret**,
|
|
92
|
+
enter a name (for example `HF_TOKEN`) and the value, and switch on **Notebook access**.
|
|
93
|
+
2. In the notebook:
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
cu.env.load_secrets("HF_TOKEN", "WANDB_API_KEY")
|
|
97
|
+
# ok HF_TOKEN (Colab Secrets)
|
|
98
|
+
# -- WANDB_API_KEY
|
|
99
|
+
# Missing: WANDB_API_KEY. In Colab: click the key icon ...
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The values are now in `os.environ`, where libraries such as Hugging Face, Weights &
|
|
103
|
+
Biases, and Kaggle look for them. Values are never printed.
|
|
104
|
+
|
|
105
|
+
Each name is looked up in this order:
|
|
106
|
+
|
|
107
|
+
1. Already set in the environment.
|
|
108
|
+
2. Colab Secrets.
|
|
109
|
+
3. A `.env` file: the `env_file=` argument, else `./.env`, else
|
|
110
|
+
`MyDrive/.colabutilscs4337/.env` when Drive is mounted (handy for keys you reuse
|
|
111
|
+
across notebooks).
|
|
112
|
+
4. A hidden password-style prompt, only with `prompt=True`.
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
cu.env.load_secrets("KAGGLE_USERNAME", "KAGGLE_KEY", required=True) # error if missing
|
|
116
|
+
cu.env.load_secrets(env_file="/content/drive/MyDrive/keys.env") # every key in a file
|
|
117
|
+
cu.env.load_secrets("OPENAI_API_KEY", prompt=True) # ask if not found
|
|
118
|
+
token = cu.env.get_secret("HF_TOKEN") # just return the value
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
A `.env` file is plain `NAME=value` lines (`#` comments and quotes are fine).
|
|
122
|
+
|
|
123
|
+
## Looking at image data
|
|
124
|
+
|
|
125
|
+
Pass whatever you have: a `Dataset`, a `DataLoader`, an `(x, y)` batch, a tensor plus
|
|
126
|
+
labels, or a list of PIL images. Class names come from `dataset.classes` automatically.
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
train_ds = torchvision.datasets.FashionMNIST(
|
|
130
|
+
"data", train=True, download=True, transform=transforms.ToTensor()
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
cu.viz.show_by_class(train_ds) # one row per class, 8 examples each
|
|
134
|
+
cu.viz.show_images(train_ds) # 32 random images titled with their class
|
|
135
|
+
cu.viz.show_images(train_loader) # the first batch of a DataLoader
|
|
136
|
+
|
|
137
|
+
x, y = next(iter(train_loader))
|
|
138
|
+
cu.viz.show_images(x, y, train_ds.classes, n=16)
|
|
139
|
+
cu.viz.show_images(x, y, train_ds.classes, preds=model(x).argmax(1)) # "pred / true", red if wrong
|
|
140
|
+
cu.viz.show_by_class(
|
|
141
|
+
x, y, labels_map, n_per_class=5
|
|
142
|
+
) # labels_map can be a list or {0: "T-Shirt", ...}
|
|
143
|
+
|
|
144
|
+
img = cu.viz.tensor_to_image(x[:8]) # PIL image: 8 images side by side
|
|
145
|
+
img = cu.viz.tensor_to_image(x[:8], dim=1) # ... stacked top to bottom
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
| Function | Shows |
|
|
149
|
+
|---|---|
|
|
150
|
+
| `show_by_class(data, labels=None, classes=None, n_per_class=8)` | one row per class |
|
|
151
|
+
| `show_images(data, labels=None, classes=None, preds=None, n=32, ncols=8)` | a grid with class (and prediction) titles |
|
|
152
|
+
| `tensor_to_image(*tensors, dim=2, pad=0)` | returns a `PIL.Image` strip |
|
|
153
|
+
|
|
154
|
+
Normalized images (for example ImageNet `Normalize`) are rescaled automatically for
|
|
155
|
+
display; pass `mean=cu.viz.IMAGENET_MEAN, std=cu.viz.IMAGENET_STD` to undo the
|
|
156
|
+
normalization exactly. Grayscale is drawn in gray, colour in RGB. With a fastai
|
|
157
|
+
`DataLoaders`, `show_images(dls)` calls fastai's own `show_batch`.
|
|
158
|
+
|
|
159
|
+
## Runs and data
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
run = cu.drive.new_run("resnet34-baseline") # mounts Drive if needed
|
|
163
|
+
print(run) # /content/drive/MyDrive/runs/Lab-3-CNNs_20261012-221500_resnet34-baseline
|
|
164
|
+
learn.path = run.path # fastai: save models into the run folder
|
|
165
|
+
torch.save(model.state_dict(), run / "model.pth")
|
|
166
|
+
|
|
167
|
+
path = cu.drive.data_dir("pets") # /content/data/pets (fast local disk)
|
|
168
|
+
cu.drive.set_config(timezone="America/Chicago", data_dir="/content/data") # saved on Drive
|
|
169
|
+
cu.drive.latest_run() # most recent run folder
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Each run folder contains a `run.json` with the notebook name, timestamp, and the
|
|
173
|
+
runtime's Python, GPU, and package versions. Without a slug, a random one such as
|
|
174
|
+
`brave-otter` is used.
|
|
175
|
+
|
|
176
|
+
Settings come from (highest first): function arguments, environment variables
|
|
177
|
+
(`COLABUTILS_DATA_DIR`, `COLABUTILS_RUNS_DIR`, `COLABUTILS_TZ`,
|
|
178
|
+
`COLABUTILS_NOTEBOOK_NAME`), `MyDrive/.colabutilscs4337/config.json`, then defaults.
|
|
179
|
+
|
|
180
|
+
## Inspecting models
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
cu.tools.summary(learn, input_size=(1, 3, 224, 224)) # fastai Learner or nn.Module
|
|
184
|
+
cu.tools.netron(learn) # interactive graph in the cell output
|
|
185
|
+
cu.tools.netron("model.onnx") # or any file Netron supports
|
|
186
|
+
|
|
187
|
+
model = cu.models.load_model("export.pkl") # fastai learn.export()
|
|
188
|
+
model = cu.models.load_model("model.pth", arch=torchvision.models.resnet18(num_classes=37))
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## Visualizing what a CNN looks at
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
cam = cu.viz.gradcam(model, x, class_idx=None) # x: (1, 3, H, W) normalized tensor
|
|
195
|
+
cu.viz.show_cam(image, cam) # image: PIL / HxWx3 array / tensor
|
|
196
|
+
|
|
197
|
+
acts = cu.viz.activations(model, x, layers=["layer1", "layer4"])
|
|
198
|
+
cu.viz.show_feature_maps(acts["layer1"], n=16)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
`target_layer` defaults to `cu.viz.suggest_target_layer(model)` (the last conv
|
|
202
|
+
layer). With `cu.install("cam")`, `cu.viz.gradcam(..., method="gradcam++")` uses the
|
|
203
|
+
[pytorch-grad-cam](https://github.com/jacobgil/pytorch-grad-cam) backend.
|
|
204
|
+
|
|
205
|
+
## Compatibility with Colab
|
|
206
|
+
|
|
207
|
+
The Colab runtime is tracked from
|
|
208
|
+
[googlecolab/backend-info](https://github.com/googlecolab/backend-info):
|
|
209
|
+
|
|
210
|
+
- `scripts/sync_colab_runtime.py` writes `constraints/colab-gpu.txt` and
|
|
211
|
+
`src/colabutilscs4337/_tested.json`.
|
|
212
|
+
- CI runs the tests inside Google's Colab runtime image
|
|
213
|
+
(`us-docker.pkg.dev/colab-images/public/cpu-runtime`) and checks that
|
|
214
|
+
`pip install ".[all]"` changes no preinstalled package.
|
|
215
|
+
- A weekly job reruns everything and opens a PR when Colab's runtime changes.
|
|
216
|
+
|
|
217
|
+
## Development
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
uv venv --python 3.13
|
|
221
|
+
uv pip install -c constraints/colab-gpu.txt -e ".[all,dev]" torch torchvision fastai
|
|
222
|
+
pytest
|
|
223
|
+
ruff check .
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
Build the documentation (Sphinx; PyTorch not needed):
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
uv venv .venv-docs --python 3.13
|
|
230
|
+
uv pip install --python .venv-docs/bin/python -r docs/requirements.txt
|
|
231
|
+
.venv-docs/bin/sphinx-build -W -b html docs docs/_build/html
|
|
232
|
+
open docs/_build/html/index.html
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
`.github/workflows/docs.yml` builds the site on every pull request and publishes it to
|
|
236
|
+
GitHub Pages on every push to `main`. See the
|
|
237
|
+
[development guide](https://txstcodeplayground.github.io/ColabUtilsCS4337/development.html)
|
|
238
|
+
for releases.
|
|
239
|
+
|
|
240
|
+
Release: push a tag `vX.Y.Z`. CI publishes to TestPyPI, then PyPI after approval.
|
|
241
|
+
Before approving, run `notebooks/smoke_test.ipynb` on a Colab GPU runtime.
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# colabutilscs4337
|
|
2
|
+
|
|
3
|
+
[](https://txstcodeplayground.github.io/ColabUtilsCS4337/)
|
|
4
|
+
[](https://pypi.org/project/colabutilscs4337/)
|
|
5
|
+
[](https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337/actions/workflows/ci.yml)
|
|
6
|
+
|
|
7
|
+
Google Colab utilities for CS4337 (computer vision with PyTorch and fastai).
|
|
8
|
+
|
|
9
|
+
**Documentation:** <https://txstcodeplayground.github.io/ColabUtilsCS4337/> |
|
|
10
|
+
**PyPI:** <https://pypi.org/project/colabutilscs4337/> |
|
|
11
|
+
**GitHub:** <https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337>
|
|
12
|
+
|
|
13
|
+
The documentation has a user guide with examples for every feature and the full API
|
|
14
|
+
reference. This README is a short overview.
|
|
15
|
+
|
|
16
|
+
- **`env`**: detect Colab, get the notebook name, load secrets (API keys) into the environment, check package versions against the Colab runtime this release was tested on.
|
|
17
|
+
- **`drive`**: mount Google Drive, create run folders like `MyDrive/runs/<notebook>_<YYYYmmdd-HHMMSS>_<slug>/`, keep default paths (`data_dir`, `runs_dir`) across sessions.
|
|
18
|
+
- **`tools`**: `torchinfo` summaries and the Netron graph viewer inside Colab.
|
|
19
|
+
- **`models`**: `load_model()` for fastai exports/checkpoints and PyTorch models/state dicts.
|
|
20
|
+
- **`viz`**: image grids for datasets and batches, Grad-CAM, layer activations, and feature-map plots.
|
|
21
|
+
|
|
22
|
+
## Quick start (first cell of every notebook)
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
%pip install -q --upgrade colabutilscs4337
|
|
26
|
+
import colabutilscs4337 as cu
|
|
27
|
+
cu.install("all") # adds torchinfo, netron, grad-cam; never changes Colab's torch/fastai/numpy
|
|
28
|
+
cu.env.check() # compares installed versions with the tested Colab runtime
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
If `%pip install --upgrade` replaced a version you had already imported, use **Runtime > Restart session**.
|
|
32
|
+
|
|
33
|
+
## Why `cu.install()` instead of `pip install colabutilscs4337[all]`?
|
|
34
|
+
|
|
35
|
+
Colab ships torch, torchvision, fastai, numpy, and matplotlib built for its GPU runtime.
|
|
36
|
+
This package never declares them as dependencies, so installing it cannot replace them.
|
|
37
|
+
`cu.install()` goes further for the optional extras: it pins every installed package
|
|
38
|
+
to its current version, dry-runs pip, and refuses if anything already installed would
|
|
39
|
+
change. Use `cu.install("cam", dry=True)` to preview.
|
|
40
|
+
|
|
41
|
+
Plain pip works too:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip list --format=freeze | grep -viE '^(colabutilscs4337|torchinfo|netron|grad-cam|ttach)==' > /tmp/c.txt
|
|
45
|
+
pip install -c /tmp/c.txt --upgrade "colabutilscs4337[all]"
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Secrets (API keys and tokens)
|
|
49
|
+
|
|
50
|
+
Never paste keys into a notebook. Store them once, then load them by name:
|
|
51
|
+
|
|
52
|
+
1. In Colab, click the **key icon** in the left sidebar, choose **Add new secret**,
|
|
53
|
+
enter a name (for example `HF_TOKEN`) and the value, and switch on **Notebook access**.
|
|
54
|
+
2. In the notebook:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
cu.env.load_secrets("HF_TOKEN", "WANDB_API_KEY")
|
|
58
|
+
# ok HF_TOKEN (Colab Secrets)
|
|
59
|
+
# -- WANDB_API_KEY
|
|
60
|
+
# Missing: WANDB_API_KEY. In Colab: click the key icon ...
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The values are now in `os.environ`, where libraries such as Hugging Face, Weights &
|
|
64
|
+
Biases, and Kaggle look for them. Values are never printed.
|
|
65
|
+
|
|
66
|
+
Each name is looked up in this order:
|
|
67
|
+
|
|
68
|
+
1. Already set in the environment.
|
|
69
|
+
2. Colab Secrets.
|
|
70
|
+
3. A `.env` file: the `env_file=` argument, else `./.env`, else
|
|
71
|
+
`MyDrive/.colabutilscs4337/.env` when Drive is mounted (handy for keys you reuse
|
|
72
|
+
across notebooks).
|
|
73
|
+
4. A hidden password-style prompt, only with `prompt=True`.
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
cu.env.load_secrets("KAGGLE_USERNAME", "KAGGLE_KEY", required=True) # error if missing
|
|
77
|
+
cu.env.load_secrets(env_file="/content/drive/MyDrive/keys.env") # every key in a file
|
|
78
|
+
cu.env.load_secrets("OPENAI_API_KEY", prompt=True) # ask if not found
|
|
79
|
+
token = cu.env.get_secret("HF_TOKEN") # just return the value
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
A `.env` file is plain `NAME=value` lines (`#` comments and quotes are fine).
|
|
83
|
+
|
|
84
|
+
## Looking at image data
|
|
85
|
+
|
|
86
|
+
Pass whatever you have: a `Dataset`, a `DataLoader`, an `(x, y)` batch, a tensor plus
|
|
87
|
+
labels, or a list of PIL images. Class names come from `dataset.classes` automatically.
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
train_ds = torchvision.datasets.FashionMNIST(
|
|
91
|
+
"data", train=True, download=True, transform=transforms.ToTensor()
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
cu.viz.show_by_class(train_ds) # one row per class, 8 examples each
|
|
95
|
+
cu.viz.show_images(train_ds) # 32 random images titled with their class
|
|
96
|
+
cu.viz.show_images(train_loader) # the first batch of a DataLoader
|
|
97
|
+
|
|
98
|
+
x, y = next(iter(train_loader))
|
|
99
|
+
cu.viz.show_images(x, y, train_ds.classes, n=16)
|
|
100
|
+
cu.viz.show_images(x, y, train_ds.classes, preds=model(x).argmax(1)) # "pred / true", red if wrong
|
|
101
|
+
cu.viz.show_by_class(
|
|
102
|
+
x, y, labels_map, n_per_class=5
|
|
103
|
+
) # labels_map can be a list or {0: "T-Shirt", ...}
|
|
104
|
+
|
|
105
|
+
img = cu.viz.tensor_to_image(x[:8]) # PIL image: 8 images side by side
|
|
106
|
+
img = cu.viz.tensor_to_image(x[:8], dim=1) # ... stacked top to bottom
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
| Function | Shows |
|
|
110
|
+
|---|---|
|
|
111
|
+
| `show_by_class(data, labels=None, classes=None, n_per_class=8)` | one row per class |
|
|
112
|
+
| `show_images(data, labels=None, classes=None, preds=None, n=32, ncols=8)` | a grid with class (and prediction) titles |
|
|
113
|
+
| `tensor_to_image(*tensors, dim=2, pad=0)` | returns a `PIL.Image` strip |
|
|
114
|
+
|
|
115
|
+
Normalized images (for example ImageNet `Normalize`) are rescaled automatically for
|
|
116
|
+
display; pass `mean=cu.viz.IMAGENET_MEAN, std=cu.viz.IMAGENET_STD` to undo the
|
|
117
|
+
normalization exactly. Grayscale is drawn in gray, colour in RGB. With a fastai
|
|
118
|
+
`DataLoaders`, `show_images(dls)` calls fastai's own `show_batch`.
|
|
119
|
+
|
|
120
|
+
## Runs and data
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
run = cu.drive.new_run("resnet34-baseline") # mounts Drive if needed
|
|
124
|
+
print(run) # /content/drive/MyDrive/runs/Lab-3-CNNs_20261012-221500_resnet34-baseline
|
|
125
|
+
learn.path = run.path # fastai: save models into the run folder
|
|
126
|
+
torch.save(model.state_dict(), run / "model.pth")
|
|
127
|
+
|
|
128
|
+
path = cu.drive.data_dir("pets") # /content/data/pets (fast local disk)
|
|
129
|
+
cu.drive.set_config(timezone="America/Chicago", data_dir="/content/data") # saved on Drive
|
|
130
|
+
cu.drive.latest_run() # most recent run folder
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Each run folder contains a `run.json` with the notebook name, timestamp, and the
|
|
134
|
+
runtime's Python, GPU, and package versions. Without a slug, a random one such as
|
|
135
|
+
`brave-otter` is used.
|
|
136
|
+
|
|
137
|
+
Settings come from (highest first): function arguments, environment variables
|
|
138
|
+
(`COLABUTILS_DATA_DIR`, `COLABUTILS_RUNS_DIR`, `COLABUTILS_TZ`,
|
|
139
|
+
`COLABUTILS_NOTEBOOK_NAME`), `MyDrive/.colabutilscs4337/config.json`, then defaults.
|
|
140
|
+
|
|
141
|
+
## Inspecting models
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
cu.tools.summary(learn, input_size=(1, 3, 224, 224)) # fastai Learner or nn.Module
|
|
145
|
+
cu.tools.netron(learn) # interactive graph in the cell output
|
|
146
|
+
cu.tools.netron("model.onnx") # or any file Netron supports
|
|
147
|
+
|
|
148
|
+
model = cu.models.load_model("export.pkl") # fastai learn.export()
|
|
149
|
+
model = cu.models.load_model("model.pth", arch=torchvision.models.resnet18(num_classes=37))
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Visualizing what a CNN looks at
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
cam = cu.viz.gradcam(model, x, class_idx=None) # x: (1, 3, H, W) normalized tensor
|
|
156
|
+
cu.viz.show_cam(image, cam) # image: PIL / HxWx3 array / tensor
|
|
157
|
+
|
|
158
|
+
acts = cu.viz.activations(model, x, layers=["layer1", "layer4"])
|
|
159
|
+
cu.viz.show_feature_maps(acts["layer1"], n=16)
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
`target_layer` defaults to `cu.viz.suggest_target_layer(model)` (the last conv
|
|
163
|
+
layer). With `cu.install("cam")`, `cu.viz.gradcam(..., method="gradcam++")` uses the
|
|
164
|
+
[pytorch-grad-cam](https://github.com/jacobgil/pytorch-grad-cam) backend.
|
|
165
|
+
|
|
166
|
+
## Compatibility with Colab
|
|
167
|
+
|
|
168
|
+
The Colab runtime is tracked from
|
|
169
|
+
[googlecolab/backend-info](https://github.com/googlecolab/backend-info):
|
|
170
|
+
|
|
171
|
+
- `scripts/sync_colab_runtime.py` writes `constraints/colab-gpu.txt` and
|
|
172
|
+
`src/colabutilscs4337/_tested.json`.
|
|
173
|
+
- CI runs the tests inside Google's Colab runtime image
|
|
174
|
+
(`us-docker.pkg.dev/colab-images/public/cpu-runtime`) and checks that
|
|
175
|
+
`pip install ".[all]"` changes no preinstalled package.
|
|
176
|
+
- A weekly job reruns everything and opens a PR when Colab's runtime changes.
|
|
177
|
+
|
|
178
|
+
## Development
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
uv venv --python 3.13
|
|
182
|
+
uv pip install -c constraints/colab-gpu.txt -e ".[all,dev]" torch torchvision fastai
|
|
183
|
+
pytest
|
|
184
|
+
ruff check .
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Build the documentation (Sphinx; PyTorch not needed):
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
uv venv .venv-docs --python 3.13
|
|
191
|
+
uv pip install --python .venv-docs/bin/python -r docs/requirements.txt
|
|
192
|
+
.venv-docs/bin/sphinx-build -W -b html docs docs/_build/html
|
|
193
|
+
open docs/_build/html/index.html
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
`.github/workflows/docs.yml` builds the site on every pull request and publishes it to
|
|
197
|
+
GitHub Pages on every push to `main`. See the
|
|
198
|
+
[development guide](https://txstcodeplayground.github.io/ColabUtilsCS4337/development.html)
|
|
199
|
+
for releases.
|
|
200
|
+
|
|
201
|
+
Release: push a tag `vX.Y.Z`. CI publishes to TestPyPI, then PyPI after approval.
|
|
202
|
+
Before approving, run `notebooks/smoke_test.ipynb` on a Colab GPU runtime.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.24", "hatch-vcs>=0.4"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "colabutilscs4337"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Google Colab utilities for CS4337: Drive run folders, model inspection, and CNN visualization (Grad-CAM, activations)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.12"
|
|
13
|
+
authors = [{ name = "CS4337 Course Staff" }]
|
|
14
|
+
keywords = ["colab", "pytorch", "fastai", "computer-vision", "grad-cam", "education"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Education",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Framework :: Jupyter",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
24
|
+
"Topic :: Scientific/Engineering :: Image Recognition",
|
|
25
|
+
]
|
|
26
|
+
# Colab already provides torch, torchvision, fastai, numpy, matplotlib, pillow, ipywidgets.
|
|
27
|
+
# Declaring them here would let pip replace Colab's builds, so this list stays empty.
|
|
28
|
+
dependencies = []
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
tools = ["torchinfo>=1.8", "netron>=7.0"]
|
|
32
|
+
cam = ["grad-cam>=1.5"]
|
|
33
|
+
fastai = ["fastai>=2.7"]
|
|
34
|
+
all = ["colabutilscs4337[tools,cam]"]
|
|
35
|
+
dev = ["pytest>=8", "ruff>=0.6"]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337"
|
|
39
|
+
Documentation = "https://txstcodeplayground.github.io/ColabUtilsCS4337/"
|
|
40
|
+
Repository = "https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337"
|
|
41
|
+
Issues = "https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337/issues"
|
|
42
|
+
Changelog = "https://github.com/TXSTCODEPLAYGROUND/ColabUtilsCS4337/blob/main/CHANGELOG.md"
|
|
43
|
+
|
|
44
|
+
[tool.hatch.version]
|
|
45
|
+
source = "vcs"
|
|
46
|
+
fallback-version = "0.1.0"
|
|
47
|
+
|
|
48
|
+
[tool.hatch.build.hooks.vcs]
|
|
49
|
+
version-file = "src/colabutilscs4337/_version.py"
|
|
50
|
+
|
|
51
|
+
[tool.hatch.build.targets.wheel]
|
|
52
|
+
packages = ["src/colabutilscs4337"]
|
|
53
|
+
|
|
54
|
+
[tool.hatch.build.targets.sdist]
|
|
55
|
+
include = ["src", "tests", "README.md", "LICENSE", "CHANGELOG.md", "pyproject.toml"]
|
|
56
|
+
|
|
57
|
+
[tool.pytest.ini_options]
|
|
58
|
+
testpaths = ["tests"]
|
|
59
|
+
pythonpath = ["src"]
|
|
60
|
+
addopts = "-ra"
|
|
61
|
+
markers = [
|
|
62
|
+
"gpu: requires a CUDA GPU (run manually in Colab)",
|
|
63
|
+
"colab: requires a real Google Colab runtime",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
[tool.ruff]
|
|
67
|
+
line-length = 100
|
|
68
|
+
target-version = "py312"
|
|
69
|
+
extend-exclude = ["src/colabutilscs4337/_version.py", "notebooks", ".venv-docs", "docs/_build"]
|
|
70
|
+
|
|
71
|
+
[tool.ruff.lint]
|
|
72
|
+
select = ["E", "F", "W", "I", "B", "UP"]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Google Colab utilities for CS4337.
|
|
2
|
+
|
|
3
|
+
Subpackages are imported lazily so ``import colabutilscs4337`` stays fast and never
|
|
4
|
+
imports torch or fastai until a feature that needs them is used::
|
|
5
|
+
|
|
6
|
+
import colabutilscs4337 as cu
|
|
7
|
+
cu.install("tools", "cam")
|
|
8
|
+
cu.env.check()
|
|
9
|
+
run = cu.drive.new_run("baseline")
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import importlib
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
|
|
17
|
+
try:
|
|
18
|
+
from ._version import __version__
|
|
19
|
+
except ImportError: # pragma: no cover - only when running from a source tree without a build
|
|
20
|
+
__version__ = "0.0.0+unknown"
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from . import drive, env, models, tools, viz # noqa: F401
|
|
24
|
+
from ._install import install # noqa: F401
|
|
25
|
+
|
|
26
|
+
_SUBMODULES = {"env", "drive", "tools", "models", "viz"}
|
|
27
|
+
|
|
28
|
+
__all__ = ["__version__", "install", *sorted(_SUBMODULES)]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def __getattr__(name: str) -> Any:
|
|
32
|
+
if name in _SUBMODULES:
|
|
33
|
+
module = importlib.import_module(f".{name}", __name__)
|
|
34
|
+
globals()[name] = module
|
|
35
|
+
return module
|
|
36
|
+
if name == "install":
|
|
37
|
+
from ._install import install
|
|
38
|
+
|
|
39
|
+
globals()["install"] = install
|
|
40
|
+
return install
|
|
41
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def __dir__() -> list[str]:
|
|
45
|
+
return sorted(set(globals()) | set(__all__))
|