eoml 0.9.0__py3-none-any.whl

eoml/torch/trainer_v2.py

@@ -0,0 +1,253 @@
+from abc import ABC
+
+from eoml import torch
+from torchmetrics import F1Score
+
+
+class Score(ABC):
+
+    def __init__(self):
+        pass
+
+    def __call__(self):
+        pass
+
+    def direction(self):
+        pass
+
+    def is_last_best(self):
+        pass
+
+    @property
+    def best(self):
+        return 0
+
+class F1MultiClass(Score):
+    def __init__(self, num_class, average="macro", device="cpu"):
+        #https://stephenallwright.com/micro-vs-macro-f1-score/
+        self._best = float("inf")
+
+        self.score = F1Score(task="multiclass", average=average, num_classes=num_class).to(device)
+
+
+    def __call__(self, output, target):
+        self.score(output, target)
+
+    def direction(self):
+        pass
+
+    def is_last_best(self):
+        pass
+
+    def best(self):
+        return self._best
+
+class F1_Score(ABC):
+
+    def __init__(self):
+        pass
+
+    def __call__(self):
+        pass
+
+class Trainer:
+    """TODO DO AGGRESSIVBE VERSION"""
+    def __init__(self, optimizer, model, loss_fn,  grad_f=None, score_function=f1, score_name="f1", score_direction=1):
+        self.optimizer = optimizer
+        self.model = model
+        self.loss_fn = loss_fn
+        self.grad_f = grad_f
+
+        self.score_direction = score_direction
+
+        self.writer = None
+
+        self.score_function = score_function
+        self.score_name = score_name
+
+    def _epoch(self, loader, epoch_index, report_frequency, device="cpu"):
+
+        """
+        :param loader:
+        :param epoch_index:
+        :param report_frequency:
+        :param device: device to move tensors to. None for do nothing
+        :return:
+        """
+
+        # Make sure gradient tracking is on, and do a pass over the data
+        self.model.train(True)
+
+        running_loss = 0.
+        last_loss = 0.
+        with tqdm(total=len(loader),desc="Batch") as pbar:
+            for i, data in enumerate(loader):
+                # Every data instance is an input + label pair
+                inputs, labels = data
+
+                if device is not None:
+                    inputs = inputs.to(device, non_blocking=True)
+                    labels = labels.to(device, non_blocking=True)
+                # Zero your gradients for every batch!
+                self.optimizer.zero_grad()
+
+                # Make predictions for this batch
+                outputs = self.model(inputs)
+
+                # Compute the loss and its gradients
+                loss = self.loss_fn(outputs, labels)
+                loss.backward()
+
+                # clip the gradient
+                if self.grad_f is not None:
+                    self.grad_f(self.model)
+
+                # Adjust learning weights
+                self.optimizer.step()
+
+                # Gather data and report
+                running_loss += loss.item()
+                if i % report_frequency == report_frequency - 1:
+                    pbar.set_postfix({'Batch ': i + 1,
+                                      'Last loss': last_loss,
+                                      }, refresh=False)
+                    pbar.update(report_frequency)
+
+                    last_loss = running_loss / report_frequency  # loss per item
+                    #print('  batch {} loss: {}'.format(i + 1, last_loss))
+                    tb_x = epoch_index * len(loader) + i + 1
+                    self.writer.add_scalar('Loss/train', last_loss, tb_x)
+                    running_loss = 0.
+
+        return last_loss
+
+    def _validate(self, validation_loader, device):
+
+        self.model.train(False)
+
+        running_vloss = 0.0
+        running_score = 0.0
+
+        for i, vdata in enumerate(validation_loader):
+            vinputs, vlabels = vdata
+
+            if device is not None:
+                vinputs = vinputs.to(device, non_blocking=True)
+                vlabels = vlabels.to(device, non_blocking=True)
+
+            voutputs = self.model(vinputs)
+            vloss = self.loss_fn(voutputs, vlabels)
+            running_vloss += vloss.item()
+
+            vf1 = self.score_function(voutputs.cpu(), vlabels.cpu())
+            running_score += vf1
+
+        avg_vloss = running_vloss / (i + 1)
+        avg_score = running_score / (i + 1)
+        # print('LOSS train {} valid {}'.format(avg_loss, avg_vloss))
+        # print('Weighted avg f1 {}'.format(avg_f1))
+        return  avg_vloss, avg_score
+
+    def train(self, epochs, training_loader, validation_loader, report_per_epoch=10,
+              writer_base_path="runs", model_base_path=".", model_tag="model", device="cpu"):
+        # Initializing in a separate cell so we can easily add more epochs to the same run
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        self.writer = SummaryWriter(f"{writer_base_path}/{model_tag}_{timestamp}")
+
+        model_name = f"{model_tag}_{timestamp}"
+        base_dir = f"{model_base_path}/{model_name}"
+        os.mkdir(base_dir)
+
+        n_batch = len(training_loader)
+
+        report_frequency = math.ceil(n_batch / report_per_epoch)
+
+        best_score_epoch = 0.
+        if self.score_direction == -1:
+            best_score = 1_000_000
+        else:
+            best_score = 0
+
+
+        best_vloss = 1_000_000.
+        best_vloss_epoch = 0
+
+        best_epoch = 0
+
+        model_path = None
+
+        with tqdm(total=epochs, desc='Epoch') as pbar:
+            for epoch in range(epochs):
+
+                #print('EPOCH {}:'.format(epoch_number + 1))
+                avg_loss = self._epoch(training_loader, epoch, report_frequency, device)
+
+                # We don't need gradients on to do reporting
+
+                avg_vloss, avg_score = self._validate(validation_loader, device)
+
+                # Log the running loss averaged per batch
+                # for both training and validation
+                self.writer.add_scalars('Training vs. Validation Loss',
+                                        {'Training': avg_loss, 'Validation': avg_vloss},
+                                        epoch + 1)
+                self.writer.add_scalars(f'Weighted avg {self.score_name}',
+                                        {f'Weighted avg {self.score_name}': avg_score},
+                                        epoch + 1)
+
+                # todo f1 for all v batch at once
+
+                self.writer.flush()
+
+
+                # Track the best performance, and save the model's state
+                if avg_vloss < best_vloss:
+                    best_vloss = avg_vloss
+                    best_vloss_epoch = epoch + 1
+                    best_epoch = best_vloss_epoch
+                    best_metric = "loss"
+                    model_path = f'{base_dir}/{best_metric}_{model_tag}_{timestamp}_{best_vloss_epoch}'
+                    torch.save(self.model.state_dict(), model_path)
+
+                if self.score_direction * avg_score > self.score_direction*best_score:
+                    best_score = avg_score
+                    best_score_epoch = epoch+1
+                    best_epoch = best_score_epoch
+                    best_metric = self.score_name
+                    model_path = f'{base_dir}/{best_metric}_{model_tag}_{timestamp}_{best_score_epoch}'
+                    torch.save(self.model.state_dict(), model_path)
+
+
+                pbar.set_postfix({f'best {self.score_name} epoch': best_score_epoch,
+                                  f'best {self.score_name}': best_score,
+                                  f'current {self.score_name}': avg_score,
+                                  'best avg loss epoch': best_vloss_epoch,
+                                  'best avg loss': best_vloss,
+                                  'current avg loss': avg_vloss}, refresh=False)
+                pbar.update(1)
+
+
+        # load best model
+        self.model.load_state_dict(torch.load(model_path))
+        #switch off training
+        self.model.train(False)
+        # git model for inference
+
+        vinputs, _ = next(iter(validation_loader))
+
+        if device is not None:
+            vinputs = vinputs.to(device)
+
+        # switch off gradient
+        #todo update py torch
+        #torch.jit.enable_onednn_fusion(True)
+        with torch.inference_mode():
+            #model_scripted = torch.jit.script(model, example_inputs=vinputs)  # Export to TorchScript, from the doc: TorchScript is actually the recommended model format for scaled inference and deployment.
+            model_scripted = torch.jit.trace(self.model, example_inputs=vinputs)
+            model_scripted = torch.jit.freeze(model_scripted)
+
+            model_path = f'{base_dir}/jited_{best_metric}_{model_tag}_{timestamp}_{best_epoch}.pt'
+            model_scripted.save(model_path)  # Save
+
+        return base_dir, model_path, model_name
+

eoml-0.9.0.dist-info/METADATA

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: eoml
+Version: 0.9.0
+Summary: library to manage GIS operation
+Project-URL: Homepage, https://ciatgit.ciat.cgiar.org/Data_driven_sustainability_public/terra-i/eoml#
+Project-URL: Documentation, https://ciatgit.ciat.cgiar.org/Data_driven_sustainability_public/terra-i/eoml#
+Project-URL: Repository, https://ciatgit.ciat.cgiar.org/Data_driven_sustainability_public/terra-i/eoml#
+Project-URL: Bug Tracker, https://ciatgit.ciat.cgiar.org/Data_driven_sustainability_public/terra-i/eoml/-/issues
+Author-email: Thibaud Vantalon <t.vantalon@cgiar.org>
+Maintainer-email: Thibaud Vantalon <t.vantalon@cgiar.org>
+License: MIT License
+Keywords: GIS,Rasterio
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Requires-Python: >=3.12
+Requires-Dist: fiona
+Requires-Dist: geopandas
+Requires-Dist: lmdb
+Requires-Dist: msgpack
+Requires-Dist: numpy
+Requires-Dist: pydantic>=2.6
+Requires-Dist: pyproj
+Requires-Dist: rasterio
+Requires-Dist: rasterstats
+Requires-Dist: scikit-learn
+Requires-Dist: shapely
+Requires-Dist: tensorboard
+Requires-Dist: toml
+Requires-Dist: tomli
+Requires-Dist: torchmetrics
+Requires-Dist: tqdm
+Requires-Dist: typer
+Description-Content-Type: text/markdown
+
+# EOML - Earth Observation Machine Learning
+
+A Python library for managing GIS operations and machine learning workflows for remote sensing applications.
+
+## Overview
+
+EOML provides a comprehensive toolkit for processing Earth observation data and building machine learning models for 
+satellite imagery analysis. The library integrates rasterio, PyTorch, and Google Earth Engine to streamline geospatial
+machine learning workflows.
+
+## Features
+
+- **PyTorch Integration**: Pre-built CNN architectures and training utilities for remote sensing
+
+
+## Installation 
+
+### PyPI
+pip install eoml
+
+### Developement mode
+Installation in development mode:
+```bash
+pip install -e .
+```
+
+
+
+### Running Tests
+
+```bash
+pytest tests/
+```
+
+## Contributing
+
+Contributions are welcome! Please ensure code follows the project style and includes appropriate docstrings.
+
+## License
+MIT License
+
+## Author
+
+**Thibaud Vantalon**
+Email: t.vantalon@cgiar.org
+Organization: CGIAR
+
+## Citation
+
+If you use this library in your research, please cite:
+
+```bibtex
+@software{eoml,
+  author = {Vantalon, Thibaud},
+  title = {EOML: Earth Observation Machine Learning},
+  year = {2024},
+  url = {https://ciatgit.ciat.cgiar.org/Data_driven_sustainability_public/terra-i/eoml#}
+}
+```

eoml-0.9.0.dist-info/RECORD

@@ -0,0 +1,47 @@
+eoml/__init__.py,sha256=ZNIQkoMq2XYKy8R4nzCKMrJ2NJl3lsAbCZzTfh9RSHc,2448
+eoml/automation/__init__.py,sha256=jBxHaR8zlPEow2QHmqIA3AWhZwmUPmEny90z3AXZBKA,241
+eoml/automation/configuration.py,sha256=Ii7CQG5RNQDubH35rtKkvqgQ69_vZk13xRUJm8ttyfE,3474
+eoml/automation/dag.py,sha256=_g-YP6nKW9MeYBtGMEURjKMKoi9l29LFw3X6Uzqc4qE,6469
+eoml/automation/experience.py,sha256=TKi3jsZPKsWRpBPHuaaUIXG-G_-mZQ2mIjTe28ZRlGM,19822
+eoml/automation/tasks.py,sha256=ALYROfvTIUPY7QmOEyfk4PpjluGx7rfUWR0YJ75ddxM,32283
+eoml/bin/__init__.py,sha256=y0ukkpx3o4fGFWE289g14JF_UIA_Z3jMI9uGJBt58aU,159
+eoml/bin/clean_checkpoint.py,sha256=PWgSsaY_9DNxy_knYpTrqVWLvtymh15G35l49k0CT-A,4070
+eoml/bin/land_cover_mapping_toml.py,sha256=lIak0jLjZ8wtWRcdlO5ngU7LpcLkoeDgt4vHhS23Xjg,16471
+eoml/bin/mosaic_images.py,sha256=yAKUwXlJY80heVLQ87if3aq8dh_yPn6Un6j1mwyP2v8,4432
+eoml/data/__init__.py,sha256=2zBgj0doFdn03fsTlrozad2JrKKE_0wMBPORit5shM8,228
+eoml/data/basic_geo_data.py,sha256=vaiQfnzxgR7UhXGnl2SarnDJw-aPXFlNBQ_zCs9QwWk,6617
+eoml/data/dataset_utils.py,sha256=Mn5XojdgVGEcndSgNMU8NEgDMpVUl6BhAlXhugTLPA8,3209
+eoml/data/persistence/__init__.py,sha256=mohqYLULxrXtU-75dR3slnZpXkCyMBXLa1V6HEewnkU,254
+eoml/data/persistence/generic.py,sha256=661dQKcS7bnN_2WgmVdPa_3sgzCkKUkltHMYrN9UFU4,6665
+eoml/data/persistence/lmdb.py,sha256=IjUUHxbwh8Q2yq6SfJH050wUmDnz7DxgCRNdKn1yiBE,14128
+eoml/data/persistence/serializer.py,sha256=pWGuCnGNyqD1We5fEV-tN8u7xjiOWap9z47QS_AJfcs,2997
+eoml/raster/__init__.py,sha256=AB9Y8A7gq-UvsYJ0lHLD18omQZ6wxYwHjxsK6x7sMpQ,248
+eoml/raster/band.py,sha256=EFQCab6MyHtVdnvHBOFH4lXsrm4LRQToknCWmu0Ai6U,3905
+eoml/raster/raster_reader.py,sha256=V3AnyTq_kYRINhQCOB-Zr7tlcMwSB8eS0H9xasg6rj8,20899
+eoml/raster/raster_utils.py,sha256=LgXyJqg0PqVUzPfHfXv9mOMEFaryJv1qi2MkygM9CNI,3931
+eoml/raster/dataset/__init__.py,sha256=Bezfq4avYoBWXjca6bWDDJiemhVY3dYcww4HmHo75rk,186
+eoml/raster/dataset/extractor.py,sha256=B96_WRN5eVkSmeujpIqZrAKcLgH9wbF3nYC0hLUSZtI,22512
+eoml/torch/__init__.py,sha256=82HZiOUYEy6fqbAGYLKrdptutql-Xc5M7ts0Z4HZNKg,237
+eoml/torch/model_low_use.py,sha256=eGKv1iAZp_2509N2tfMF9q_1FRPEsKs-mLoSHnbEiAM,18416
+eoml/torch/models.py,sha256=s9J5fjurunFon6WkeW78X1E8fCqNzPL5unGQJP1EP5A,10568
+eoml/torch/resnet.py,sha256=fs9kZykkshKF4B5qnir96LUezKCSrET48zmO5HAem9Y,16332
+eoml/torch/sample_statistic.py,sha256=pUcYkf6c7BFXhNq4z0UDlNyosvrxvWP8Ep-oUlrENwA,10489
+eoml/torch/trainer.py,sha256=h46fQ9pekHi5CCenhEXGYS_HSjeseL2SNRKL9zaEMG0,28561
+eoml/torch/trainer_v2.py,sha256=jTQ2FpU1jY-x1S0NNZqNPcKjFQGeVuhVM7IVdqwB9-8,8655
+eoml/torch/cnn/__init__.py,sha256=aczbA04QW_pGDxOHoMqLU-vUNiibyBG82HMDjw5CFWQ,246
+eoml/torch/cnn/augmentation.py,sha256=HkRNHRBeysjqulGPJHRnyYyhrPu6dkggsIRcZySkUxA,4779
+eoml/torch/cnn/dataset_evaluator.py,sha256=MucZBc0iCbB2jrRXfbGnGfnqoYvu3JUlxxB8gKvlXJs,2100
+eoml/torch/cnn/db_dataset.py,sha256=EttHGEOh8XSL-MKRkWLArPQUB6ZZJ0tGiNeyr454aJg,21170
+eoml/torch/cnn/map_dataset.py,sha256=49xQjzR4D9qe5TgADGKKwfNbjlYdL5qM6T7zvGZ2TIE,21008
+eoml/torch/cnn/map_dataset_const_mem.py,sha256=fKgA8X8Vd_3aeFLgnZhQdDVy5JKOgaE0X2qSIkt3fn4,4901
+eoml/torch/cnn/outputs_transformer.py,sha256=0852oWf86Dlq_1942g-Hil9sSXj3vkqa_lmhTXmwsVU,3674
+eoml/torch/cnn/torch_utils.py,sha256=ThKoiF812CZgfc3J7XY47-QoD2dmbmihOb5Un0_rPUc,12661
+eoml/torch/cnn/training_dataset.py,sha256=ncLAIEKe1hXAh7TWZhgfVQtoCdG2c2wrXipU91Q9FrM,8607
+eoml/torch/cnn/windows_dataset.py,sha256=--kWNGRBO86oYSY3JoHIEYFC4_vBTpfI7AdWAImqEuA,3758
+eoml/torch/dataset/__init__.py,sha256=oFoDD6oeFyws8wvoOrHEBvIqJQTlTvYU9OPuiDOWB3o,178
+eoml/torch/dataset/shade_dataset_tester.py,sha256=UZaOKl-twF_74r8I9rxFVchq7t2HQzlmm7_eHIRjDYw,1512
+eoml/torch/dataset/shade_tree_dataset_creators.py,sha256=afqhJzwQcEytpdqflThb2tDWbIn6HiigJxIF0EfaqUw,21183
+eoml-0.9.0.dist-info/METADATA,sha256=CGaytOPlGtq-X0Uhfm9O9LZwmNBz61bGqOyBaf2brpo,2472
+eoml-0.9.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+eoml-0.9.0.dist-info/entry_points.txt,sha256=QmOKUZQNv8HGTC9J1Q-O_Nsdnhy8a0-6mOfHWzoYpEY,115
+eoml-0.9.0.dist-info/RECORD,,

eoml-0.9.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

eoml-0.9.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+eoml_map = eoml.bin.land_cover_mapping_toml:main
+eoml_mosaic_images = eoml.bin.mosaic_images:app