eotdl 2024.8.21__py3-none-any.whl → 2024.10.7__py3-none-any.whl

eotdl/__init__.py

@@ -1 +1 @@
-__version__ = "2024.08.21"
+__version__ = "2024.10.07"

eotdl/datasets/ingest.py

@@ -3,8 +3,6 @@ import yaml
 from tqdm import tqdm
 import json
 import frontmatter
-import markdown
-from markdown.extensions.fenced_code import FencedCodeExtension
 
 from ..auth import with_auth
 from .metadata import Metadata
@@ -73,8 +71,7 @@ def ingest_folder(
         raise Exception("Error loading metadata")
     # retrieve dataset (create if doesn't exist)
     dataset = retrieve_dataset(metadata, user)
-    if content:
-        content = markdown.markdown(content, extensions=[FencedCodeExtension()])
+
     update_metadata = True
     if "description" in dataset:
         # do not do this if the dataset is new, only if it already exists

eotdl/datasets/metadata.py

@@ -1,6 +1,5 @@
 from pydantic import BaseModel, validator
 from typing import List, Optional
-import markdownify
 from pathlib import Path
 
 
@@ -39,7 +38,7 @@ def generate_metadata(download_path, dataset):
         for author in dataset["authors"]:
             f.write(f"  - {author}\n")
         f.write("---\n")
-        f.write(markdownify.markdownify(dataset["description"], heading_style="ATX"))
+        f.write(dataset["description"])
     # remove metadata.yml if exists
     if Path(download_path + "/metadata.yml").exists():
         Path(download_path + "/metadata.yml").unlink()

eotdl/models/download.py

@@ -74,9 +74,12 @@ def download_model(
         )
         if error:
             raise Exception(error)
+        print(gdf)
         df = STACDataFrame(gdf)
         # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
         df.to_stac(download_path)
+        print("----")
+        print(df)
         # download assets
         if assets:
             if verbose:

eotdl/models/ingest.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 import yaml
 import frontmatter
-import markdown
 from tqdm import tqdm
 import json
 
@@ -67,8 +66,7 @@ def ingest_folder(
         raise Exception(f"Error loading metadata: {e}")
     # retrieve model (create if doesn't exist)
     model = retrieve_model(metadata, user)
-    if content:
-        content = markdown.markdown(content)
+
     update_metadata = True
     if "description" in model:
         # do not do this if the model is new, only if it already exists

eotdl/models/metadata.py

@@ -1,6 +1,5 @@
 from pydantic import BaseModel, validator
 from typing import List, Optional
-import markdownify
 from pathlib import Path
 
 
@@ -37,7 +36,7 @@ def generate_metadata(download_path, model):
         for author in model["authors"]:
             f.write(f"  - {author}\n")
         f.write("---\n")
-        f.write(markdownify.markdownify(model["description"], heading_style="ATX"))
+        f.write(model["description"])
     # remove metadata.yml if exists
     if Path(download_path + "/metadata.yml").exists():
         Path(download_path + "/metadata.yml").unlink()

eotdl/wrappers/__init__.py

@@ -0,0 +1 @@
+from .models import ModelWrapper

eotdl/wrappers/models.py

@@ -0,0 +1,158 @@
+# Q1+ model wrapper
+# only works with some models, extend as we include more models in EOTDL and improve MLM extension
+
+import os 
+from pathlib import Path
+from tqdm import tqdm
+import numpy as np
+
+from ..models.retrieve import retrieve_model
+from ..curation.stac import STACDataFrame
+from ..repos import FilesAPIRepo, ModelsAPIRepo
+from ..auth import with_auth
+
+class ModelWrapper:
+    def __init__(self, model_name, version=None, path=None, force=False, assets=True, verbose=True):
+        self.model_name = model_name
+        self.version = version
+        self.path = path
+        self.force = force
+        self.assets = assets
+        self.verbose = verbose
+        self.ready = False
+        self.setup()
+
+    def setup(self):
+        download_path, gdf = self.download()
+        self.download_path = download_path
+        self.gdf = gdf
+        # get model name from stac metadata
+        item = gdf[gdf['type'] == "Feature"]
+        assert item.shape[0] == 1, "Only one item is supported in stac metadata, found " + str(item.shape[0])
+        self.props = item.iloc[0].properties
+        assert self.props["mlm:framework"] == "ONNX", "Only ONNX models are supported, found " + self.props["mlm:framework"]
+        model_name = self.props["mlm:name"]
+        self.model_path = download_path + '/assets/' + model_name
+        self.ready = True
+
+    def predict(self, x):
+        if not self.ready:
+            self.setup()
+        ort_session = self.get_onnx_session(self.model_path)
+        # preprocess input
+        x = self.process_inputs(x)
+        # execute model
+        input_name = ort_session.get_inputs()[0].name
+        ort_inputs = {input_name: x}
+        ort_outs = ort_session.run(None, ort_inputs)
+        output_nodes = ort_session.get_outputs()
+        output_names = [node.name for node in output_nodes]
+        # format and return outputs
+        return self.return_outputs(ort_outs, output_names)
+
+    @with_auth
+    def download(self, user=None):
+        # download the model
+        model = retrieve_model(self.model_name)
+        if model["quality"] == 0:
+            raise Exception("Only Q1+ models are supported")
+        if self.version is None:
+            self.version = sorted(model["versions"], key=lambda v: v["version_id"])[-1][
+                "version_id"
+            ]
+        else:
+            assert self.version in [
+                v["version_id"] for v in model["versions"]
+            ], f"Version {self.version} not found"
+        download_base_path = os.getenv(
+            "EOTDL_DOWNLOAD_PATH", str(Path.home()) + "/.cache/eotdl/models"
+        )
+        if self.path is None:
+            download_path = download_base_path + "/" + self.model_name + "/v" + str(self.version)
+        else:
+            download_path = self.path + "/" + self.model_name + "/v" + str(self.version)
+        # check if model already exists
+        if os.path.exists(download_path) and not self.force:
+            os.makedirs(download_path, exist_ok=True)
+            gdf = STACDataFrame.from_stac_file(download_path + f"/{self.model_name}/catalog.json")
+            return download_path, gdf
+        if self.verbose:
+            print("Downloading STAC metadata...")
+        repo = ModelsAPIRepo()
+        gdf, error = repo.download_stac(
+            model["id"],
+            user,
+        )
+        if error:
+            raise Exception(error)
+        df = STACDataFrame(gdf)
+        # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
+        df.to_stac(download_path)
+        # download assets
+        if self.assets:
+            if self.verbose:
+                print("Downloading assets...")
+            repo = FilesAPIRepo()
+            df = df.dropna(subset=["assets"])
+            for row in tqdm(df.iterrows(), total=len(df)):
+                for k, v in row[1]["assets"].items():
+                    href = v["href"]
+                    _, filename = href.split("/download/")
+                    # will overwrite assets with same name :(
+                    repo.download_file_url(
+                        href, filename, f"{download_path}/assets", user
+                    )
+        else:
+            print("To download assets, set assets=True.")
+        if self.verbose:
+            print("Done")
+        return download_path, gdf
+
+    def process_inputs(self, x):
+        # pre-process and validate input
+        input = self.props["mlm:input"]
+        # input data type
+        dtype = input["input"]["data_type"]
+        x = x.astype(dtype)
+        # input shape
+        input_shape = input["input"]["shape"]
+        ndims = len(input_shape)
+        if ndims != x.ndim:
+            if ndims == 4:
+                x = np.expand_dims(x, axis=0).astype(np.float32)
+            else:
+                raise Exception("Input shape not valid", input_shape, x.ndim)
+        for i, dim in enumerate(input_shape):
+            if dim != -1:
+                assert dim == x.shape[i], f"Input dimension not valid: The model expects {input_shape} but input has {x.shape} (-1 means any dimension)."
+        # TODO: should apply normalization if defined in metadata
+        return x
+    
+    def return_outputs(self, ort_outputs, output_names):
+        if self.props["mlm:output"]["tasks"] == ["classification"]:
+            return {
+                "model": self.model_name,
+                **{
+                    output: ort_outputs[i].tolist() for i, output in enumerate(output_names)
+                },
+            }
+        elif self.props["mlm:output"]["tasks"] == ["segmentation"]:
+            outputs = {output: ort_outputs[i] for i, output in enumerate(output_names)}
+            batch = outputs[output_names[0]]
+            image = batch[0]
+            return image
+        else:
+            raise Exception("Output task not supported:", self.props["mlm:output"]["tasks"])
+
+    def get_onnx_session(self, model):        
+        try:
+            import onnxruntime as ort
+            # gpu requires `pip install onnxruntime-gpu` but no extra imports
+        except ImportError:
+            raise ImportError("onnxruntime is not installed. Please install it with `pip install onnxruntime`")
+        providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+        try:
+            session = ort.InferenceSession(model, providers=providers)
+        except Exception as e:
+            raise RuntimeError(f"Error loading ONNX model: {str(e)}")
+        return session

eotdl/wrappers/utils.py

@@ -0,0 +1,35 @@
+from ..curation.stac import STACDataFrame
+
+def download_model(model_name, dst_path, version, force=False, download=True):
+    # check if model already downloaded
+    version = 1 if version is None else version
+    download_path = dst_path + "/" + model_name + "/v" + str(version)
+    if os.path.exists(download_path) and not force:
+        df = STACDataFrame.from_stac_file(download_path + f"/{model_name}/catalog.json")
+        return download_path, df
+    # check model exists
+    model, error = retrieve_model(model_name)
+    if error:
+        raise Exception(error)
+    if model["quality"] < 2:
+        raise Exception("Only Q2+ models are supported")
+    # check version exist
+    assert version in [
+        v["version_id"] for v in model["versions"]
+    ], f"Version {version} not found"
+    # download model files
+    gdf, error = retrieve_model_stac(model["id"], version)
+    if error:
+        raise Exception(error)
+    df = STACDataFrame(gdf)
+    if not download:
+        return download_path, df
+    os.makedirs(download_path, exist_ok=True)
+    df.to_stac(download_path)
+    df = df.dropna(subset=["assets"])
+    for row in df.iterrows():
+        for k, v in row[1]["assets"].items():
+            href = v["href"]
+            _, filename = href.split("/download/")
+            download_file_url(href, filename, f"{download_path}/assets")
+    return download_path, df

{eotdl-2024.8.21.dist-info → eotdl-2024.10.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2024.8.21
+Version: 2024.10.7
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: black (>=23.10.1,<24.0.0)
 Requires-Dist: geomet (>=1.0.0,<2.0.0)
 Requires-Dist: geopandas (>=0.13.2,<0.14.0)

{eotdl-2024.8.21.dist-info → eotdl-2024.10.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-eotdl/__init__.py,sha256=n2OwrPMIbMfYBFKreUbRkS6EzKE96Cyf_7fANIHPTf8,27
+eotdl/__init__.py,sha256=9dJlE6aniPM75bpVKz06wufJdjPqaOg_opOHL9R4swI,27
 eotdl/access/__init__.py,sha256=jbyjD7BRGJURlTNmtcbBBhw3Xk4EiZvkqmEykM-bJ1k,231
 eotdl/access/airbus/__init__.py,sha256=G_kkRS9eFjXbQ-aehmTLXeAxh7zpAxz_rgB7J_w0NRg,107
 eotdl/access/airbus/client.py,sha256=zjfgB_NTsCCIszoQesYkyLJgheKg-eTh28vbleXYxfw,12018
@@ -44,17 +44,17 @@ eotdl/curation/stac/parsers.py,sha256=H5IukLA61mpLojeuhWNQdiZk2eiYHAfpJBFxmjdGDs
 eotdl/curation/stac/stac.py,sha256=9GdiB1mV8iyREvXZoJmNJKQGawWyK3h9Eu0hJX5u6O4,13180
 eotdl/datasets/__init__.py,sha256=xNbkUjqI1mf0TGjy52tpZtr5DnWNX5EVgyM04P4sU3Y,175
 eotdl/datasets/download.py,sha256=JSOi7OKiENYQ3TRVj2WsC1CPYD22VlC9W5kYHPfx8Vs,3667
-eotdl/datasets/ingest.py,sha256=ketP5ZElDRHbUjBSLowiAjsnjVKjiqQU40DBkw0rBsA,5918
-eotdl/datasets/metadata.py,sha256=C2LSgVo0SNjEaHP8rXW1JQo4kW0aZ7MLoGa81wMRwTc,1497
+eotdl/datasets/ingest.py,sha256=F7mwQQok3rqZkGiQC-71Jc4dYDN5OSB_SXEACdCZKkk,5742
+eotdl/datasets/metadata.py,sha256=jRoW9n32sUQgX3439WUCL8BJZojXKKXpmCM344XMDMc,1432
 eotdl/datasets/retrieve.py,sha256=DJz5K1bCLizg9YNwBnhHMFzcxMXar2socYkFONdSL4c,1041
 eotdl/datasets/update.py,sha256=x-rpfxnavn9X-7QYkFMGtbn1b3bKmAZydOeS7Tjr5AQ,386
 eotdl/files/__init__.py,sha256=2zfAxgLHmU_jWq_3emnfPXsX-R20gSt-yZX0bPa9h0g,87
 eotdl/files/ingest.py,sha256=dgjZfd-ACCKradDo2B02CPahwEhFtWvnKvTm372K5eo,6185
 eotdl/files/list_files.py,sha256=k4OgdbQ7u6tUEE9nJZGXw5s5HtvG0ZApOVTy0KbfTqs,519
 eotdl/models/__init__.py,sha256=-PvGWG0iSRNBqeFWpxol12dYw-QodXjR81n-JX3x6zI,146
-eotdl/models/download.py,sha256=4dgxE9ytT8QqiCyx1r19vL5UASfttUN_mCPKhFwMTfs,3410
-eotdl/models/ingest.py,sha256=8xhGlsADi5dZSNbph4WWMk0cs2J_StxNCYRkSjMhUtg,5747
-eotdl/models/metadata.py,sha256=S5bpIB4e2pivDnReszJKC3bYBZcaHu-KMYOc3AwHbQ4,1443
+eotdl/models/download.py,sha256=ppCeKsK2YzdimYRKXiF5dfGI6VyXw_T9lOnKO54Ojjg,3469
+eotdl/models/ingest.py,sha256=ipWNSyL8WwW9PPUAFcS9mLYow_1PTCpNQ_DsXKQ3Hvw,5671
+eotdl/models/metadata.py,sha256=Lu_Vv6YJJ9aLPvJa44tFfbBZiyxweOI2BHfq00aNRjY,1378
 eotdl/models/retrieve.py,sha256=-Ij7dT4J1p7MW4n13OlPB9OW4tBaBXPwk9dW8IuCZPc,664
 eotdl/models/update.py,sha256=4FWeD95cXvRpefRjw3Foqb30e30otxqWUZ6nQM9cbmM,374
 eotdl/repos/APIRepo.py,sha256=dJNdX3atBpug0FZNako7uyom25iccNPQrnoIRTIZEXE,791
@@ -73,7 +73,10 @@ eotdl/tools/paths.py,sha256=yWhOtVxX4NxrDrrBX2fuye5N1mAqrxXFy_eA7dffd84,1152
 eotdl/tools/stac.py,sha256=ovXdrPm4Sn9AAJmrP88WnxDmq2Ut-xPoscjphxz3Iyo,5763
 eotdl/tools/time_utils.py,sha256=qJ3-rk1I7ne722SLfAP6-59kahQ0vLQqIf9VpOi0Kpg,4691
 eotdl/tools/tools.py,sha256=Tl4_v2ejkQo_zyZek8oofJwoYcdVosdOwW1C0lvWaNM,6354
-eotdl-2024.8.21.dist-info/METADATA,sha256=ZNq8rxclA_OIzKxaR1VHpWqgtU7LtXw1-SwO1LTe1us,4093
-eotdl-2024.8.21.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-eotdl-2024.8.21.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
-eotdl-2024.8.21.dist-info/RECORD,,
+eotdl/wrappers/__init__.py,sha256=IY3DK_5LMbc5bIQFleQA9kzFbPhWuTLesJ8dwfvpkdA,32
+eotdl/wrappers/models.py,sha256=kNO4pYw9KKKmElE7bZWWHGs7FIThNUXj8XciKh_3rNw,6432
+eotdl/wrappers/utils.py,sha256=BoG1Gtt1jXM_62BD6amrrw9jE25ki-zyY468LVgK0gM,1379
+eotdl-2024.10.7.dist-info/METADATA,sha256=17YF30YXxtAE8jCX9-zKEyyrDPTKxjkYFbCj0qf4Gk0,4144
+eotdl-2024.10.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+eotdl-2024.10.7.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
+eotdl-2024.10.7.dist-info/RECORD,,

{eotdl-2024.8.21.dist-info → eotdl-2024.10.7.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.6.1
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any