eotdl 2023.6.14.post7__tar.gz → 2023.6.14.post9__tar.gz

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2023.6.14.post7
+Version: 2023.6.14.post9
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse

eotdl-2023.6.14.post9/eotdl/auth/__init__.py

@@ -0,0 +1 @@
+from .main import is_logged, auth, generate_logout_url, with_auth

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/auth/main.py

@@ -1,7 +1,5 @@
-from src.repos import AuthRepo, APIRepo
-from src.usecases.auth.IsLogged import IsLogged
-from src.usecases.auth.Auth import Auth
-from src.usecases.auth.Logout import Logout
+from ..src.repos import AuthRepo, APIRepo
+from ..src.usecases.auth import IsLogged, Auth, Logout
 
 
 def is_logged():
@@ -31,3 +29,12 @@ def generate_logout_url():
     inputs = _logout.Inputs()
     outputs = _logout(inputs)
     return outputs.logout_url
+
+
+# auth decorator
+def with_auth(func):
+    def wrapper(*args, **kwargs):
+        user = auth()
+        return func(*args, **kwargs, user=user)
+
+    return wrapper

eotdl-2023.6.14.post9/eotdl/cli.py

@@ -0,0 +1,11 @@
+import typer
+from .commands import auth, datasets
+
+app = typer.Typer()
+
+app.add_typer(auth.app, name="auth")
+app.add_typer(datasets.app, name="datasets")
+
+
+if __name__ == "__main__":
+    app()

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/commands/auth.py

@@ -1,6 +1,6 @@
 import typer
-from auth import is_logged, auth, generate_logout_url
-from src.errors.auth import LoginError
+from ..auth import is_logged, auth, generate_logout_url
+from ..src.errors.auth import LoginError
 
 app = typer.Typer()
 

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/commands/datasets.py

@@ -1,12 +1,12 @@
 import typer
-from datasets import (
+from ..datasets import (
     retrieve_datasets,
     download_dataset,
     update_dataset,
     ingest_large_dataset,
     # ingest_large_dataset_parallel,
 )
-from auth import auth
+from .auth import auth
 
 app = typer.Typer()
 
@@ -29,7 +29,6 @@ def get(name: str, path: str = None):
     path: Path to download the dataset to
     """
     try:
-        user = auth()
         dst_path = download_dataset(name, path, user, typer.echo)
         typer.echo(f"Dataset {name} downloaded to {dst_path}")
     except Exception as e:
@@ -49,10 +48,9 @@ def ingest(
     n: Name of the dataset
     """
     try:
-        user = auth()
         # if p:
         #     ingest_large_dataset_parallel(name, path, user, p, typer.echo)
-        ingest_large_dataset(name, path, user, typer.echo)
+        ingest_large_dataset(name, path, typer.echo)
         typer.echo(f"Dataset {name} ingested")
     except Exception as e:
         typer.echo(e)
@@ -70,8 +68,7 @@ def update(
     path: Path to dataset to ingest
     """
     try:
-        user = auth()
-        update_dataset(name, path, user, typer.echo)
+        update_dataset(name, path, typer.echo)
         typer.echo(f"Dataset {name} updated")
     except Exception as e:
         typer.echo(e)

eotdl-2023.6.14.post9/eotdl/datasets/__init__.py

@@ -0,0 +1,4 @@
+from .ingest import ingest_dataset, ingest_large_dataset, ingest_q0, ingest_q1
+from .download import download_dataset
+from .retrieve import retrieve_datasets, retrieve_dataset, list_datasets
+from .update import update_dataset

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/datasets/download.py

@@ -1,9 +1,11 @@
-from src.repos import APIRepo
-from src.usecases.datasets.DownloadDataset import DownloadDataset
+from ..src.repos import APIRepo
+from ..src.usecases.datasets import DownloadDataset
 from .retrieve import retrieve_dataset
+from ..auth import with_auth
 
 
-def download_dataset(name, path, user, logger):
+@with_auth
+def download_dataset(name, path=None, logger=None, user=None):
     dataset = retrieve_dataset(name)
     dataset_id = dataset["id"]
     checksum = dataset["checksum"]

eotdl-2023.6.14.post9/eotdl/datasets/ingest.py

@@ -0,0 +1,32 @@
+from ..src.repos import APIRepo
+from ..src.usecases.datasets import IngestDataset, IngestLargeDataset
+from ..auth import with_auth
+
+
+@with_auth
+def ingest_dataset(name, description, path, logger=None, user=None):
+    api_repo = APIRepo()
+    ingest = IngestDataset(
+        api_repo,
+    )
+    inputs = ingest.Inputs(name=name, description=description, path=path, user=user)
+    outputs = ingest(inputs)
+    return outputs.dataset
+
+
+@with_auth
+def ingest_large_dataset(name, path, logger=None, user=None):
+    api_repo = APIRepo()
+    ingest = IngestLargeDataset(api_repo, logger)
+    inputs = ingest.Inputs(name=name, path=path, user=user)
+    outputs = ingest(inputs)
+    return outputs.dataset
+
+
+def ingest_q0(dataset, path):
+    return ingest_large_dataset(dataset, path)
+
+
+def ingest_q1(dataset, stac_catalog):
+    print("holas")
+    return

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/datasets/retrieve.py

@@ -1,6 +1,9 @@
-from src.repos import APIRepo
-from src.usecases.datasets.RetrieveDatasets import RetrieveDatasets
-from src.usecases.datasets.RetrieveDataset import RetrieveDataset
+from ..src.repos import APIRepo
+from ..src.usecases.datasets import RetrieveDatasets, RetrieveDataset
+
+
+def list_datasets():
+    return retrieve_datasets()
 
 
 def retrieve_datasets():

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/datasets/update.py

@@ -1,8 +1,10 @@
-from src.repos import APIRepo
-from src.usecases.datasets.UpdateDataset import UpdateDataset
+from ..src.repos import APIRepo
+from ..src.usecases.datasets import UpdateDataset
+from ..auth import with_auth
 
 
-def update_dataset(name, path, user, logger):
+@with_auth
+def update_dataset(name, path, logger=None, user=None):
     api_repo = APIRepo()
     ingest = UpdateDataset(api_repo, logger)
     inputs = ingest.Inputs(name=name, path=path, user=user)

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/src/repos/APIRepo.py

@@ -35,7 +35,7 @@ class APIRepo:
         url = self.url + "datasets/" + dataset_id + "/download"
         headers = {"Authorization": "Bearer " + id_token}
         if path is None:
-            path = str(Path.home()) + "/.etodl/datasets"
+            path = str(Path.home()) + "/.eotdl/datasets"
             os.makedirs(path, exist_ok=True)
         with requests.get(url, headers=headers, stream=True) as r:
             r.raise_for_status()
@@ -46,6 +46,8 @@ class APIRepo:
             )
             filename = r.headers.get("content-disposition").split("filename=")[1][1:-1]
             path = f"{path}/{filename}"
+            if os.path.exists(path):
+                raise Exception("File already exists")
             with open(path, "wb") as f:
                 for chunk in r.iter_content(block_size):
                     progress_bar.update(len(chunk))

eotdl-2023.6.14.post9/eotdl/src/repos/AuthRepo.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+import os
+import json
+import jwt
+
+
+class AuthRepo:
+    def __init__(self):
+        self.algorithms = ["RS256"]
+        self.home = str(Path.home())
+        self.creds_path = self.home + "/.eotdl/creds.json"
+
+    def save_creds(self, data):
+        os.makedirs(self.home + "/.eotdl", exist_ok=True)
+        with open(self.creds_path, "w") as f:
+            json.dump(data, f)
+        return self.creds_path
+
+    def load_creds(self):
+        if os.path.exists(self.creds_path):
+            with open(self.creds_path, "r") as f:
+                creds = json.load(f)
+            user = self.decode_token(creds)
+            user["id_token"] = creds["id_token"]
+            return user
+        return None
+
+    def decode_token(self, token_data):
+        return jwt.decode(
+            token_data["id_token"],
+            algorithms=self.algorithms,
+            options={"verify_signature": False},
+        )
+
+    def logout(self):
+        os.remove(self.creds_path)

eotdl-2023.6.14.post9/eotdl/src/usecases/auth/__init__.py

@@ -0,0 +1,3 @@
+from .Auth import Auth
+from .IsLogged import IsLogged
+from .Logout import Logout

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/src/usecases/datasets/DownloadDataset.py

@@ -1,11 +1,11 @@
 from pydantic import BaseModel
-from src.utils import calculate_checksum
+from ....src.utils import calculate_checksum
 
 
 class DownloadDataset:
     def __init__(self, repo, logger):
         self.repo = repo
-        self.logger = logger
+        self.logger = logger if logger else print
 
     class Inputs(BaseModel):
         dataset: str

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/src/usecases/datasets/IngestDataset.py

@@ -4,7 +4,7 @@ from pydantic import BaseModel
 class IngestDataset:
     def __init__(self, repo, logger):
         self.repo = repo
-        self.logger = logger
+        self.logger = logger if logger else print
 
     class Inputs(BaseModel):
         name: str

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/src/usecases/datasets/IngestLargeDataset.py

@@ -1,11 +1,11 @@
 from pydantic import BaseModel
-from src.utils import calculate_checksum
+from ....src.utils import calculate_checksum
 
 
 class IngestLargeDataset:
     def __init__(self, repo, logger):
         self.repo = repo
-        self.logger = logger
+        self.logger = logger if logger else print
 
     class Inputs(BaseModel):
         name: str

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/eotdl/src/usecases/datasets/UpdateDataset.py

@@ -1,11 +1,11 @@
 from pydantic import BaseModel
-from src.utils import calculate_checksum
+from ....src.utils import calculate_checksum
 
 
 class UpdateDataset:
     def __init__(self, repo, logger):
         self.repo = repo
-        self.logger = logger
+        self.logger = logger if logger else print
 
     class Inputs(BaseModel):
         name: str

eotdl-2023.6.14.post9/eotdl/src/usecases/datasets/__init__.py

@@ -0,0 +1,6 @@
+from .DownloadDataset import DownloadDataset
+from .IngestDataset import IngestDataset
+from .IngestLargeDataset import IngestLargeDataset
+from .RetrieveDataset import RetrieveDataset
+from .RetrieveDatasets import RetrieveDatasets
+from .UpdateDataset import UpdateDataset

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "eotdl"
-version = "2023.06.14-7"
+version = "2023.06.14-9"
 description = "Earth Observation Training Data Lab"
 authors = ["EarthPulse <it@earthpulse.es>"]
 license = "MIT"
@@ -8,7 +8,7 @@ readme = "README.md"
 packages = [{include = "eotdl"}]
 
 [tool.poetry.scripts]
-eotdl = "eotdl.main:app"
+eotdl = "eotdl.cli:app"
 
 [tool.poetry.dependencies]
 python = "^3.8"

{eotdl-2023.6.14.post7 → eotdl-2023.6.14.post9}/setup.py

@@ -30,11 +30,11 @@ install_requires = \
  'typer[all]>=0.7.0,<0.8.0']
 
 entry_points = \
-{'console_scripts': ['eotdl = eotdl.main:app']}
+{'console_scripts': ['eotdl = eotdl.cli:app']}
 
 setup_kwargs = {
     'name': 'eotdl',
-    'version': '2023.6.14.post7',
+    'version': '2023.6.14.post9',
     'description': 'Earth Observation Training Data Lab',
     'long_description': '# eotdl \n\nThis is the main library and CLI for EOTDL.\n\n',
     'author': 'EarthPulse',

eotdl-2023.6.14.post7/eotdl/auth/__init__.py

@@ -1 +0,0 @@
-from .main import is_logged, auth, generate_logout_url

eotdl-2023.6.14.post7/eotdl/datasets/__init__.py

@@ -1,5 +0,0 @@
-from .ingest import ingest_dataset, ingest_large_dataset, ingest_q0, ingest_q1
-
-# from .download import download_dataset
-# from .retrieve import retrieve_datasets, retrieve_dataset
-# from .update import update_dataset

eotdl-2023.6.14.post7/eotdl/datasets/ingest.py

@@ -1,36 +0,0 @@
-# para que la lib funcione, necesito que sea relativo
-# pero para la cli, necesito que sea absoluto :(
-
-from ..src.repos import APIRepo
-from ..src.usecases.datasets.IngestDataset import IngestDataset
-
-# from ..src.usecases.datasets.IngestLargeDataset import IngestLargeDataset
-
-
-def ingest_q0(dataset, file):
-    print("hola")
-    return
-
-
-def ingest_q1(dataset, stac_catalog):
-    print("hola")
-    return
-
-
-def ingest_dataset(name, description, path, user, logger):
-    api_repo = APIRepo()
-    ingest = IngestDataset(
-        api_repo,
-    )
-    inputs = ingest.Inputs(name=name, description=description, path=path, user=user)
-    outputs = ingest(inputs)
-    return outputs.dataset
-
-
-def ingest_large_dataset(name, path, user, logger):
-    # api_repo = APIRepo()
-    # ingest = IngestLargeDataset(api_repo, logger)
-    # inputs = ingest.Inputs(name=name, path=path, user=user)
-    # outputs = ingest(inputs)
-    # return outputs.dataset
-    return

eotdl-2023.6.14.post7/eotdl/main.py

@@ -1,16 +0,0 @@
-import typer
-import os
-import sys
-
-# Add the eotdl_cli directory to the Python path
-eotdl_dir = os.path.dirname(os.path.realpath(__file__))
-sys.path.append(os.path.join(eotdl_dir))
-
-from commands import auth, datasets
-
-app = typer.Typer()
-app.add_typer(auth.app, name="auth")
-app.add_typer(datasets.app, name="datasets")
-
-if __name__ == "__main__":
-    app()

eotdl-2023.6.14.post7/eotdl/src/repos/AuthRepo.py

@@ -1,31 +0,0 @@
-from pathlib import Path
-import os 
-import json 
-import jwt
-
-class AuthRepo():
-    def __init__(self):
-        self.algorithms = ['RS256']
-        self.home = str(Path.home())
-        self.creds_path = self.home + '/.etodl/creds.json'
-
-    def save_creds(self, data):
-        os.makedirs(self.home + '/.etodl', exist_ok=True)
-        with open(self.creds_path, 'w') as f:
-            json.dump(data, f)
-        return self.creds_path
-    
-    def load_creds(self):
-        if os.path.exists(self.creds_path):
-            with open(self.creds_path, 'r') as f:
-                creds = json.load(f)
-            user = self.decode_token(creds)
-            user['id_token'] = creds['id_token']
-            return user
-        return None
-    
-    def decode_token(self, token_data):
-        return jwt.decode(token_data['id_token'], algorithms=self.algorithms, options={"verify_signature": False})
-    
-    def logout(self):
-        os.remove(self.creds_path)