datamint 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

datamint/client_cmd_tools/datamint_config.py

@@ -4,6 +4,10 @@ from datamint import configs
 from datamint.utils.logging_utils import load_cmdline_logging_config, ConsoleWrapperHandler
 from rich.prompt import Prompt, Confirm
 from rich.console import Console
+import os
+import shutil
+from pathlib import Path
+from rich.table import Table
 
 _LOGGER = logging.getLogger(__name__)
 _USER_LOGGER = logging.getLogger('user_logger')
@@ -85,18 +89,246 @@ def configure_api_key():
 def test_connection():
     """Test the API connection with current settings."""
     try:
-        from datamint import APIHandler
+        from datamint import Api
         console.print("[accent]🔄 Testing connection...[/accent]")
-        api = APIHandler()
-        # Simple test - try to get projects
-        projects = api.get_projects()
-        console.print(f"[success]✅ Connection successful! Found {len(projects)} projects.[/success]")
+        Api(check_connection=True)
+        console.print(f"[success]✅ Connection successful![/success]")
     except ImportError:
         console.print("[error]❌ Full API not available. Install with: pip install datamint[/error]")
     except Exception as e:
         console.print(f"[error]❌ Connection failed: {e}[/error]")
 
 
+def discover_local_datasets() -> list[dict[str, str]]:
+    """Discover locally downloaded datasets.
+    
+    Returns:
+        List of dictionaries containing dataset info with keys: 'name', 'path', 'size'
+    """
+    from datamint.dataset.base_dataset import DatamintBaseDataset
+    
+    # Check default datamint directory
+    default_root = os.path.join(
+        os.path.expanduser("~"),
+        DatamintBaseDataset.DATAMINT_DEFAULT_DIR,
+        DatamintBaseDataset.DATAMINT_DATASETS_DIR
+    )
+    
+    datasets = []
+    
+    if not os.path.exists(default_root):
+        return datasets
+    
+    for item in os.listdir(default_root):
+        dataset_path = os.path.join(default_root, item)
+        if os.path.isdir(dataset_path):
+            # Check if it has a dataset.json file (indicating it's a datamint dataset)
+            dataset_json = os.path.join(dataset_path, 'dataset.json')
+            if os.path.exists(dataset_json):
+                # Calculate directory size
+                total_size = sum(
+                    os.path.getsize(os.path.join(dirpath, filename))
+                    for dirpath, dirnames, filenames in os.walk(dataset_path)
+                    for filename in filenames
+                )
+                
+                datasets.append({
+                    'name': item,
+                    'path': dataset_path,
+                    'size': _format_size(total_size),
+                    'size_bytes': total_size
+                })
+    
+    return sorted(datasets, key=lambda x: x['name'])
+
+
+def _format_size(size_bytes: int) -> str:
+    """Format size in bytes to human readable format."""
+    if size_bytes == 0:
+        return "0 B"
+    
+    size_names = ["B", "KB", "MB", "GB", "TB"]
+    i = 0
+    while size_bytes >= 1024 and i < len(size_names) - 1:
+        size_bytes /= 1024.0
+        i += 1
+    
+    return f"{size_bytes:.1f} {size_names[i]}"
+
+
+def show_local_datasets() -> list[dict[str, str]]:
+    """Display all locally downloaded datasets."""
+    datasets = discover_local_datasets()
+    
+    if not datasets:
+        console.print("[dim]No local datasets found.[/dim]")
+        return datasets
+    
+    console.print("[title]📁 Local Datasets:[/title]")
+    
+    table = Table(show_header=True, header_style="bold blue")
+    table.add_column("Dataset Name", style="cyan")
+    table.add_column("Size", justify="right", style="green")
+    table.add_column("Path", style="dim")
+    
+    total_size = 0
+    for dataset in datasets:
+        table.add_row(dataset['name'], dataset['size'], dataset['path'])
+        total_size += dataset['size_bytes']
+    
+    console.print(table)
+    console.print(f"\n[bold]Total size:[/bold] {_format_size(total_size)}")
+    
+    return datasets
+
+
+def clean_dataset(dataset_name: str) -> bool:
+    """Clean a specific dataset.
+    
+    Args:
+        dataset_name: Name of the dataset to clean
+        
+    Returns:
+        True if dataset was cleaned, False otherwise
+    """
+    datasets = discover_local_datasets()
+    dataset_to_clean = None
+    
+    for dataset in datasets:
+        if dataset['name'] == dataset_name:
+            dataset_to_clean = dataset
+            break
+    
+    if dataset_to_clean is None:
+        console.print(f"[error]❌ Dataset '{dataset_name}' not found locally.[/error]")
+        return False
+    
+    console.print(f"[warning]⚠️  About to delete dataset: {dataset_name}[/warning]")
+    console.print(f"[dim]Path: {dataset_to_clean['path']}[/dim]")
+    console.print(f"[dim]Size: {dataset_to_clean['size']}[/dim]")
+    
+    confirmed = Confirm.ask("Are you sure you want to delete this dataset?", 
+                           default=False, console=console)
+    
+    if not confirmed:
+        console.print("[dim]Operation cancelled.[/dim]")
+        return False
+    
+    try:
+        shutil.rmtree(dataset_to_clean['path'])
+        console.print(f"[success]✅ Dataset '{dataset_name}' has been deleted.[/success]")
+        return True
+    except Exception as e:
+        console.print(f"[error]❌ Error deleting dataset: {e}[/error]")
+        _LOGGER.exception(e)
+        return False
+
+
+def clean_all_datasets() -> bool:
+    """Clean all locally downloaded datasets.
+    
+    Returns:
+        True if datasets were cleaned, False otherwise
+    """
+    datasets = discover_local_datasets()
+    
+    if not datasets:
+        console.print("[dim]No local datasets found to clean.[/dim]")
+        return True
+    
+    console.print(f"[warning]⚠️  About to delete {len(datasets)} dataset(s):[/warning]")
+    
+    table = Table(show_header=True, header_style="bold red")
+    table.add_column("Dataset Name", style="cyan")
+    table.add_column("Size", justify="right", style="green")
+    
+    total_size = 0
+    for dataset in datasets:
+        table.add_row(dataset['name'], dataset['size'])
+        total_size += dataset['size_bytes']
+    
+    console.print(table)
+    console.print(f"\n[bold red]Total size to be deleted:[/bold red] {_format_size(total_size)}")
+
+    confirmed = Confirm.ask("Are you sure you want to delete ALL local datasets? (this does not affect remote datasets)", 
+                           default=False, console=console)
+    
+    if not confirmed:
+        console.print("[dim]Operation cancelled.[/dim]")
+        return False
+    
+    success_count = 0
+    for dataset in datasets:
+        try:
+            shutil.rmtree(dataset['path'])
+            console.print(f"[success]✅ Deleted: {dataset['name']}[/success]")
+            success_count += 1
+        except Exception as e:
+            console.print(f"[error]❌ Failed to delete {dataset['name']}: {e}[/error]")
+            _LOGGER.exception(e)
+    
+    if success_count == len(datasets):
+        console.print(f"[success]✅ Successfully deleted all {success_count} datasets.[/success]")
+        return True
+    else:
+        console.print(f"[warning]⚠️  Deleted {success_count} out of {len(datasets)} datasets.[/warning]")
+        return False
+
+
+def interactive_dataset_cleaning() -> None:
+    """Interactive dataset cleaning menu."""
+    datasets = show_local_datasets()
+    
+    if not datasets:
+        return
+    
+    console.print("\n[title]🧹 Dataset Cleaning Options:[/title]")
+    console.print(" [accent](1)[/accent] Clean a specific dataset")
+    console.print(" [accent](2)[/accent] Clean all datasets")
+    console.print(" [accent](b)[/accent] Back to main menu")
+
+    try:
+        choice = Prompt.ask("Enter your choice", console=console).lower().strip()
+        
+        # Handle ESC key (appears as escape sequence)
+        if choice in ('', '\x1b', 'esc', 'escape'):
+            return
+        
+        if choice == '1':
+            dataset_names = [d['name'] for d in datasets]
+            console.print("\n[title]Available datasets:[/title]")
+            for i, name in enumerate(dataset_names, 1):
+                console.print(f" [accent]({i})[/accent] {name}")
+            
+            dataset_choice = Prompt.ask("Enter dataset number or name", console=console).strip()
+            
+            # Handle ESC key in dataset selection
+            if dataset_choice in ('', '\x1b', 'esc', 'escape'):
+                return
+            
+            # Handle numeric choice
+            try:
+                dataset_idx = int(dataset_choice) - 1
+                if 0 <= dataset_idx < len(dataset_names):
+                    clean_dataset(dataset_names[dataset_idx])
+                    return
+            except ValueError:
+                pass
+            
+            # Handle name choice
+            if dataset_choice in dataset_names:
+                clean_dataset(dataset_choice)
+            else:
+                console.print("[error]❌ Invalid dataset selection.[/error]")
+                
+        elif choice == '2':
+            clean_all_datasets()
+        elif choice != 'b':
+            console.print("[error]❌ Invalid choice.[/error]")
+    except KeyboardInterrupt:
+        pass
+
+
 def interactive_mode():
     """Run the interactive configuration mode."""
     console.print("[title]🔧 Datamint Configuration Tool[/title]")
@@ -113,6 +345,7 @@ def interactive_mode():
             console.print(" [accent](3)[/accent] Show all configuration settings")
             console.print(" [accent](4)[/accent] Clear all configuration settings")
             console.print(" [accent](5)[/accent] Test connection")
+            console.print(" [accent](6)[/accent] Manage/Show local datasets...")
             console.print(" [accent](q)[/accent] Exit")
             choice = Prompt.ask("Enter your choice", console=console).lower().strip()
 
@@ -126,15 +359,18 @@ def interactive_mode():
                 clear_all_configurations()
             elif choice == '5':
                 test_connection()
+            elif choice == '6':
+                interactive_dataset_cleaning()
             elif choice in ('q', 'exit', 'quit'):
                 break
             else:
-                console.print("[error]❌ Invalid choice. Please enter a number between 1 and 5 or 'q' to quit.[/error]")
+                console.print("[error]❌ Invalid choice. Please enter a number between 1 and 7 or 'q' to quit.[/error]")
     except KeyboardInterrupt:
         console.print('')
 
     console.print("[success]👋 Goodbye![/success]")
 
+
 def main():
     """Main entry point for the configuration tool."""
     global console
@@ -146,6 +382,9 @@ def main():
 Examples:
   datamint-config                           # Interactive mode
   datamint-config --api-key YOUR_KEY        # Set API key
+  datamint-config --list-datasets           # Show local datasets
+  datamint-config --clean-dataset NAME      # Clean specific dataset
+  datamint-config --clean-all-datasets      # Clean all datasets
   
 More Documentation: https://sonanceai.github.io/datamint-python-api/command_line_tools.html
         """,
@@ -155,6 +394,12 @@ More Documentation: https://sonanceai.github.io/datamint-python-api/command_line
     parser.add_argument('--default-url', '--url', type=str, help='Default URL to set')
     parser.add_argument('-i', '--interactive', action='store_true',
                         help='Interactive mode (default if no other arguments provided)')
+    parser.add_argument('--list-datasets', action='store_true',
+                        help='List all locally downloaded datasets')
+    parser.add_argument('--clean-dataset', type=str, metavar='DATASET_NAME',
+                        help='Clean a specific dataset by name')
+    parser.add_argument('--clean-all-datasets', action='store_true',
+                        help='Clean all locally downloaded datasets')
 
     args = parser.parse_args()
 
@@ -170,7 +415,18 @@ More Documentation: https://sonanceai.github.io/datamint-python-api/command_line
         configs.set_value(configs.APIURL_KEY, args.default_url)
         console.print("[success]✅ Default URL saved.[/success]")
 
-    no_arguments_provided = args.api_key is None and args.default_url is None
+    if args.list_datasets:
+        show_local_datasets()
+
+    if args.clean_dataset:
+        clean_dataset(args.clean_dataset)
+
+    if args.clean_all_datasets:
+        clean_all_datasets()
+
+    no_arguments_provided = (args.api_key is None and args.default_url is None and
+                           not args.list_datasets and not args.clean_dataset and
+                           not args.clean_all_datasets)
 
     if no_arguments_provided or args.interactive:
         interactive_mode()

datamint/dataset/base_dataset.py

@@ -1,7 +1,7 @@
 import os
 import requests
-from tqdm import tqdm
-from typing import Optional, Callable, Any, Literal
+from tqdm.auto import tqdm
+from typing import Optional, Callable, Any, Literal, Sequence
 import logging
 import shutil
 import json
@@ -66,8 +66,8 @@ class DatamintBaseDataset:
         project_name: str,
         root: str | None = None,
         auto_update: bool = True,
-        api_key: Optional[str] = None,
-        server_url: Optional[str] = None,
+        api_key: str | None = None,
+        server_url: str | None = None,
         return_dicom: bool = False,
         return_metainfo: bool = True,
         return_annotations: bool = True,
@@ -75,14 +75,14 @@ class DatamintBaseDataset:
         include_unannotated: bool = True,
         all_annotations: bool = False,
         # Filtering parameters
-        include_annotators: Optional[list[str]] = None,
-        exclude_annotators: Optional[list[str]] = None,
-        include_segmentation_names: Optional[list[str]] = None,
-        exclude_segmentation_names: Optional[list[str]] = None,
-        include_image_label_names: Optional[list[str]] = None,
-        exclude_image_label_names: Optional[list[str]] = None,
-        include_frame_label_names: Optional[list[str]] = None,
-        exclude_frame_label_names: Optional[list[str]] = None,
+        include_annotators: list[str] | None = None,
+        exclude_annotators: list[str] | None = None,
+        include_segmentation_names: list[str] | None = None,
+        exclude_segmentation_names: list[str] | None = None,
+        include_image_label_names: list[str] | None = None,
+        exclude_image_label_names: list[str] | None = None,
+        include_frame_label_names: list[str] | None = None,
+        exclude_frame_label_names: list[str] | None = None,
     ):
         self._validate_inputs(project_name, include_annotators, exclude_annotators,
                               include_segmentation_names, exclude_segmentation_names,
@@ -106,14 +106,14 @@ class DatamintBaseDataset:
     def _validate_inputs(
         self,
         project_name: str,
-        include_annotators: Optional[list[str]],
-        exclude_annotators: Optional[list[str]],
-        include_segmentation_names: Optional[list[str]],
-        exclude_segmentation_names: Optional[list[str]],
-        include_image_label_names: Optional[list[str]],
-        exclude_image_label_names: Optional[list[str]],
-        include_frame_label_names: Optional[list[str]],
-        exclude_frame_label_names: Optional[list[str]],
+        include_annotators: Sequence[str] | None,
+        exclude_annotators: Sequence[str] | None,
+        include_segmentation_names: Sequence[str] | None,
+        exclude_segmentation_names: Sequence[str] | None,
+        include_image_label_names: Sequence[str] | None,
+        exclude_image_label_names: Sequence[str] | None,
+        include_frame_label_names: Sequence[str] | None,
+        exclude_frame_label_names: Sequence[str] | None,
     ) -> None:
         """Validate input parameters."""
         if project_name is None:
@@ -905,8 +905,10 @@ class DatamintBaseDataset:
         ################
 
         ### ANNOTATIONS ###
-        all_annotations = self.api.annotations.get_list(worklist_id=None if self.all_annotations else self.project_info['worklist_id'],
-                                                        status=None if self.all_annotations else 'published')
+        _LOGGER.info("Fetching new annotations...")
+        all_annotations = self.api.annotations.get_list(worklist_id=self.project_info['worklist_id'],
+                                                        status=None if self.all_annotations else 'published',
+                                                        load_ai_segmentations=self.all_annotations)
 
         # group annotations by resource ID
         annotations_by_resource: dict[str, list[Annotation]] = {}

{datamint-2.0.1.dist-info → datamint-2.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datamint
-Version: 2.0.1
+Version: 2.1.0
 Summary: A library for interacting with the Datamint API, designed for efficient data management, processing and Deep Learning workflows.
 Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
@@ -16,11 +16,12 @@ Requires-Dist: aiohttp (>=3.0.0,<4.0.0)
 Requires-Dist: aioresponses (>=0.7.8,<0.8.0) ; extra == "dev"
 Requires-Dist: albumentations (>=2.0.0)
 Requires-Dist: datamintapi (==0.0.*)
+Requires-Dist: httpx
 Requires-Dist: humanize (>=4.0.0,<5.0.0)
 Requires-Dist: lazy-loader (>=0.3.0)
 Requires-Dist: lightning
 Requires-Dist: matplotlib
-Requires-Dist: medimgkit (>=0.6.0)
+Requires-Dist: medimgkit (>=0.6.2)
 Requires-Dist: nest-asyncio (>=1.0.0,<2.0.0)
 Requires-Dist: nibabel (>=4.0.0)
 Requires-Dist: numpy

{datamint-2.0.1.dist-info → datamint-2.1.0.dist-info}/RECORD

@@ -19,12 +19,12 @@ datamint/apihandler/dto/annotation_dto.py,sha256=KUeHbxLYols16q-ANNxC48eH4EA8Tc-
 datamint/apihandler/exp_api_handler.py,sha256=hFUgUgBc5rL7odK7gTW3MnrvMY1pVfJUpUdzRNobMQE,6226
 datamint/apihandler/root_api_handler.py,sha256=jBof_XPTeq4o41CW-l-I5GHQKVa76kaX75RovS_qAM4,63384
 datamint/client_cmd_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamint/client_cmd_tools/datamint_config.py,sha256=5S9sgS64F141ZcvvjCHEEgfNKwhGt4g2oJQaJqeiugA,7228
+datamint/client_cmd_tools/datamint_config.py,sha256=MpR5UHv_xpElOOYyEESBkDg2n3JjP_PNLI2jqmZgYQ8,16222
 datamint/client_cmd_tools/datamint_upload.py,sha256=jPzvlNeBZfOOxuG6ryswJ8OG4jXuTrPtArUetoKVGj0,36073
 datamint/configs.py,sha256=Bdp6NydYwyCJ2dk19_gf_o3M2ZyQOmMHpLi8wEWNHUk,1426
 datamint/dataset/__init__.py,sha256=4PlUKSvVhdfQvvuq8jQXrkdqnot-iTTizM3aM1vgSwg,47
 datamint/dataset/annotation.py,sha256=qN1IMjdfLD2ceQ6va3l76jOXA8Vb_c-eBk1oWQu6hW0,7994
-datamint/dataset/base_dataset.py,sha256=FtbuZ0YXjj44CTt5IN30XqCWetCSNGL8rTYpO0UWpNM,49472
+datamint/dataset/base_dataset.py,sha256=Ai_fEMW1lmA9ZxGc_7Zuz3zjVUsbzpbnWeDMy7eZLVQ,49583
 datamint/dataset/dataset.py,sha256=It-HOTi83ls4ww2qCAvFYU0_OLLrFclj0QQapMYgDAE,27333
 datamint/entities/__init__.py,sha256=tbHE7rZb0R9Hm-Dc8VWEq3PlRl7BYOzffumrV0ZdsMs,444
 datamint/entities/annotation.py,sha256=ochAEh_JqxAe_FyYTNUfPT47KiIAG7CkBTim52bu7M8,6636
@@ -44,7 +44,7 @@ datamint/logging.yaml,sha256=tOMxtc2UmwlIMTK6ljtnBwTco1PNrPeq3mx2iMuSbiw,482
 datamint/utils/logging_utils.py,sha256=9pRoaPrWu2jOdDCiAoUsjEdP5ZwaealWL3hjUqFvx9g,4022
 datamint/utils/torchmetrics.py,sha256=lwU0nOtsSWfebyp7dvjlAggaqXtj5ohSEUXOg3L0hJE,2837
 datamint/utils/visualization.py,sha256=yaUVAOHar59VrGUjpAWv5eVvQSfztFG0eP9p5Vt3l-M,4470
-datamint-2.0.1.dist-info/METADATA,sha256=cv5nPgETktlGFYe_NrQ4p15zPj0glLobP0nzAZxwTNU,4182
-datamint-2.0.1.dist-info/WHEEL,sha256=M5asmiAlL6HEcOq52Yi5mmk9KmTVjY2RDPtO4p9DMrc,88
-datamint-2.0.1.dist-info/entry_points.txt,sha256=mn5H6jPjO-rY0W0CAZ6Z_KKWhMLvyVaSpoqk77jlTI4,145
-datamint-2.0.1.dist-info/RECORD,,
+datamint-2.1.0.dist-info/METADATA,sha256=JFlip7Zr_9y7MzxTz4NUw3vNbDQtAI8rOtiSRtqPbwM,4203
+datamint-2.1.0.dist-info/WHEEL,sha256=M5asmiAlL6HEcOq52Yi5mmk9KmTVjY2RDPtO4p9DMrc,88
+datamint-2.1.0.dist-info/entry_points.txt,sha256=mn5H6jPjO-rY0W0CAZ6Z_KKWhMLvyVaSpoqk77jlTI4,145
+datamint-2.1.0.dist-info/RECORD,,