mapreader 1.3.11__tar.gz → 1.4.0__tar.gz

{mapreader-1.3.11 → mapreader-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mapreader
-Version: 1.3.11
+Version: 1.4.0
 Summary: A computer vision pipeline for the semantic exploration of maps/images at scale
 Home-page: https://github.com/Living-with-machines/MapReader
 Download-URL: https://github.com/Living-with-machines/MapReader/archive/refs/heads/main.zip

{mapreader-1.3.11 → mapreader-1.4.0}/mapreader/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-09-12T08:29:12+0100",
+ "date": "2024-09-12T19:16:25+0100",
  "dirty": false,
  "error": null,
- "full-revisionid": "3db8e6fbaef97b79e4cd663f664585041b539c99",
- "version": "1.3.11"
+ "full-revisionid": "2651e3de148e0069aa93151268b18534108dbe91",
+ "version": "1.4.0"
 }
 '''  # END VERSION_JSON
 

{mapreader-1.3.11 → mapreader-1.4.0}/mapreader/annotate/annotator.py

@@ -4,13 +4,15 @@ import functools
 import hashlib
 import json
 import os
+import pathlib
 import random
+import re
 import string
 import warnings
-from ast import literal_eval
 from itertools import product
 from pathlib import Path
 
+import geopandas as gpd
 import ipywidgets as widgets
 import numpy as np
 import pandas as pd
@@ -18,6 +20,8 @@ from IPython.display import clear_output, display
 from numpy import array_split
 from PIL import Image, ImageOps
 
+from mapreader.utils.load_frames import load_from_csv, load_from_geojson
+
 from ..load.loader import load_patches
 
 warnings.filterwarnings("ignore", category=UserWarning)
@@ -33,10 +37,10 @@ class Annotator:
 
     Parameters
     ----------
-    patch_df : str or pd.DataFrame or None, optional
-        Path to a CSV file or a pandas DataFrame containing patch data, by default None
-    parent_df : str or pd.DataFrame or None, optional
-        Path to a CSV file or a pandas DataFrame containing parent data, by default None
+    patch_df : str, pathlib.Path, pd.DataFrame or gpd.GeoDataFrame or None, optional
+        Path to a CSV/geojson file or a pandas DataFrame/ geopandas GeoDataFrame containing patch data, by default None
+    parent_df : str, pathlib.Path, pd.DataFrame or gpd.GeoDataFrame or None, optional
+        Path to a CSV/geojson file or a pandas DataFrame/ geopandas GeoDataFrame containing parent data, by default None
     labels : list, optional
         List of labels for annotation, by default None
     patch_paths : str or None, optional
@@ -89,7 +93,7 @@ class Annotator:
     FileNotFoundError
         If the provided patch_df or parent_df file path does not exist
     ValueError
-        If patch_df or parent_df is not a valid path to a CSV file or a pandas DataFrame
+        If patch_df or parent_df is not a valid path to a CSV/geojson file or a pandas DataFrame or a geopandas GeoDataFrame
         If patch_df or patch_paths is not provided
         If the DataFrame does not have the required columns
         If sortby is not a string or None
@@ -100,8 +104,8 @@ class Annotator:
 
     def __init__(
         self,
-        patch_df: str | pd.DataFrame | None = None,
-        parent_df: str | pd.DataFrame | None = None,
+        patch_df: str | pathlib.Path | pd.DataFrame | gpd.GeoDataFrame | None = None,
+        parent_df: str | pathlib.Path | pd.DataFrame | gpd.GeoDataFrame | None = None,
         labels: list = None,
         patch_paths: str | None = None,
         parent_paths: str | None = None,
@@ -127,36 +131,40 @@ class Annotator:
         if labels is None:
             labels = []
         if patch_df is not None:
-            if isinstance(patch_df, str):
-                if os.path.exists(patch_df):
-                    patch_df = pd.read_csv(
+            if isinstance(patch_df, (str, pathlib.Path)):
+                if re.search(r"\..?sv$", str(patch_df)):
+                    patch_df = load_from_csv(
                         patch_df,
-                        index_col=0,
-                        sep=delimiter,
+                        delimiter=delimiter,
                     )
+                elif re.search(r"\..*?json$", str(patch_df)):
+                    patch_df = load_from_geojson(patch_df)
                 else:
-                    raise FileNotFoundError(f"[ERROR] Could not find {patch_df}.")
-            if not isinstance(patch_df, pd.DataFrame):
+                    raise ValueError(
+                        "[ERROR] ``patch_df`` must be a path to a CSV/TSV/etc or geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
+                    )
+            elif not isinstance(patch_df, pd.DataFrame):
                 raise ValueError(
-                    "[ERROR] ``patch_df`` must be a path to a csv or a pandas DataFrame."
+                    "[ERROR] ``patch_df`` must be a path to a CSV/TSV/etc or geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
                 )
-            patch_df = self._eval_df(patch_df)  # eval tuples/lists in df
 
         if parent_df is not None:
-            if isinstance(parent_df, str):
-                if os.path.exists(parent_df):
-                    parent_df = pd.read_csv(
+            if isinstance(parent_df, (str, pathlib.Path)):
+                if re.search(r"\..?sv$", str(parent_df)):
+                    parent_df = load_from_csv(
                         parent_df,
-                        index_col=0,
-                        sep=delimiter,
+                        delimiter=delimiter,
                     )
+                elif re.search(r"\..*?json$", str(parent_df)):
+                    parent_df = load_from_geojson(parent_df)
                 else:
-                    raise FileNotFoundError(f"[ERROR] Could not find {parent_df}.")
+                    raise ValueError(
+                        "[ERROR] ``parent_df`` must be a path to a CSV/TSV/etc or geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
+                    )
             if not isinstance(parent_df, pd.DataFrame):
                 raise ValueError(
-                    "[ERROR] ``parent_df`` must be a path to a csv or a pandas DataFrame."
+                    "[ERROR] ``parent_df`` must be a path to a CSV/TSV/etc or geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
                 )
-            parent_df = self._eval_df(parent_df)  # eval tuples/lists in df
 
         if patch_df is None:
             # If we don't get patch data provided, we'll use the patches and parents to create the dataframes
@@ -354,18 +362,9 @@ class Annotator:
 
         return parent_df, patch_df
 
-    @staticmethod
-    def _eval_df(df):
-        for col in df.columns:
-            try:
-                df[col] = df[col].apply(literal_eval)
-            except (ValueError, TypeError, SyntaxError):
-                pass
-        return df
-
     @staticmethod
     def _load_annotations(
-        patch_df: pd.DataFrame,
+        patch_df: pd.DataFrame | gpd.GeoDataFrame,
         annotations_file: str,
         labels: list,
         label_col: str,
@@ -375,7 +374,7 @@ class Annotator:
 
         Parameters
         ----------
-        patch_df : pd.DataFrame
+        patch_df : pd.DataFrame or gpd.GeoDataFrame
             Current patch dataframe.
         annotations_file : str
             Name of the annotations file
@@ -387,7 +386,9 @@ class Annotator:
             Delimiter used in CSV files
 
         """
-        existing_annotations = pd.read_csv(annotations_file, index_col=0, sep=delimiter)
+        existing_annotations = load_from_csv(
+            annotations_file, index_col=0, sep=delimiter
+        )
 
         if label_col not in existing_annotations.columns:
             raise ValueError(
@@ -995,8 +996,8 @@ class Annotator:
 
         Returns
         -------
-        pandas.DataFrame
-            A dataframe containing the labelled images and their associated
+        pandas.DataFrame or geopandas.GeoDataFrame
+            A DataFrame/GeoDataFrame containing the labelled images and their associated
             label index.
         """
         filtered_df = self.patch_df[self.patch_df[self.label_col].notna()].copy(

{mapreader-1.3.11 → mapreader-1.4.0}/mapreader/classify/datasets.py

@@ -2,10 +2,12 @@
 from __future__ import annotations
 
 import os
-from ast import literal_eval
+import pathlib
+import re
 from itertools import product
 from typing import Callable
 
+import geopandas as gpd
 import matplotlib.pyplot as plt
 import pandas as pd
 import torch
@@ -24,14 +26,16 @@ except ImportError:
     )
     parhugin_installed = False
 
+from mapreader.utils.load_frames import eval_dataframe, load_from_csv, load_from_geojson
+
 
 class PatchDataset(Dataset):
     """A PyTorch Dataset class for loading image patches from a DataFrame.
 
     Parameters
     ----------
-    patch_df : pandas.DataFrame or str
-        DataFrame or path to csv file containing the paths to image patches and their labels.
+    patch_df : str or pathlib.Path or pandas.DataFrame or gpd.GeoDataFrame
+        DataFrame or path to CSV/TSV/geojson file containing the paths to image patches and their labels.
     transform : Union[str, transforms.Compose, Callable]
         The transform to use on the image.
         A string can be used to call default transforms - options are "train", "test" or "val".
@@ -39,7 +43,7 @@ class PatchDataset(Dataset):
         and performs image transformations can be used.
         At minimum, transform should be ``torchvision.transforms.ToTensor()``.
     delimiter : str, optional
-        The delimiter to use when reading the dataframe. By default ``","``.
+        The delimiter to use when reading the CSV/TSV file. By default ``","``.
     patch_paths_col : str, optional
         The name of the column in the DataFrame containing the image paths. Default is "image_path".
     label_col : str, optional
@@ -51,7 +55,7 @@ class PatchDataset(Dataset):
 
     Attributes
     ----------
-    patch_df : pandas.DataFrame
+    patch_df : pandas.DataFrame or gpd.GeoDataFrame
         DataFrame containing the paths to image patches and their labels.
     label_col : str
         The name of the column containing the image labels.
@@ -91,7 +95,7 @@ class PatchDataset(Dataset):
 
     def __init__(
         self,
-        patch_df: pd.DataFrame | str,
+        patch_df: str | pathlib.Path | pd.DataFrame | gpd.GeoDataFrame,
         transform: str | (transforms.Compose | Callable),
         delimiter: str = ",",
         patch_paths_col: str | None = "image_path",
@@ -102,19 +106,22 @@ class PatchDataset(Dataset):
         if isinstance(patch_df, pd.DataFrame):
             self.patch_df = patch_df
 
-        elif isinstance(patch_df, str):
-            if os.path.isfile(patch_df):
-                print(f'[INFO] Reading "{patch_df}".')
-                patch_df = pd.read_csv(patch_df, sep=delimiter)
-                # ensure tuple/list columns are read as such
-                patch_df = self._eval_df(patch_df)
-                self.patch_df = patch_df
+        elif isinstance(patch_df, (str, pathlib.Path)):
+            print(f'[INFO] Reading "{patch_df}".')
+            if re.search(r"\..?sv$", str(patch_df)):
+                self.patch_df = load_from_csv(
+                    patch_df,
+                    delimiter=delimiter,
+                )
+            elif re.search(r"\..*?json$", str(patch_df)):
+                self.patch_df = load_from_geojson(patch_df)
             else:
-                raise ValueError(f'[ERROR] "{patch_df}" cannot be found.')
-
+                raise ValueError(
+                    "[ERROR] ``patch_df`` must be a path to a CSV/geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
+                )
         else:
             raise ValueError(
-                "[ERROR] Please pass ``patch_df`` as a string (path to csv file) or pd.DataFrame."
+                "[ERROR] ``patch_df`` must be a path to a CSV/geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
             )
 
         # force index to be image_id
@@ -133,7 +140,7 @@ class PatchDataset(Dataset):
         if self.label_col:
             if self.label_col not in self.patch_df.columns:
                 raise ValueError(
-                    f"[ERROR] Label column ({label_col}) not in dataframe."
+                    f"[ERROR] Label column ({label_col}) not in DataFrame."
                 )
             else:
                 self.unique_labels = self.patch_df[self.label_col].unique().tolist()
@@ -142,14 +149,14 @@ class PatchDataset(Dataset):
             if self.label_index_col not in self.patch_df.columns:
                 if self.label_col:
                     print(
-                        f"[INFO] Label index column ({label_index_col}) not in dataframe. Creating column."
+                        f"[INFO] Label index column ({label_index_col}) not in DataFrame. Creating column."
                     )
                     self.patch_df[self.label_index_col] = self.patch_df[
                         self.label_col
                     ].apply(self._get_label_index)
                 else:
                     raise ValueError(
-                        f"[ERROR] Label index column ({label_index_col}) not in dataframe."
+                        f"[ERROR] Label index column ({label_index_col}) not in DataFrame."
                     )
 
         if isinstance(transform, str):
@@ -162,15 +169,6 @@ class PatchDataset(Dataset):
         else:
             self.transform = transform
 
-    @staticmethod
-    def _eval_df(df):
-        for col in df.columns:
-            try:
-                df[col] = df[col].apply(literal_eval)
-            except (ValueError, TypeError, SyntaxError):
-                pass
-        return df
-
     def __len__(self) -> int:
         """
         Return the length of the dataset.
@@ -393,15 +391,15 @@ class PatchContextDataset(PatchDataset):
 
     Parameters
     ----------
-    patch_df : pandas.DataFrame or str
-        DataFrame or path to csv file containing the paths to image patches and their labels.
-    total_df : pandas.DataFrame or str
-        DataFrame or path to csv file containing the paths to all images and their labels.
+    patch_df : str or pathlib.Path or pandas.DataFrame or gpd.GeoDataFrame
+        DataFrame or path to CSV/TSV/geojson file containing the paths to image patches and their labels.
+    total_df : str or pathlib.Path or pandas.DataFrame or gpd.GeoDataFrame
+        DataFrame or path to CSV/TSV/geojson file containing the paths to all images and their labels.
     transform : str
         Torchvision transform to be applied to context  images.
         Either "train" or "val".
     delimiter : str
-        The delimiter to use when reading the csv file. By default ``","``.
+        The delimiter to use when reading the CSV/TSV file. By default ``","``.
     patch_paths_col : str, optional
         The name of the column in the DataFrame containing the image paths. Default is "image_path".
     label_col : str, optional
@@ -421,8 +419,8 @@ class PatchContextDataset(PatchDataset):
 
     Attributes
     ----------
-    patch_df : pandas.DataFrame
-        A pandas DataFrame with columns representing image paths, labels,
+    patch_df : pandas.DataFrame or gpd.GeoDataFrame
+        DataFrame with columns representing image paths, labels,
         and object bounding boxes.
     label_col : str
         The name of the column containing the image labels.
@@ -445,8 +443,8 @@ class PatchContextDataset(PatchDataset):
 
     def __init__(
         self,
-        patch_df: pd.DataFrame | str,
-        total_df: pd.DataFrame | str,
+        patch_df: str | pathlib.Path | pd.DataFrame | gpd.GeoDataFrame,
+        total_df: str | pathlib.Path | pd.DataFrame | gpd.GeoDataFrame,
         transform: str,
         delimiter: str = ",",
         patch_paths_col: str | None = "image_path",
@@ -459,35 +457,42 @@ class PatchContextDataset(PatchDataset):
     ):
         if isinstance(patch_df, pd.DataFrame):
             self.patch_df = patch_df
-
-        elif isinstance(patch_df, str):
-            if os.path.isfile(patch_df):
-                print(f'[INFO] Reading "{patch_df}".')
-                patch_df = pd.read_csv(patch_df, sep=delimiter)
-                patch_df = self._eval_df(patch_df)
-                self.patch_df = patch_df
+        elif isinstance(patch_df, (str, pathlib.Path)):
+            print(f'[INFO] Reading "{patch_df}".')
+            if re.search(r"\..?sv$", str(patch_df)):
+                self.patch_df = load_from_csv(
+                    patch_df,
+                    delimiter=delimiter,
+                )
+            elif re.search(r"\..*?json$", str(patch_df)):
+                self.patch_df = load_from_geojson(patch_df)
             else:
-                raise ValueError(f'[ERROR] "{patch_df}" cannot be found.')
-
+                raise ValueError(
+                    "[ERROR] ``patch_df`` must be a path to a CSV/geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
+                )
         else:
             raise ValueError(
-                "[ERROR] Please pass ``patch_df`` as a string (path to csv file) or pd.DataFrame."
+                "[ERROR] ``patch_df`` must be a path to a CSV/geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
             )
 
         if isinstance(total_df, pd.DataFrame):
             self.total_df = total_df
-
-        elif isinstance(total_df, str):
-            if os.path.isfile(total_df):
-                total_df = pd.read_csv(total_df, sep=delimiter)
-                total_df = self._eval_df(total_df)
-                self.total_df = total_df
+        elif isinstance(total_df, (str, pathlib.Path)):
+            print(f'[INFO] Reading "{total_df}".')
+            if re.search(r"\..?sv$", str(total_df)):
+                self.total_df = load_from_csv(
+                    total_df,
+                    delimiter=delimiter,
+                )
+            elif re.search(r"\..*?json$", str(total_df)):
+                self.total_df = load_from_geojson(total_df)
             else:
-                raise ValueError(f'[ERROR] "{total_df}" cannot be found.')
-
+                raise ValueError(
+                    "[ERROR] ``total_df`` must be a path to a CSV/geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
+                )
         else:
             raise ValueError(
-                "[ERROR] Please pass ``total_df`` as a string (path to csv file) or pd.DataFrame."
+                "[ERROR] ``total_df`` must be a path to a CSV/geojson file or a pandas DataFrame or a geopandas GeoDataFrame."
             )
 
         # force index to be image_id
@@ -506,18 +511,23 @@ class PatchContextDataset(PatchDataset):
         if self.label_col:
             if self.label_col not in self.patch_df.columns:
                 raise ValueError(
-                    f"[ERROR] Label column ({self.label_col}) not in dataframe."
+                    f"[ERROR] Label column ({self.label_col}) not in DataFrame."
                 )
             self.unique_labels = self.patch_df[self.label_col].unique().tolist()
 
         if self.label_index_col:
             if self.label_index_col not in self.patch_df.columns:
-                print(
-                    f"[INFO] Label index column ({label_index_col}) not in dataframe. Creating column."
-                )
-                self.patch_df[self.label_index_col] = self.patch_df[
-                    self.label_col
-                ].apply(self._get_label_index)
+                if self.label_col:
+                    print(
+                        f"[INFO] Label index column ({label_index_col}) not in DataFrame. Creating column."
+                    )
+                    self.patch_df[self.label_index_col] = self.patch_df[
+                        self.label_col
+                    ].apply(self._get_label_index)
+                else:
+                    raise ValueError(
+                        f"[ERROR] Label index column ({label_index_col}) not in DataFrame."
+                    )
 
         if isinstance(transform, str):
             if transform in ["train", "val", "test"]:
@@ -646,7 +656,7 @@ class PatchContextDataset(PatchDataset):
             )
         if "parent_id" not in total_df.columns:
             total_df["parent_id"] = total_df.index.map(lambda x: x.split("#")[1])
-        total_df = self._eval_df(total_df)
+        total_df = eval_dataframe(total_df)
 
         if not all(
             [col in total_df.columns for col in ["min_x", "min_y", "max_x", "max_y"]]