syft-dataset 0.1.3__py3-none-any.whl

syft_dataset-0.1.3.dist-info/METADATA

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: syft-dataset
+Version: 0.1.3
+Summary: Syft Datasets
+Author-email: OpenMined <info@openmined.org>
+License: Apache-2.0
+Requires-Python: >=3.12
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: syft-notebook-ui

syft_dataset-0.1.3.dist-info/RECORD

@@ -0,0 +1,10 @@
+syft_datasets/__init__.py,sha256=VisH6OXM2j-SCOZUCsPD0kBcZvrAQOvHz6QZAvcK4xI,85
+syft_datasets/config.py,sha256=iObuEfNr5_uwxTXvKqQ1lt5lk-aYKe8gF10mbsmegcw,499
+syft_datasets/dataset.py,sha256=7UNnX7EpSXcYg9cIYx37NnkLFD8L5YfMmdFK4AChrEI,6979
+syft_datasets/dataset_manager.py,sha256=zCQktHvP3uYI0HT2KDvUnq8rO1G3wPhvN_6t0YPbP5Q,13658
+syft_datasets/file_utils.py,sha256=IM9Kw9Vth3ghNqayVJ5_FvyPmXQEaqLdxLI4NDTEt2M,1468
+syft_datasets/types.py,sha256=n7vfKW1BYn1_6223nlYPfeA4rDQoFlTMzM-2WUwCAtg,238
+syft_datasets/url.py,sha256=662XabfCEygaURQtMyB9QW7izHccTjxnyPXyDUw6nSQ,4379
+syft_dataset-0.1.3.dist-info/METADATA,sha256=l4hg1xotXEUk00z7w7P5lu4Lmt_nAJugul6CRTVDAEA,229
+syft_dataset-0.1.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+syft_dataset-0.1.3.dist-info/RECORD,,

syft_dataset-0.1.3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

syft_datasets/__init__.py

@@ -0,0 +1,4 @@
+from .config import SyftBoxConfig
+
+__version__ = "0.1.3"
+__all__ = ["SyftBoxConfig"]

syft_datasets/config.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+from pathlib import Path
+
+
+class SyftBoxConfig(BaseModel):
+    syftbox_folder: Path = Field(
+        ..., description="Path to the SyftBox folder on the local filesystem."
+    )
+    email: str = Field(..., description="Email associated with the SyftBox.")
+
+    @property
+    def private_dir(self) -> Path:
+        return self.syftbox_folder / "private"
+
+    @property
+    def public_dir(self) -> Path:
+        return self.syftbox_folder / self.email / "public"

syft_datasets/dataset.py

@@ -0,0 +1,228 @@
+from datetime import datetime, timezone
+from functools import cached_property
+from pathlib import Path
+from typing import ClassVar, Self
+from uuid import UUID, uuid4
+
+import yaml
+from pydantic import BaseModel, Field
+from syft_notebook_ui.formatter_mixin import (
+    ANSIPydanticFormatter,
+    PydanticFormatter,
+    PydanticFormatterMixin,
+)
+
+from .types import PathLike, to_path
+from .url import SyftBoxURL
+from .config import SyftBoxConfig
+
+
+def _utcnow():
+    return datetime.now(tz=timezone.utc)
+
+
+class DatasetBase(BaseModel):
+    __display_formatter__: ClassVar[PydanticFormatter] = ANSIPydanticFormatter()
+    _syftbox_config: SyftBoxConfig | None = None
+
+    def save(self, filepath: PathLike) -> None:
+        filepath = to_path(filepath)
+        if not filepath.suffix == ".yaml":
+            raise ValueError("Model must be saved as a .yaml file.")
+
+        if not filepath.parent.exists():
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+
+        data = self.model_dump(mode="json")
+        yaml_dump = yaml.safe_dump(data, indent=2, sort_keys=False)
+        filepath.write_text(yaml_dump)
+
+    @classmethod
+    def load(
+        cls, filepath: PathLike, syftbox_config: SyftBoxConfig | None = None
+    ) -> Self:
+        filepath = to_path(filepath)
+        if not filepath.exists():
+            raise FileNotFoundError(f"Config file not found: {filepath}")
+
+        data = yaml.safe_load(filepath.read_text())
+        res = cls.model_validate(data)
+        res._syftbox_config = syftbox_config
+        return res
+
+    def __str__(self) -> str:
+        return self.__display_formatter__.format_str(self)
+
+    def __repr__(self) -> str:
+        return self.__display_formatter__.format_repr(self)
+
+    def _repr_html_(self) -> str:
+        return self.__display_formatter__.format_html(self)
+
+    def _repr_markdown_(self) -> str:
+        return self.__display_formatter__.format_markdown(self)
+
+
+class PrivateDatasetConfig(DatasetBase, PydanticFormatterMixin):
+    """Used to store private dataset metadata, outside of the sync folder."""
+
+    uid: UUID  # id for this dataset
+    data_dir: Path
+
+
+class Dataset(DatasetBase, PydanticFormatterMixin):
+    __table_extra_fields__ = [
+        "name",
+        "owner",
+    ]
+
+    uid: UUID = Field(default_factory=uuid4)
+    created_at: datetime = Field(default_factory=_utcnow)
+    updated_at: datetime = Field(default_factory=_utcnow)
+    name: str
+    summary: str | None = None
+    tags: list[str] = []
+    location: str | None = None
+
+    mock_url: SyftBoxURL
+    private_url: SyftBoxURL
+    readme_url: SyftBoxURL | None = None
+
+    # Absolute paths to uploaded files (excluding metadata files)
+    mock_files_paths: list[Path] = Field(default_factory=list)
+    private_files_paths: list[Path] = Field(default_factory=list)
+
+    @property
+    def owner(self) -> str:
+        return self.mock_url.host
+
+    @property
+    def syftbox_config(self) -> SyftBoxConfig:
+        if self._syftbox_config is None:
+            raise ValueError("SyftBox config is not set.")
+        return self._syftbox_config
+
+    def _url_to_path(self, url: SyftBoxURL) -> Path:
+        return url.to_local_path(
+            syftbox_folder=self.syftbox_config.syftbox_folder,
+        )
+
+    @property
+    def readme_path(self) -> Path | None:
+        if self.readme_url is None:
+            return None
+        return self._url_to_path(self.readme_url)
+
+    def get_readme(self) -> str | None:
+        """Get the content of the README file."""
+        if self.readme_path and self.readme_path.exists():
+            return self.readme_path.read_text()
+        return None
+
+    @property
+    def mock_dir(self) -> Path:
+        return self._url_to_path(self.mock_url)
+
+    @property
+    def private_config_path(self) -> Path:
+        if self.syftbox_config.email != self.owner:
+            raise ValueError(
+                "Cannot access private config for a dataset owned by another user."
+            )
+        return self._private_metadata_dir / "private_metadata.yaml"
+
+    @cached_property
+    def private_config(self) -> PrivateDatasetConfig:
+        config_path = self.private_config_path
+        if not config_path.exists():
+            raise FileNotFoundError(
+                f"Private dataset config not found at {config_path}"
+            )
+
+        return PrivateDatasetConfig.load(
+            filepath=config_path, syftbox_config=self._syftbox_config
+        )
+
+    @property
+    def private_dir(self) -> Path:
+        private_config = self.private_config
+        return private_config.data_dir
+
+    @property
+    def _private_metadata_dir(self) -> Path:
+        if self.syftbox_config.email != self.owner:
+            raise ValueError(
+                "Cannot access private data for a dataset owned by another user."
+            )
+
+        # TODO add 'private' to sb workspace
+        private_datasets_dir = (
+            self.syftbox_config.syftbox_folder / "private" / "syft_datasets"
+        )
+
+        return private_datasets_dir / self.name
+
+    @property
+    def mock_files(self) -> list[Path]:
+        """
+        Get absolute paths to all mock files uploaded during dataset.create.
+        Excludes dataset.yaml and readme.md files.
+        """
+        return self.mock_files_paths
+
+    @property
+    def private_files(self) -> list[Path]:
+        """
+        Get absolute paths to all private files uploaded during dataset.create.
+        Excludes private_metadata.yaml file.
+        """
+        return self.private_files_paths
+
+    @property
+    def files(self) -> list[Path]:
+        """
+        Get absolute paths to all files (both mock and private) uploaded during dataset.create.
+        """
+        return self.mock_files + self.private_files
+
+    def _generate_description_html(self) -> str:
+        from syft_notebook_ui.pydantic_html_repr import create_html_repr
+
+        fields_to_include = ["name", "created_at", "summary", "tags", "location"]
+
+        paths_to_include = []
+        try:
+            paths_to_include.append("mock_dir")
+        except Exception:
+            fields_to_include.append("mock_url")
+
+        try:
+            private_dir = self.private_dir
+            if private_dir.is_dir():
+                paths_to_include.append("private_dir")
+        except Exception:
+            pass
+
+        try:
+            readme_path = self.readme_path
+            if readme_path and readme_path.exists():
+                paths_to_include.append("readme_path")
+        except Exception:
+            fields_to_include.append("readme_url")
+
+        description = create_html_repr(
+            obj=self,
+            fields=fields_to_include,
+            display_paths=paths_to_include,
+        )
+
+        return description
+
+    def describe(self) -> None:
+        from IPython.display import HTML, display
+
+        description = self._generate_description_html()
+        display(HTML(description))
+
+    def _repr_html_(self) -> str:
+        return self._generate_description_html()

syft_datasets/dataset_manager.py

@@ -0,0 +1,377 @@
+import re
+import shutil
+from pathlib import Path
+from typing import Self
+
+from .types import PathLike, to_path
+from syft_notebook_ui.types import TableList
+from typing_extensions import Literal
+
+from syft_datasets.dataset import Dataset, PrivateDatasetConfig
+from syft_datasets.file_utils import copy_dir_contents, copy_paths, is_empty_dir
+
+from .url import SyftBoxURL
+from .config import SyftBoxConfig
+
+FOLDER_NAME = "syft_datasets"
+METADATA_FILENAME = "dataset.yaml"
+
+
+class SyftDatasetManager:
+    def __init__(self, syftbox_folder_path: PathLike, email: str):
+        self.syftbox_config = SyftBoxConfig(
+            syftbox_folder=to_path(syftbox_folder_path), email=email
+        )
+
+    @classmethod
+    def from_config(cls, config: SyftBoxConfig) -> Self:
+        return cls(syftbox_folder_path=config.syftbox_folder, email=config.email)
+
+    def public_dir_for_datasite(self, datasite: str) -> Path:
+        dir = self.syftbox_config.syftbox_folder / datasite / "public" / FOLDER_NAME
+        dir.mkdir(parents=True, exist_ok=True)
+        return dir
+
+    def get_mock_dataset_dir(self, dataset_name: str, datasite: str) -> Path:
+        return self.public_dir_for_datasite(datasite) / dataset_name
+
+    def _validate_dataset_name(self, dataset_name: str) -> None:
+        # Returns True if the dataset is a valid path name on unix or windows.
+        if not re.match(r"^[\w-]+$", dataset_name):
+            raise ValueError(
+                f"Invalid dataset name '{dataset_name}'. Only alphanumeric characters, underscores, and hyphens are allowed."
+            )
+
+    def _prepare_mock_data(self, dataset: Dataset, src_path: Path) -> list[Path]:
+        # Validate src data
+        if not src_path.exists():
+            raise FileNotFoundError(f"Could not find mock data at {src_path}")
+
+        if (src_path / METADATA_FILENAME).exists():
+            raise ValueError(
+                f"Mock data at {src_path} contains reserved file {METADATA_FILENAME}. Please rename it and try again."
+            )
+
+        # Validate dir we're making on Syftbox
+        if dataset.mock_dir.exists() and not is_empty_dir(dataset.mock_dir):
+            raise FileExistsError(
+                f"Mock dir {dataset.mock_dir} already exists and is not empty."
+            )
+        dataset.mock_dir.mkdir(parents=True, exist_ok=True)
+
+        copied_files = []
+        if src_path.is_dir():
+            copied_files = copy_dir_contents(
+                src=src_path,
+                dst=dataset.mock_dir,
+                exists_ok=True,
+            )
+        elif src_path.is_file():
+            copied_files = copy_paths(
+                files=[src_path],
+                dst=dataset.mock_dir,
+                exists_ok=True,
+            )
+        else:
+            raise ValueError(
+                f"Mock data path {src_path} must be an existing file or directory."
+            )
+
+        return copied_files
+
+    def _prepare_private_data(
+        self,
+        dataset: Dataset,
+        src_path: Path,
+    ) -> list[Path]:
+        dataset.private_dir.mkdir(parents=True, exist_ok=True)
+
+        copied_files = []
+        if src_path.is_dir():
+            # TODO: Implementing without copying private data to `SyftBox/private``
+            copied_files = copy_dir_contents(
+                src=src_path,
+                dst=dataset.private_dir,
+                exists_ok=True,
+            )
+        elif src_path.is_file():
+            copied_files = copy_paths(
+                files=[src_path],
+                dst=dataset.private_dir,
+                exists_ok=True,
+            )
+        else:
+            raise ValueError(
+                f"Private data path {src_path} must be an existing file or directory."
+            )
+
+        return copied_files
+
+    def _prepare_private_config(
+        self,
+        dataset: Dataset,
+        private_data_dir: Path,
+        location: str | None = None,
+    ) -> None:
+        """
+        The private dataset config is used to store private metadata separately from the public dataset metadata.
+        """
+        if dataset._private_metadata_dir.exists() and not is_empty_dir(
+            dataset._private_metadata_dir
+        ):
+            raise FileExistsError(
+                f"Private dir {dataset.private_dir} already exists and is not empty."
+            )
+
+        private_config = PrivateDatasetConfig(
+            uid=dataset.uid,
+            data_dir=private_data_dir,
+        )
+
+        private_config_path = dataset.private_config_path
+        private_config_path.parent.mkdir(parents=True, exist_ok=True)
+        private_config.save(filepath=private_config_path)
+
+    def _prepare_readme(self, dataset: Dataset, src_file: Path | None) -> list[Path]:
+        copied_files = []
+        if src_file is not None:
+            if not src_file.is_file():
+                raise FileNotFoundError(f"Could not find README at {src_file}")
+            if not src_file.suffix.lower() == ".md":
+                raise ValueError("readme file must be a markdown (.md) file.")
+            copied_files = copy_paths(
+                files=[src_file],
+                dst=dataset.mock_dir,
+                exists_ok=True,
+            )
+        return copied_files
+
+    def create(
+        self,
+        name: str,
+        mock_path: PathLike,
+        private_path: PathLike,
+        summary: str | None = None,
+        readme_path: Path | None = None,
+        location: str | None = None,
+        tags: list[str] | None = None,
+        # copy_private_data: bool = True, # TODO
+    ) -> Dataset:
+        """_summary_
+
+        Args:
+            name (str): Unique of the dataset to create.
+            mock_path (PathLike): Path to the existing mock data. This can be a file or a directory.
+            private_path (PathLike): Path to the existing private data. This can be a file or a directory.
+            summary (str | None, optional): Short summary of the dataset. Defaults to None.
+            readme_path (Path | None, optional): Markdown README in the public dataset. Defaults to None.
+            location (str | None, optional): Location identifier for the dataset, e.g. 'high-side-1234'.
+                Only required for datasets that are hosted on a remote location and require manual syncing.
+                Defaults to None.
+            tags (list[str] | None, optional): Optional tags for the dataset. Defaults to None.
+
+        Returns:
+            Dataset: The created Dataset object.
+        """
+        mock_path = to_path(mock_path)
+        private_path = to_path(private_path)
+        readme_path = to_path(readme_path) if readme_path else None
+        tags = tags or []
+
+        mock_dir = self.get_mock_dataset_dir(
+            dataset_name=name,
+            datasite=self.syftbox_config.email,
+        )
+        mock_url = SyftBoxURL.from_path(
+            path=mock_dir,
+            syftbox_folder=self.syftbox_config.syftbox_folder,
+        )
+        readme_url = None
+        if readme_path:
+            readme_url = SyftBoxURL.from_path(
+                path=mock_dir / readme_path.name,
+                syftbox_folder=self.syftbox_config.syftbox_folder,
+            )
+
+        # Generate private_url for the dataset
+        # Private URLs use a simple path format
+        private_url = SyftBoxURL(
+            f"syft://private/syft_datasets/{name}",
+        )
+
+        dataset = Dataset(
+            name=name,
+            mock_url=mock_url,
+            private_url=private_url,
+            readme_url=readme_url,
+            summary=summary,
+            location=location,
+            tags=tags,
+        )
+        dataset._syftbox_config = self.syftbox_config
+
+        # Prepare mock data and collect file paths
+        mock_data_files = self._prepare_mock_data(
+            dataset=dataset,
+            src_path=mock_path,
+        )
+
+        # Prepare readme and collect file paths
+        readme_files = self._prepare_readme(
+            dataset=dataset,
+            src_file=readme_path,
+        )
+
+        # TODO enable adding private data without copying to SyftBox
+        # e.g. private_data_dir = dataset._private_metadata_dir if copy_private_data else private_path
+        self._prepare_private_config(
+            dataset=dataset,
+            private_data_dir=dataset._private_metadata_dir,
+        )
+
+        # Prepare private data and collect file paths
+        private_data_files = self._prepare_private_data(
+            dataset=dataset,
+            src_path=private_path,
+        )
+
+        # Store file paths, excluding metadata files
+        dataset_yaml_path = (mock_dir / METADATA_FILENAME).absolute()
+        private_metadata_path = dataset.private_config_path.absolute()
+
+        # Mock files exclude dataset.yaml and readme.md
+        dataset.mock_files_paths = [
+            f
+            for f in mock_data_files
+            if f != dataset_yaml_path and f not in readme_files
+        ]
+
+        # Private files exclude private_metadata.yaml
+        dataset.private_files_paths = [
+            f for f in private_data_files if f != private_metadata_path
+        ]
+
+        # Save dataset metadata
+        dataset.save(filepath=dataset_yaml_path)
+        return dataset
+
+    def _load_dataset_from_dir(self, dataset_dir: Path) -> Dataset:
+        metadata_path = dataset_dir / METADATA_FILENAME
+        if not metadata_path.exists():
+            raise FileNotFoundError(f"Dataset metadata not found at {metadata_path}")
+
+        return Dataset.load(
+            filepath=metadata_path,
+            syftbox_config=self.syftbox_config,
+        )
+
+    def get(self, name: str, datasite: str | None = None) -> Dataset:
+        datasite = datasite or self.syftbox_config.email
+        mock_dir = self.get_mock_dataset_dir(
+            dataset_name=name,
+            datasite=datasite,
+        )
+
+        if not mock_dir.exists():
+            raise FileNotFoundError(f"Dataset {name} not found in {mock_dir}")
+        return self._load_dataset_from_dir(mock_dir)
+
+    def __getitem__(self, key: str) -> Dataset:
+        return self.get(name=key)
+
+    def get_all(
+        self,
+        datasite: str | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
+        order_by: str | None = None,
+        sort_order: Literal["asc", "desc"] = "asc",
+    ) -> list[Dataset]:
+        all_datasets = []
+
+        if datasite:
+            datasites_to_check = [datasite]
+        else:
+            syftbox_folder = self.syftbox_config.syftbox_folder
+            # All directories with "@" in the name are peer/owner email directories
+            datasites_to_check = [
+                d.name for d in syftbox_folder.iterdir() if d.is_dir() and "@" in d.name
+            ]
+
+        for datasite in datasites_to_check:
+            public_datasets_dir = self.public_dir_for_datasite(datasite)
+            if not public_datasets_dir.exists():
+                continue
+            for dataset_dir in public_datasets_dir.iterdir():
+                if dataset_dir.is_dir():
+                    try:
+                        dataset = self._load_dataset_from_dir(dataset_dir)
+                        all_datasets.append(dataset)
+                    except Exception:
+                        continue
+
+        if order_by is not None:
+            all_datasets.sort(
+                key=lambda d: getattr(d, order_by),
+                reverse=(sort_order.lower() == "desc"),
+            )
+
+        if offset is not None:
+            all_datasets = all_datasets[offset:]
+        if limit is not None:
+            all_datasets = all_datasets[:limit]
+
+        return TableList(all_datasets)
+
+    def delete(
+        self,
+        name: str,
+        datasite: str | None = None,
+        require_confirmation: bool = True,
+    ) -> None:
+        datasite = datasite or self.syftbox_config.email
+
+        if datasite != self.syftbox_config.email:
+            # NOTE this check is easily bypassed, but bypassing does not have any effect.
+            # When bypassed, the dataset will be restored because the user only has
+            # read access to someone else's datasite.
+            raise ValueError(
+                "Cannot delete datasets from a datasite that is not your own."
+            )
+
+        try:
+            dataset = self.get(
+                name=name,
+                datasite=datasite,
+            )
+        except FileNotFoundError:
+            raise FileNotFoundError(f"Dataset {name} not found in datasite {datasite}")
+
+        if require_confirmation:
+            msg = (
+                "Deleting this dataset will remove the following folders:\n"
+                f"Mock data: {dataset.mock_dir}\n"
+                f"Private metadata: {dataset._private_metadata_dir}\n"
+            )
+            if (
+                dataset._private_metadata_dir.exists()
+                and dataset.private_dir.resolve().absolute()
+                == dataset._private_metadata_dir.resolve().absolute()
+            ):
+                msg += (
+                    "WARNING: this will also delete the private data from your system\n"
+                )
+            else:
+                msg += "Private data will not be deleted from your system, it is not managed by SyftBox.\n"
+
+            msg += "Are you sure you want to delete these folders? (yes/no): "
+            confirmation = input(msg).strip().lower()
+            if confirmation != "yes":
+                print("Dataset deletion cancelled.")
+                return
+
+        # Delete the dataset directories
+        if dataset.mock_dir.exists():
+            shutil.rmtree(dataset.mock_dir)
+        if dataset._private_metadata_dir.exists():
+            shutil.rmtree(dataset._private_metadata_dir)

syft_datasets/file_utils.py

@@ -0,0 +1,44 @@
+import shutil
+from pathlib import Path
+from typing import Iterator
+
+
+def copy_dir_contents(src: Path, dst: Path, exists_ok: bool = False) -> list[Path]:
+    if not src.is_dir():
+        raise ValueError(f"Source path {src} is not a directory.")
+    return copy_paths(src.iterdir(), dst, exists_ok)
+
+
+def copy_paths(files: Iterator[Path], dst: Path, exists_ok: bool = False) -> list[Path]:
+    """
+    Copy a list of files to a destination directory.
+    If `dst` does not exist, it will be created.
+
+    Returns:
+        list[Path]: List of absolute paths to all copied files (destination paths).
+    """
+    if not dst.exists():
+        dst.mkdir(parents=True, exist_ok=True)
+
+    copied_files = []
+    for file in files:
+        dst_path = dst / file.name
+        if dst_path.exists() and not exists_ok:
+            raise FileExistsError(f"Destination path {dst_path} already exists.")
+        if file.is_file():
+            shutil.copy2(file, dst_path)
+            copied_files.append(dst_path.absolute())
+        elif file.is_dir():
+            shutil.copytree(file, dst_path, dirs_exist_ok=exists_ok)
+            # Get all files in the copied directory tree
+            for copied_file in dst_path.rglob("*"):
+                if copied_file.is_file():
+                    copied_files.append(copied_file.absolute())
+
+    return copied_files
+
+
+def is_empty_dir(path: Path) -> bool:
+    if not path.is_dir():
+        return False
+    return not any(path.iterdir())

syft_datasets/types.py

@@ -0,0 +1,11 @@
+import os
+from pathlib import Path
+from typing import Union
+
+from typing_extensions import TypeAlias
+
+PathLike: TypeAlias = Union[str, Path, os.PathLike]
+
+
+def to_path(path: PathLike) -> Path:
+    return Path(path).expanduser().resolve()

syft_datasets/url.py

@@ -0,0 +1,135 @@
+import re
+from pathlib import Path
+from typing import Any, Dict, Union
+from urllib.parse import urlencode, urlparse
+
+from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler, ValidationInfo
+from pydantic.json_schema import JsonSchemaValue
+from pydantic_core import core_schema
+from typing_extensions import Self
+
+from .types import PathLike, to_path
+
+
+class SyftBoxURL(str):
+    def __new__(cls, url: str):
+        instance = super().__new__(cls, url)
+        if not cls.is_valid(url):
+            raise ValueError(f"Invalid SyftBoxURL: {url}")
+        instance.parsed = urlparse(url)
+        return instance
+
+    @classmethod
+    def is_valid(cls, url: str) -> bool:
+        """Validates the given URL matches the syft:// protocol.
+
+        Supports two formats:
+        1. Email-based: syft://user@domain.com/path (for public data)
+        2. Simple path: syft://path (for private/local data)
+        """
+        # Pattern for email-based URLs (e.g., syft://user@domain.com/path)
+        email_pattern = (
+            r"^syft://([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)(/.*)?$"
+        )
+        # Pattern for simple path URLs (e.g., syft://private/path)
+        simple_pattern = r"^syft://([a-zA-Z0-9_.-]+)(/.*)?$"
+
+        return bool(re.match(email_pattern, url)) or bool(re.match(simple_pattern, url))
+
+    @property
+    def query(self) -> Dict[str, str]:
+        """Returns the query parameters as a dictionary."""
+        if not self.parsed.query:
+            return {}
+
+        return dict(
+            param.split("=", 1)
+            for param in self.parsed.query.split("&")
+            if "=" in param
+        )
+
+    @property
+    def protocol(self) -> str:
+        """Returns the protocol (syft://)."""
+        return self.parsed.scheme + "://"
+
+    @property
+    def host(self) -> str:
+        """Returns the host, which is the email part."""
+        return self.parsed.netloc
+
+    @property
+    def path(self) -> str:
+        """Returns the path component after the email."""
+        return self.parsed.path
+
+    def to_local_path(self, syftbox_folder: PathLike) -> Path:
+        """
+        Converts the SyftBoxURL to a local file system path.
+
+        Args:
+            syftbox_folder (Path): Base SyftBox directory.
+
+        Returns:
+            Path: Local file system path.
+        """
+        # Remove the protocol and prepend the syftbox_folder
+        local_path = to_path(syftbox_folder) / self.host / self.path.lstrip("/")
+        return local_path.resolve()
+
+    def as_http_params(self) -> Dict[str, str]:
+        return {
+            "method": "get",
+            "datasite": self.host,
+            "path": self.path,
+        }
+
+    def to_http_get(self, rpc_url: str) -> str:
+        rpc_url = rpc_url.split("//")[-1]
+        params = self.as_http_params()
+        url_params = urlencode(params)
+        http_url = f"http://{rpc_url}?{url_params}"
+        return http_url
+
+    @classmethod
+    def from_path(cls, path: PathLike, syftbox_folder: PathLike) -> Self:
+        rel_path = to_path(path).relative_to(to_path(syftbox_folder))
+        # convert to posix path to make it work on Windows OS
+        rel_path = rel_path.as_posix()
+        return cls(f"syft://{rel_path}")
+
+    @classmethod
+    def validate(
+        cls, value: Union["SyftBoxURL", str], info: ValidationInfo
+    ) -> "SyftBoxURL":
+        if type(value) not in (str, cls):
+            raise ValueError(
+                f"Invalid type for url: {type(value)}. Expected str or SyftBoxURL."
+            )
+        value = str(value)
+        if not cls.is_valid(value):
+            raise ValueError(f"Invalid SyftBoxURL: {value}")
+        return cls(value)
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls,
+        source_type: Any,
+        handler: GetCoreSchemaHandler,
+    ) -> core_schema.CoreSchema:
+        """Pydantic V2 core schema for custom type validation."""
+        return core_schema.with_info_after_validator_function(
+            cls.validate,
+            handler(str),
+        )
+
+    @classmethod
+    def __get_pydantic_json_schema__(
+        cls, schema_or_field: Any, schema_handler: GetJsonSchemaHandler
+    ) -> JsonSchemaValue:
+        """Define the JSON schema representation for Pydantic models."""
+        return {
+            "type": "string",
+            "format": "uri",
+            "description": "A SyftBox URL",
+        }