syft-dataset 0.1.0__tar.gz

syft_dataset-0.1.0/.gitignore

@@ -0,0 +1,175 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+credentials/
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# poetry
+poetry.lock
+
+# pdm
+.pdm.toml
+
+# PEP 582
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+.idea/
+
+# VS Code
+.vscode/
+
+# macOS
+.DS_Store
+
+# Google Drive credentials
+credentials.json
+client_secret*.json
+token.pickle
+token.json
+
+# SyftBox related
+~/SyftBox/
+SyftBox/
+
+# Temporary files
+*.tmp
+*.bak
+*.swp
+*~
+
+# Claude AI settings
+.claude/
+# CI test outputs
+test_outputs/
+test_suite_output.log
+
+
+# Notebooks
+notebooks/e2e/sales_mock.csv
+notebooks/e2e/sales_private.csv
+notebooks/e2e/readme.md

syft_dataset-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: syft-dataset
+Version: 0.1.0
+Summary: Syft Datasets
+Author-email: OpenMined <info@openmined.org>
+License: Apache-2.0
+Requires-Python: >=3.12
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: syft-notebook-ui

syft_dataset-0.1.0/pyproject.toml

@@ -0,0 +1,25 @@
+[project]
+name = "syft-dataset"
+version = "0.1.0"
+description = "Syft Datasets"
+authors = [{ name = "OpenMined", email = "info@openmined.org" }]
+license = { text = "Apache-2.0" }
+requires-python = ">=3.12"
+
+dependencies = [
+    "pyyaml>=6.0.3",
+    "syft-notebook-ui",
+]
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/syft_datasets"]
+
+[dependency-groups]
+dev = [
+    "ipykernel>=7.1.0",
+]

syft_dataset-0.1.0/src/syft_datasets/__init__.py

@@ -0,0 +1,3 @@
+from .config import SyftBoxConfig
+
+__all__ = ["SyftBoxConfig"]

syft_dataset-0.1.0/src/syft_datasets/config.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+from pathlib import Path
+
+
+class SyftBoxConfig(BaseModel):
+    syftbox_folder: Path = Field(
+        ..., description="Path to the SyftBox folder on the local filesystem."
+    )
+    email: str = Field(..., description="Email associated with the SyftBox.")
+
+    @property
+    def private_dir(self) -> Path:
+        return self.syftbox_folder / "private"
+
+    @property
+    def public_dir(self) -> Path:
+        return self.syftbox_folder / self.email / "public"

syft_dataset-0.1.0/src/syft_datasets/dataset.py

@@ -0,0 +1,192 @@
+from datetime import datetime, timezone
+from functools import cached_property
+from pathlib import Path
+from typing import ClassVar, Self
+from uuid import UUID, uuid4
+
+import yaml
+from pydantic import BaseModel, Field
+from syft_notebook_ui.formatter_mixin import (
+    ANSIPydanticFormatter,
+    PydanticFormatter,
+    PydanticFormatterMixin,
+)
+
+from .types import PathLike, to_path
+from .url import SyftBoxURL
+from .config import SyftBoxConfig
+
+
+def _utcnow():
+    return datetime.now(tz=timezone.utc)
+
+
+class DatasetBase(BaseModel):
+    __display_formatter__: ClassVar[PydanticFormatter] = ANSIPydanticFormatter()
+    _syftbox_config: SyftBoxConfig | None = None
+
+    def save(self, filepath: PathLike) -> None:
+        filepath = to_path(filepath)
+        if not filepath.suffix == ".yaml":
+            raise ValueError("Model must be saved as a .yaml file.")
+
+        if not filepath.parent.exists():
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+
+        data = self.model_dump(mode="json")
+        yaml_dump = yaml.safe_dump(data, indent=2, sort_keys=False)
+        filepath.write_text(yaml_dump)
+
+    @classmethod
+    def load(
+        cls, filepath: PathLike, syftbox_config: SyftBoxConfig | None = None
+    ) -> Self:
+        filepath = to_path(filepath)
+        if not filepath.exists():
+            raise FileNotFoundError(f"Config file not found: {filepath}")
+
+        data = yaml.safe_load(filepath.read_text())
+        res = cls.model_validate(data)
+        res._syftbox_config = syftbox_config
+        return res
+
+    def __str__(self) -> str:
+        return self.__display_formatter__.format_str(self)
+
+    def __repr__(self) -> str:
+        return self.__display_formatter__.format_repr(self)
+
+    def _repr_html_(self) -> str:
+        return self.__display_formatter__.format_html(self)
+
+    def _repr_markdown_(self) -> str:
+        return self.__display_formatter__.format_markdown(self)
+
+
+class PrivateDatasetConfig(DatasetBase, PydanticFormatterMixin):
+    """Used to store private dataset metadata, outside of the sync folder."""
+
+    uid: UUID  # id for this dataset
+    data_dir: Path
+
+
+class Dataset(DatasetBase, PydanticFormatterMixin):
+    __table_extra_fields__ = [
+        "name",
+        "owner",
+    ]
+
+    uid: UUID = Field(default_factory=uuid4)
+    created_at: datetime = Field(default_factory=_utcnow)
+    updated_at: datetime = Field(default_factory=_utcnow)
+    name: str
+    summary: str | None = None
+    tags: list[str] = []
+    location: str | None = None
+
+    mock_url: SyftBoxURL
+    readme_url: SyftBoxURL | None = None
+
+    @property
+    def owner(self) -> str:
+        return self.mock_url.host
+
+    @property
+    def syftbox_config(self) -> SyftBoxConfig:
+        if self._syftbox_config is None:
+            raise ValueError("SyftBox config is not set.")
+        return self._syftbox_config
+
+    def _url_to_path(self, url: SyftBoxURL) -> Path:
+        return url.to_local_path(
+            syftbox_folder=self.syftbox_config.syftbox_folder,
+        )
+
+    @property
+    def readme_path(self) -> Path | None:
+        if self.readme_url is None:
+            return None
+        return self._url_to_path(self.readme_url)
+
+    def get_readme(self) -> str | None:
+        """Get the content of the README file."""
+        if self.readme_path and self.readme_path.exists():
+            return self.readme_path.read_text()
+        return None
+
+    @property
+    def mock_dir(self) -> Path:
+        return self._url_to_path(self.mock_url)
+
+    @property
+    def private_config_path(self) -> Path:
+        if self.syftbox_config.email != self.owner:
+            raise ValueError(
+                "Cannot access private config for a dataset owned by another user."
+            )
+        return self._private_metadata_dir / "private_metadata.yaml"
+
+    @cached_property
+    def private_config(self) -> PrivateDatasetConfig:
+        config_path = self.private_config_path
+        if not config_path.exists():
+            raise FileNotFoundError(
+                f"Private dataset config not found at {config_path}"
+            )
+
+        return PrivateDatasetConfig.load(
+            filepath=config_path, syftbox_config=self._syftbox_config
+        )
+
+    @property
+    def private_dir(self) -> Path:
+        private_config = self.private_config
+        return private_config.data_dir
+
+    @property
+    def _private_metadata_dir(self) -> Path:
+        if self.syftbox_config.email != self.owner:
+            raise ValueError(
+                "Cannot access private data for a dataset owned by another user."
+            )
+
+        # TODO add 'private' to sb workspace
+        private_datasets_dir = (
+            self.syftbox_config.syftbox_folder / "private" / "syft_datasets"
+        )
+
+        return private_datasets_dir / self.name
+
+    def describe(self) -> None:
+        from IPython.display import HTML, display
+        from syft_notebook_ui.pydantic_html_repr import create_html_repr
+
+        fields_to_include = ["name", "created_at", "summary", "tags", "location"]
+
+        paths_to_include = []
+        try:
+            paths_to_include.append("mock_dir")
+        except Exception:
+            fields_to_include.append("mock_url")
+
+        try:
+            private_dir = self.private_dir
+            if private_dir.is_dir():
+                paths_to_include.append("private_dir")
+        except Exception:
+            pass
+
+        try:
+            readme_path = self.readme_path
+            if readme_path and readme_path.exists():
+                paths_to_include.append("readme_path")
+        except Exception:
+            fields_to_include.append("readme_url")
+
+        description = create_html_repr(
+            obj=self,
+            fields=fields_to_include,
+            display_paths=paths_to_include,
+        )
+
+        display(HTML(description))

syft_dataset-0.1.0/src/syft_datasets/dataset_manager.py

@@ -0,0 +1,341 @@
+import re
+import shutil
+from pathlib import Path
+from typing import Self
+
+from .types import PathLike, to_path
+from syft_notebook_ui.types import TableList
+from typing_extensions import Literal
+
+from syft_datasets.dataset import Dataset, PrivateDatasetConfig
+from syft_datasets.file_utils import copy_dir_contents, copy_paths, is_empty_dir
+
+from .url import SyftBoxURL
+from .config import SyftBoxConfig
+
+FOLDER_NAME = "syft_datasets"
+METADATA_FILENAME = "dataset.yaml"
+
+
+class SyftDatasetManager:
+    def __init__(self, syftbox_folder_path: PathLike, email: str):
+        self.syftbox_config = SyftBoxConfig(
+            syftbox_folder=to_path(syftbox_folder_path), email=email
+        )
+
+    @classmethod
+    def from_config(cls, config: SyftBoxConfig) -> Self:
+        return cls(syftbox_folder_path=config.syftbox_folder, email=config.email)
+
+    def public_dir_for_datasite(self, datasite: str) -> Path:
+        dir = self.syftbox_config.syftbox_folder / datasite / "public" / FOLDER_NAME
+        dir.mkdir(parents=True, exist_ok=True)
+        return dir
+
+    def get_mock_dataset_dir(self, dataset_name: str, datasite: str) -> Path:
+        return self.public_dir_for_datasite(datasite) / dataset_name
+
+    def _validate_dataset_name(self, dataset_name: str) -> None:
+        # Returns True if the dataset is a valid path name on unix or windows.
+        if not re.match(r"^[\w-]+$", dataset_name):
+            raise ValueError(
+                f"Invalid dataset name '{dataset_name}'. Only alphanumeric characters, underscores, and hyphens are allowed."
+            )
+
+    def _prepare_mock_data(self, dataset: Dataset, src_path: Path) -> None:
+        # Validate src data
+        if not src_path.exists():
+            raise FileNotFoundError(f"Could not find mock data at {src_path}")
+
+        if (src_path / METADATA_FILENAME).exists():
+            raise ValueError(
+                f"Mock data at {src_path} contains reserved file {METADATA_FILENAME}. Please rename it and try again."
+            )
+
+        # Validate dir we're making on Syftbox
+        if dataset.mock_dir.exists() and not is_empty_dir(dataset.mock_dir):
+            raise FileExistsError(
+                f"Mock dir {dataset.mock_dir} already exists and is not empty."
+            )
+        dataset.mock_dir.mkdir(parents=True, exist_ok=True)
+
+        if src_path.is_dir():
+            copy_dir_contents(
+                src=src_path,
+                dst=dataset.mock_dir,
+                exists_ok=True,
+            )
+        elif src_path.is_file():
+            copy_paths(
+                files=[src_path],
+                dst=dataset.mock_dir,
+                exists_ok=True,
+            )
+        else:
+            raise ValueError(
+                f"Mock data path {src_path} must be an existing file or directory."
+            )
+
+    def _prepare_private_data(
+        self,
+        dataset: Dataset,
+        src_path: Path,
+    ) -> None:
+        dataset.private_dir.mkdir(parents=True, exist_ok=True)
+
+        if src_path.is_dir():
+            # TODO: Implementing without copying private data to `SyftBox/private``
+            copy_dir_contents(
+                src=src_path,
+                dst=dataset.private_dir,
+                exists_ok=True,
+            )
+        elif src_path.is_file():
+            copy_paths(
+                files=[src_path],
+                dst=dataset.private_dir,
+                exists_ok=True,
+            )
+        else:
+            raise ValueError(
+                f"Private data path {src_path} must be an existing file or directory."
+            )
+
+    def _prepare_private_config(
+        self,
+        dataset: Dataset,
+        private_data_dir: Path,
+        location: str | None = None,
+    ) -> None:
+        """
+        The private dataset config is used to store private metadata separately from the public dataset metadata.
+        """
+        if dataset._private_metadata_dir.exists() and not is_empty_dir(
+            dataset._private_metadata_dir
+        ):
+            raise FileExistsError(
+                f"Private dir {dataset.private_dir} already exists and is not empty."
+            )
+
+        private_config = PrivateDatasetConfig(
+            uid=dataset.uid,
+            data_dir=private_data_dir,
+            location=location,
+        )
+
+        private_config_path = dataset.private_config_path
+        private_config_path.parent.mkdir(parents=True, exist_ok=True)
+        private_config.save(filepath=private_config_path)
+
+    def _prepare_readme(self, dataset: Dataset, src_file: Path | None) -> None:
+        if src_file is not None:
+            if not src_file.is_file():
+                raise FileNotFoundError(f"Could not find README at {src_file}")
+            if not src_file.suffix.lower() == ".md":
+                raise ValueError("readme file must be a markdown (.md) file.")
+            copy_paths(
+                files=[src_file],
+                dst=dataset.mock_dir,
+                exists_ok=True,
+            )
+
+    def create(
+        self,
+        name: str,
+        mock_path: PathLike,
+        private_path: PathLike,
+        summary: str | None = None,
+        readme_path: Path | None = None,
+        location: str | None = None,
+        tags: list[str] | None = None,
+        # copy_private_data: bool = True, # TODO
+    ) -> Dataset:
+        """_summary_
+
+        Args:
+            name (str): Unique of the dataset to create.
+            mock_path (PathLike): Path to the existing mock data. This can be a file or a directory.
+            private_path (PathLike): Path to the existing private data. This can be a file or a directory.
+            summary (str | None, optional): Short summary of the dataset. Defaults to None.
+            readme_path (Path | None, optional): Markdown README in the public dataset. Defaults to None.
+            location (str | None, optional): Location identifier for the dataset, e.g. 'high-side-1234'.
+                Only required for datasets that are hosted on a remote location and require manual syncing.
+                Defaults to None.
+            tags (list[str] | None, optional): Optional tags for the dataset. Defaults to None.
+
+        Returns:
+            Dataset: The created Dataset object.
+        """
+        mock_path = to_path(mock_path)
+        private_path = to_path(private_path)
+        readme_path = to_path(readme_path) if readme_path else None
+        tags = tags or []
+
+        mock_dir = self.get_mock_dataset_dir(
+            dataset_name=name,
+            datasite=self.syftbox_config.email,
+        )
+        mock_url = SyftBoxURL.from_path(
+            path=mock_dir,
+            syftbox_folder=self.syftbox_config.syftbox_folder,
+        )
+        readme_url = None
+        if readme_path:
+            readme_url = SyftBoxURL.from_path(
+                path=mock_dir / readme_path.name,
+                syftbox_folder=self.syftbox_config.syftbox_folder,
+            )
+
+        dataset = Dataset(
+            name=name,
+            mock_url=mock_url,
+            readme_url=readme_url,
+            summary=summary,
+            location=location,
+            tags=tags,
+        )
+        dataset._syftbox_config = self.syftbox_config
+
+        self._prepare_mock_data(
+            dataset=dataset,
+            src_path=mock_path,
+        )
+        self._prepare_readme(
+            dataset=dataset,
+            src_file=readme_path,
+        )
+
+        # TODO enable adding private data without copying to SyftBox
+        # e.g. private_data_dir = dataset._private_metadata_dir if copy_private_data else private_path
+        self._prepare_private_config(
+            dataset=dataset,
+            private_data_dir=dataset._private_metadata_dir,
+        )
+        self._prepare_private_data(
+            dataset=dataset,
+            src_path=private_path,
+        )
+
+        dataset_yaml_path = mock_dir / METADATA_FILENAME
+        dataset.save(filepath=dataset_yaml_path)
+        return dataset
+
+    def _load_dataset_from_dir(self, dataset_dir: Path) -> Dataset:
+        metadata_path = dataset_dir / METADATA_FILENAME
+        if not metadata_path.exists():
+            raise FileNotFoundError(f"Dataset metadata not found at {metadata_path}")
+
+        return Dataset.load(
+            filepath=metadata_path,
+            syftbox_config=self.syftbox_config,
+        )
+
+    def get(self, name: str, datasite: str | None = None) -> Dataset:
+        datasite = datasite or self.syftbox_config.email
+        mock_dir = self.get_mock_dataset_dir(
+            dataset_name=name,
+            datasite=datasite,
+        )
+
+        if not mock_dir.exists():
+            raise FileNotFoundError(f"Dataset {name} not found in {mock_dir}")
+        return self._load_dataset_from_dir(mock_dir)
+
+    def __getitem__(self, key: str) -> Dataset:
+        return self.get(name=key)
+
+    def get_all(
+        self,
+        datasite: str | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
+        order_by: str | None = None,
+        sort_order: Literal["asc", "desc"] = "asc",
+    ) -> list[Dataset]:
+        all_datasets = []
+
+        if datasite:
+            datasites_to_check = [datasite]
+        else:
+            syftbox_folder = self.syftbox_config.syftbox_folder
+            # All directories with "@" in the name are peer/owner email directories
+            datasites_to_check = [
+                d.name for d in syftbox_folder.iterdir() if d.is_dir() and "@" in d.name
+            ]
+
+        for datasite in datasites_to_check:
+            public_datasets_dir = self.public_dir_for_datasite(datasite)
+            if not public_datasets_dir.exists():
+                continue
+            for dataset_dir in public_datasets_dir.iterdir():
+                if dataset_dir.is_dir():
+                    try:
+                        dataset = self._load_dataset_from_dir(dataset_dir)
+                        all_datasets.append(dataset)
+                    except Exception:
+                        continue
+
+        if order_by is not None:
+            all_datasets.sort(
+                key=lambda d: getattr(d, order_by),
+                reverse=(sort_order.lower() == "desc"),
+            )
+
+        if offset is not None:
+            all_datasets = all_datasets[offset:]
+        if limit is not None:
+            all_datasets = all_datasets[:limit]
+
+        return TableList(all_datasets)
+
+    def delete(
+        self,
+        name: str,
+        datasite: str | None = None,
+        require_confirmation: bool = True,
+    ) -> None:
+        datasite = datasite or self.syftbox_config.email
+
+        if datasite != self.syftbox_config.email:
+            # NOTE this check is easily bypassed, but bypassing does not have any effect.
+            # When bypassed, the dataset will be restored because the user only has
+            # read access to someone else's datasite.
+            raise ValueError(
+                "Cannot delete datasets from a datasite that is not your own."
+            )
+
+        try:
+            dataset = self.get(
+                name=name,
+                datasite=datasite,
+            )
+        except FileNotFoundError:
+            raise FileNotFoundError(f"Dataset {name} not found in datasite {datasite}")
+
+        if require_confirmation:
+            msg = (
+                "Deleting this dataset will remove the following folders:\n"
+                f"Mock data: {dataset.mock_dir}\n"
+                f"Private metadata: {dataset._private_metadata_dir}\n"
+            )
+            if (
+                dataset.private_dir.resolve().absolute()
+                == dataset._private_metadata_dir.resolve().absolute()
+            ):
+                msg += (
+                    "WARNING: this will also delete the private data from your system\n"
+                )
+            else:
+                msg += "Private data will not be deleted from your system, it is not managed by SyftBox.\n"
+
+            msg += "Are you sure you want to delete these folders? (yes/no): "
+            confirmation = input(msg).strip().lower()
+            if confirmation != "yes":
+                print("Dataset deletion cancelled.")
+                return
+
+        # Delete the dataset directories
+        if dataset.mock_dir.exists():
+            shutil.rmtree(dataset.mock_dir)
+        if dataset._private_metadata_dir.exists():
+            shutil.rmtree(dataset._private_metadata_dir)

syft_dataset-0.1.0/src/syft_datasets/file_utils.py

@@ -0,0 +1,33 @@
+import shutil
+from pathlib import Path
+from typing import Iterator
+
+
+def copy_dir_contents(src: Path, dst: Path, exists_ok: bool = False) -> None:
+    if not src.is_dir():
+        raise ValueError(f"Source path {src} is not a directory.")
+    return copy_paths(src.iterdir(), dst, exists_ok)
+
+
+def copy_paths(files: Iterator[Path], dst: Path, exists_ok: bool = False) -> None:
+    """
+    Copy a list of files to a destination directory.
+    If `dst` does not exist, it will be created.
+    """
+    if not dst.exists():
+        dst.mkdir(parents=True, exist_ok=True)
+
+    for file in files:
+        dst_path = dst / file.name
+        if dst_path.exists() and not exists_ok:
+            raise FileExistsError(f"Destination path {dst_path} already exists.")
+        if file.is_file():
+            shutil.copy2(file, dst_path)
+        elif file.is_dir():
+            shutil.copytree(file, dst_path, dirs_exist_ok=exists_ok)
+
+
+def is_empty_dir(path: Path) -> bool:
+    if not path.is_dir():
+        return False
+    return not any(path.iterdir())

syft_dataset-0.1.0/src/syft_datasets/types.py

@@ -0,0 +1,11 @@
+import os
+from pathlib import Path
+from typing import Union
+
+from typing_extensions import TypeAlias
+
+PathLike: TypeAlias = Union[str, Path, os.PathLike]
+
+
+def to_path(path: PathLike) -> Path:
+    return Path(path).expanduser().resolve()

syft_dataset-0.1.0/src/syft_datasets/url.py

@@ -0,0 +1,124 @@
+import re
+from pathlib import Path
+from typing import Any, Dict, Union
+from urllib.parse import urlencode, urlparse
+
+from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler, ValidationInfo
+from pydantic.json_schema import JsonSchemaValue
+from pydantic_core import core_schema
+from typing_extensions import Self
+
+from .types import PathLike, to_path
+
+
+class SyftBoxURL(str):
+    def __new__(cls, url: str):
+        instance = super().__new__(cls, url)
+        if not cls.is_valid(url):
+            raise ValueError(f"Invalid SyftBoxURL: {url}")
+        instance.parsed = urlparse(url)
+        return instance
+
+    @classmethod
+    def is_valid(cls, url: str) -> bool:
+        """Validates the given URL matches the syft:// protocol and email-based schema."""
+        pattern = r"^syft://([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)(/.*)?$"
+        return bool(re.match(pattern, url))
+
+    @property
+    def query(self) -> Dict[str, str]:
+        """Returns the query parameters as a dictionary."""
+        if not self.parsed.query:
+            return {}
+
+        return dict(
+            param.split("=", 1)
+            for param in self.parsed.query.split("&")
+            if "=" in param
+        )
+
+    @property
+    def protocol(self) -> str:
+        """Returns the protocol (syft://)."""
+        return self.parsed.scheme + "://"
+
+    @property
+    def host(self) -> str:
+        """Returns the host, which is the email part."""
+        return self.parsed.netloc
+
+    @property
+    def path(self) -> str:
+        """Returns the path component after the email."""
+        return self.parsed.path
+
+    def to_local_path(self, syftbox_folder: PathLike) -> Path:
+        """
+        Converts the SyftBoxURL to a local file system path.
+
+        Args:
+            syftbox_folder (Path): Base SyftBox directory.
+
+        Returns:
+            Path: Local file system path.
+        """
+        # Remove the protocol and prepend the syftbox_folder
+        local_path = to_path(syftbox_folder) / self.host / self.path.lstrip("/")
+        return local_path.resolve()
+
+    def as_http_params(self) -> Dict[str, str]:
+        return {
+            "method": "get",
+            "datasite": self.host,
+            "path": self.path,
+        }
+
+    def to_http_get(self, rpc_url: str) -> str:
+        rpc_url = rpc_url.split("//")[-1]
+        params = self.as_http_params()
+        url_params = urlencode(params)
+        http_url = f"http://{rpc_url}?{url_params}"
+        return http_url
+
+    @classmethod
+    def from_path(cls, path: PathLike, syftbox_folder: PathLike) -> Self:
+        rel_path = to_path(path).relative_to(to_path(syftbox_folder))
+        # convert to posix path to make it work on Windows OS
+        rel_path = rel_path.as_posix()
+        return cls(f"syft://{rel_path}")
+
+    @classmethod
+    def validate(
+        cls, value: Union["SyftBoxURL", str], info: ValidationInfo
+    ) -> "SyftBoxURL":
+        if type(value) not in (str, cls):
+            raise ValueError(
+                f"Invalid type for url: {type(value)}. Expected str or SyftBoxURL."
+            )
+        value = str(value)
+        if not cls.is_valid(value):
+            raise ValueError(f"Invalid SyftBoxURL: {value}")
+        return cls(value)
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls,
+        source_type: Any,
+        handler: GetCoreSchemaHandler,
+    ) -> core_schema.CoreSchema:
+        """Pydantic V2 core schema for custom type validation."""
+        return core_schema.with_info_after_validator_function(
+            cls.validate,
+            handler(str),
+        )
+
+    @classmethod
+    def __get_pydantic_json_schema__(
+        cls, schema_or_field: Any, schema_handler: GetJsonSchemaHandler
+    ) -> JsonSchemaValue:
+        """Define the JSON schema representation for Pydantic models."""
+        return {
+            "type": "string",
+            "format": "uri",
+            "description": "A SyftBox URL",
+        }