redivis 0.22.0__py3-none-any.whl

redivis/__init__.py

@@ -0,0 +1,194 @@
+from .classes.Workflow import Workflow as workflow
+from .classes.Dataset import Dataset as dataset
+from .classes.Datasource import Datasource as datasource
+from .classes.User import User as user
+from .classes.Organization import Organization as organization
+from .classes.Parameter import Parameter as parameter
+from .classes.Query import Query as query
+from .classes.Table import Table as table
+from .classes.Notebook import Notebook as notebook
+from .classes.Transform import Transform as transform
+from .common import exceptions
+from .common.api_request import make_request as make_api_request
+from .common.config import config, get_config as _get_config
+
+# Note: these should be deleted at the end to clean up the namespace
+import warnings
+import os
+
+from ._version import __version__
+
+
+def file(*args, **kwargs):
+
+    raise exceptions.DeprecationError(
+        'Calling redivis.file() is no longer supported. Please use redivis.table("table_reference").file("filename") instead.'
+    )
+
+
+def _warning(message, category, filename, lineno, file=None, line=None):
+    print(f"Warning: {message}")
+
+
+warnings.showwarning = _warning
+
+
+def authenticate(scope=None, force_reauthentication=False):
+    from .common.auth import clear_cached_credentials, get_auth_token
+
+    if force_reauthentication:
+        clear_cached_credentials()
+
+    if isinstance(scope, str):
+        scope = [scope]
+
+    get_auth_token(scope=scope)
+
+
+def current_notebook():
+    import os
+
+    if os.getenv("REDIVIS_DEFAULT_NOTEBOOK") is not None:
+        return notebook(os.getenv("REDIVIS_DEFAULT_NOTEBOOK"))
+
+    return None
+
+
+def current_user():
+    res = make_api_request(method="GET", path="/users/me")
+    return user(res["name"], properties=res)
+
+
+def current_workflow():
+    if _get_config("default_workflow") is not None:
+        return workflow(_get_config("default_workflow"))
+
+    return None
+
+
+__all__ = [
+    "workflow",
+    "dataset",
+    "datasource",
+    "user",
+    "organization",
+    "parameter",
+    "query",
+    "table",
+    "notebook",
+    "transform",
+    "exceptions",
+    "make_api_request",
+    "__version__",
+    "authenticate",
+    "current_notebook",
+    "current_user",
+    "current_workflow",
+    "config",
+    "file",
+]
+
+
+def _install_excepthook():
+    import sys
+    import traceback
+    import os
+
+    _package_dir = os.path.dirname(os.path.abspath(__file__))
+    _original_hook = sys.excepthook
+
+    def _format_filtered_tb(exc_type, exc_value, exc_tb):
+
+        entries = traceback.extract_tb(exc_tb)
+
+        # Extract all frames, keep only those outside the redivis package
+        # Also exclude ipython noise
+        # Directories to filter out of tracebacks
+        _filter_dirs = [
+            _package_dir,
+            os.path.join("IPython", ""),
+            os.path.join("ipykernel", ""),
+        ]
+
+        user_entries = [
+            e
+            for e in entries
+            if not any(d in os.path.abspath(e.filename) for d in _filter_dirs)
+        ]
+
+        lines = []
+        lines.append("Traceback (most recent call last):\n")
+        if user_entries:
+            lines.extend(traceback.format_list(user_entries))
+        else:
+            # No user frames; show the last frame before filtering so there's some context
+            last_entry = entries[-1] if entries else None
+            if last_entry:
+                lines.extend(traceback.format_list([last_entry]))
+            else:
+                lines.append("  (full traceback omitted from redivis internals)\n")
+
+        lines.append(f"{exc_type.__name__}: {exc_value}\n")
+        return "".join(lines)
+
+    def _custom_excepthook(exc_type, exc_value, exc_tb):
+        if isinstance(exc_value, exceptions.RedivisError):
+            print(
+                _format_filtered_tb(exc_type, exc_value, exc_tb),
+                file=sys.stderr,
+                end="",
+            )
+        else:
+            _original_hook(exc_type, exc_value, exc_tb)
+
+    sys.excepthook = _custom_excepthook
+
+    try:
+        from IPython import get_ipython
+
+        ipython = get_ipython()
+        if ipython is not None:
+
+            def _ipython_custom_exc(shell, exc_type, exc_value, exc_tb, tb_offset=None):
+                try:
+                    print(
+                        _format_filtered_tb(exc_type, exc_value, exc_tb),
+                        file=sys.stderr,
+                        end="",
+                    )
+                except Exception as e:
+                    print(
+                        "An error occurred while formatting the exception:",
+                        e,
+                        file=sys.stderr,
+                    )
+                    # Fallback to original IPython exception handler
+                    shell.showtraceback(
+                        (exc_type, exc_value, exc_tb), tb_offset=tb_offset
+                    )
+
+            ipython.set_custom_exc((exceptions.RedivisError,), _ipython_custom_exc)
+    except ImportError:
+        pass
+
+
+if os.getenv("REDIVIS_INSTALL_EXCEPTHOOK", "1").lower() not in ("0", "false", "no"):
+    _install_excepthook()
+
+# Register fsspec filesystem if available
+try:
+    from .common.fsspec import register
+
+    register()
+except (ImportError, ModuleNotFoundError):
+    # fsspec or its integration is not installed; registration is optional
+    pass
+except Exception as exc:
+    # Unexpected error during registration; warn rather than fail silently
+    warnings.warn(
+        f"Failed to register Redivis fsspec filesystem due to an unexpected error: {exc}",
+        RuntimeWarning,
+    )
+
+# clean up namespace
+del os, warnings

redivis/_version.py

@@ -0,0 +1 @@
+__version__ = "0.22.0"

redivis/classes/Base.py

@@ -0,0 +1,22 @@
+class Base:
+    def __getitem__(self, key):
+        return (
+            self.properties[key]
+            if hasattr(self, "properties")
+            and self.properties is not None
+            and key in self.properties
+            else None
+        )
+
+    def __repr__(self) -> str:
+        field_strings = []
+        for key, field in vars(self).items():
+            if (
+                key != "properties"
+                and key != "scoped_reference"
+                and field is not None
+                and not isinstance(field, Base)
+            ):
+                field_strings.append(f"{key}:{field!r}")
+
+        return f"<{self.__class__.__name__} {' '.join(field_strings)}>"

redivis/classes/Dataset.py

@@ -0,0 +1,301 @@
+from .Table import Table
+from .Version import Version
+import os
+import warnings
+from .Query import Query
+from .Base import Base
+from urllib.parse import quote as quote_uri
+from ..common.util import split_reference
+import re
+
+from ..common import exceptions
+from ..common.api_request import make_request, make_paginated_request
+from ..common.config import get_config
+
+
+class Dataset(Base):
+    def __init__(
+        self,
+        name,
+        *,
+        version=None,
+        user=None,
+        organization=None,
+        properties=None,
+    ):
+        if not user and not organization:
+            from .User import User
+            from .Organization import Organization
+
+            if len(split_reference(name)) == 2:
+                username, name = split_reference(name)
+                organization = Organization(username)
+                user = User(username)
+            elif get_config("default_user"):
+                user = User(get_config("default_user"))
+            elif get_config("default_organization"):
+                organization = Organization(get_config("default_organization"))
+            else:
+                raise exceptions.ValueError(
+                    "Invalid dataset specifier, must be the fully qualified reference if no owner is specified"
+                )
+
+        self.name = name.split(":")[0]
+        self.user = user
+        self.organization = organization
+        reference_id = ""
+        reference_id_split = re.sub(
+            r":(v\d+[._]\d+|current|next|sample)", "", name
+        ).split(":")
+        if len(reference_id_split) > 1:
+            reference_id = reference_id_split[1]
+
+        if reference_id:
+            reference_id = f":{reference_id}"
+
+        if not version:
+            version_search = re.search(r":(v\d+[._]\d+|current|next)", name)
+            if version_search:
+                version = version_search[1]
+
+        if (
+            version
+            and version != "current"
+            and version != "next"
+            and not version.lower().startswith("v")
+        ):
+            version = f"v{version}"
+
+        self.version_tag = version
+        self.scoped_reference = (properties or {}).get(
+            "scopedReference",
+            f"{self.name}{reference_id}{f':{version}' if version else ''}",
+        )
+        self.qualified_reference = (properties or {}).get(
+            "qualifiedReference",
+            f"{(self.organization or self.user).name}.{self.scoped_reference}",
+        )
+
+        self.uri = f"/datasets/{quote_uri(self.qualified_reference, '')}"
+        self.properties = properties
+
+    def __repr__(self):
+        return f"<Dataset {self.qualified_reference}>"
+
+    def _rectify_ambiguous_owner(self):
+        if self.user and self.organization:
+            if (self.properties or {}).get("owner"):
+                if self.properties.get("owner")["kind"] == "user":
+                    self.organization = None
+                else:
+                    self.user = None
+            elif self.organization.exists():
+                self.user = None
+            else:
+                self.organization = None
+
+    def create(self, *, public_access_level="none", description=None):
+        self._rectify_ambiguous_owner()
+        if self.organization:
+            path = f"{self.organization.uri}/datasets"
+        else:
+            path = f"{self.user.uri}/datasets"
+
+        properties = make_request(
+            method="POST",
+            path=path,
+            payload={
+                "name": self.name,
+                "publicAccessLevel": public_access_level,
+                "description": description,
+            },
+        )
+        update_properties(self, properties)
+        return self
+
+    def create_next_version(self, *, if_not_exists=False):
+        if not self.properties or "nextVersion" not in self.properties:
+            self.get()
+
+        if not self.properties["nextVersion"]:
+            make_request(method="POST", path=f"{self.uri}/versions")
+        elif not if_not_exists:
+            raise exceptions.ValueError(
+                f"Next version already exists at {self.properties['nextVersion']['datasetUri']}. To avoid this error, set argument if_not_exists to True"
+            )
+
+        self._rectify_ambiguous_owner()
+        next_version_dataset = Dataset(
+            name=self.scoped_reference,
+            user=self.user,
+            organization=self.organization,
+            version="next",
+        )
+
+        return next_version_dataset.get()
+
+    def delete(self):
+        make_request(
+            method="DELETE",
+            path=self.uri,
+        )
+        return
+
+    def exists(self):
+        try:
+            make_request(method="HEAD", path=self.uri)
+            return True
+        except exceptions.NotFoundError:
+            return False
+
+    def get(self):
+        properties = make_request(method="GET", path=self.uri)
+        update_properties(self, properties)
+        return self
+
+    def list_tables(self, max_results=None):
+        tables = make_paginated_request(
+            path=f"{self.uri}/tables", page_size=100, max_results=max_results
+        )
+        return [
+            Table(table["name"], dataset=self, properties=table) for table in tables
+        ]
+
+    def list_versions(self, max_results=None):
+        versions = make_paginated_request(
+            path=f"{self.uri}/versions", page_size=100, max_results=max_results
+        )
+        return [
+            Version(version["tag"], dataset=self, properties=version)
+            for version in versions
+        ]
+
+    def query(self, query):
+        return Query(query, default_dataset=self.qualified_reference)
+
+    def release(self, *, release_notes=None, wait_for_finalization=False):
+        version_res = make_request(
+            method="POST",
+            path=f"{self.uri}/versions/next/release",
+            payload={"releaseNotes": release_notes},
+        )
+        if wait_for_finalization:
+            Version(
+                version_res["tag"], dataset=self, properties=version_res
+            )._wait_for_finalization()
+        self.uri = version_res["datasetUri"]
+        self.get()
+        return self
+
+    def unrelease(self, *, wait_for_finalization=False):
+        version_res = make_request(
+            method="POST",
+            path=f"{self.uri}/versions/current/unrelease",
+        )
+        if wait_for_finalization:
+            Version(
+                version_res["tag"], dataset=self, properties=version_res
+            )._wait_for_finalization()
+        self.uri = version_res["datasetUri"]
+        self.get()
+        return self
+
+    def version(self, tag=None):
+        if not self.version_tag and not tag:
+            self.get()
+
+        tag = self.version_tag if tag is None else tag
+        return Version(tag, dataset=self)
+
+    def previous_version(self):
+        version = self.version().previous_version()
+        return version.dataset() if version else None
+
+    def next_version(self):
+        version = self.version().next_version()
+        return version.dataset() if version else None
+
+    def table(self, name):
+        return Table(name, dataset=self)
+
+    def add_labels(self, labels):
+        warnings.warn(
+            "This method is deprecated, and will be removed in a future version. Use dataset.update(labels_to_add=[...]) instead",
+            FutureWarning,
+            stacklevel=2,
+        )
+        return self.update(labels_to_add=labels)
+
+    def remove_labels(self, labels):
+        warnings.warn(
+            "This method is deprecated, and will be removed in a future version. Use dataset.update(labels_to_remove=[...]) instead",
+            FutureWarning,
+            stacklevel=2,
+        )
+        return self.update(labels_to_remove=labels)
+
+    def update(
+        self,
+        *,
+        name=None,
+        public_access_level=None,
+        description=None,
+        methodology_markdown=None,
+        usage_markdown=None,
+        labels=None,
+        labels_to_add=None,
+        labels_to_remove=None,
+        tags=None,
+        tags_to_add=None,
+        tags_to_remove=None,
+    ):
+        payload = {}
+        if name:
+            payload["name"] = name
+        if public_access_level:
+            payload["publicAccessLevel"] = public_access_level
+        if description is not None:
+            payload["description"] = description
+        if methodology_markdown is not None:
+            payload["methodologyMarkdown"] = methodology_markdown
+        if usage_markdown is not None:
+            payload["usageMarkdown"] = usage_markdown
+        if labels is not None:
+            payload["labels"] = labels
+        if labels_to_add is not None:
+            payload["labelsToAdd"] = labels_to_add
+        if labels_to_remove is not None:
+            payload["labelsToRemove"] = labels_to_remove
+        if tags is not None:
+            payload["tags"] = tags
+        if tags_to_add is not None:
+            payload["tagsToAdd"] = tags_to_add
+        if tags_to_remove is not None:
+            payload["tagsToRemove"] = tags_to_remove
+
+        res = make_request(
+            method="PATCH",
+            path=self.uri,
+            payload=payload,
+        )
+        update_properties(self, res)
+        return self
+
+    def update_variables(self, variables):
+        make_request(
+            method="PATCH",
+            path=f"{self.uri}/variables",
+            payload={"variables": variables},
+        )
+
+
+def update_properties(instance, properties):
+    instance.properties = properties
+    instance.qualified_reference = properties["qualifiedReference"]
+    instance.scoped_reference = properties["scopedReference"]
+    instance.name = properties["name"]
+    instance.uri = properties["uri"]
+    if (properties.get("version") or {}).get("tag"):
+        # In the form used in references (v1_0, rather than v1.0)
+        instance.version_tag = properties["version"]["tag"].replace(".", "_")
+    instance._rectify_ambiguous_owner()

redivis/classes/Datasource.py

@@ -0,0 +1,178 @@
+from .Table import Table
+from .Base import Base
+from ..common import exceptions
+from ..common.api_request import make_request
+
+from urllib.parse import quote as quote_uri
+import os
+import re
+from ..common.config import get_config
+
+
+class Datasource(Base):
+    def __init__(self, source, *, workflow=None, properties=None):
+        from .Workflow import Workflow
+        from .Dataset import Dataset
+
+        if not workflow:
+            if get_config("default_workflow"):
+                workflow = Workflow(get_config("default_workflow"))
+            else:
+                raise exceptions.ValueError(
+                    "A workflow must be specified in the Datasource constructor, or set as the default via redivis.config(default_workflow=...) or the REDIVIS_DEFAULT_WORKFLOW environment variable"
+                )
+        if isinstance(source, Dataset) or isinstance(source, Workflow):
+            source = source.qualified_reference
+
+        self.workflow = workflow
+        self.source_reference = source
+        self.uri = (properties or {}).get(
+            "uri", f"{workflow.uri}/dataSources/{quote_uri(source)}"
+        )
+        self.properties = properties
+
+    def __repr__(self):
+        return f"<Datasource {self.source_reference}>"
+
+    def get(self):
+        self.properties = make_request(
+            method="GET",
+            path=self.uri,
+        )
+        self.uri = self.properties["uri"]
+
+        return self
+
+    def create(self):
+        from .Dataset import Dataset
+
+        payload = {}
+        if Dataset(self.source_reference).exists():
+            payload["sourceDataset"] = self.source_reference
+        else:
+            payload["sourceWorkflow"] = self.source_reference
+
+        self.properties = make_request(
+            method="POST", path=f"{self.workflow.uri}/dataSources", payload=payload
+        )
+        self.uri = self.properties["uri"]
+        return self
+
+    def delete(self):
+        # Also deletes the transforms and notebooks that depend on the datasource
+        make_request(method="DELETE", path=self.uri)
+        return
+
+    def exists(self):
+        try:
+            make_request(method="HEAD", path=self.uri)
+            return True
+        except exceptions.NotFoundError:
+            return False
+
+    def source_dataset(self):
+        from .Dataset import Dataset
+
+        try:
+            source_dataset = (self.properties or {})["sourceDataset"]
+        except KeyError:
+            self.get()
+            if not "sourceDataset" in self.properties:
+                raise exceptions.ValueError(
+                    "This datasource doesn't have a source dataset. Use the source_workflow() method instead."
+                )
+            source_dataset = self.properties["sourceDataset"]
+        return Dataset(source_dataset["qualifiedReference"], properties=source_dataset)
+
+    def source_workflow(self):
+        from .Workflow import Workflow
+
+        try:
+            source_workflow = (self.properties or {})["sourceWorkflow"]
+        except KeyError:
+            self.get()
+            if not "sourceWorkflow" in self.properties:
+                raise exceptions.ValueError(
+                    "This datasource doesn't have a source workflow. Use the source_dataset() method instead."
+                )
+            source_workflow = self.properties["sourceWorkflow"]
+        return Workflow(
+            source_workflow["qualifiedReference"], properties=source_workflow
+        )
+
+    def update(
+        self,
+        *,
+        source_dataset=None,
+        source_workflow=None,
+        sample=None,
+        version=None,
+        mapped_tables=None,
+    ):
+        # avoid circular import
+        from .Workflow import Workflow
+        from .Dataset import Dataset
+        from .Version import Version
+
+        if isinstance(source_workflow, Workflow):
+            source_workflow = source_workflow.qualified_reference
+        if isinstance(source_dataset, Dataset):
+            source_dataset = source_dataset.qualified_reference
+        if isinstance(mapped_tables, list):
+            mapped_tables_dict = {}
+            for source, dest in mapped_tables:
+                if isinstance(source, Table):
+                    source = source.scoped_reference
+                if isinstance(dest, Table):
+                    dest = dest.scoped_reference
+                mapped_tables_dict[source] = dest
+            mapped_tables = mapped_tables_dict
+
+        if sample is not None or version is not None:
+            if not source_dataset and not source_workflow:
+                self.get()
+                source_dataset = (self.properties.get("sourceDataset") or {}).get(
+                    "qualifiedReference"
+                )
+            if not source_dataset or source_workflow:
+                if sample:
+                    raise exceptions.ValueError(
+                        "Sampling is not applicable to datasources that reference a workflow"
+                    )
+                if version:
+                    raise exceptions.ValueError(
+                        "Versions are not applicable to datasources that reference a workflow"
+                    )
+            else:
+                # Keep the existing sample setting, unless it's being changed
+                is_sample = (
+                    sample
+                    if sample is not None
+                    else re.search(r":sample\b", source_dataset) is not None
+                )
+                source_dataset = re.sub(r":sample\b", "", source_dataset)
+
+                if version is not None:
+                    if isinstance(version, Version):
+                        version = version.tag
+                    source_dataset = Dataset(
+                        source_dataset, version=version
+                    ).qualified_reference
+
+                if is_sample:
+                    source_dataset += ":sample"
+
+        payload = {
+            "sourceDataset": source_dataset,
+            "sourceWorkflow": source_workflow,
+            "mappedTables": mapped_tables,
+        }
+        self.properties = make_request(
+            method="PATCH",
+            path=self.uri,
+            # Only change what was specified
+            payload={key: value for key, value in payload.items() if value is not None},
+        )
+        self.uri = self.properties["uri"]
+
+        return self