temporal-workdir 0.1.0__py3-none-any.whl

temporal_workdir/__init__.py

@@ -0,0 +1,20 @@
+"""Remote workspace sync for Temporal activities.
+
+This package provides a :class:`Workspace` that syncs a local directory with
+remote storage (GCS, S3, Azure, local, etc.) before and after a Temporal
+activity executes. This enables file-based activities to work correctly across
+distributed workers where disk state is not shared.
+
+The storage backend is auto-detected from the URL scheme via `fsspec`_.
+
+.. _fsspec: https://filesystem-spec.readthedocs.io/
+"""
+
+from temporal_workdir._temporal import get_workspace_path, workspace
+from temporal_workdir._workspace import Workspace
+
+__all__ = [
+    "Workspace",
+    "get_workspace_path",
+    "workspace",
+]

temporal_workdir/_archive.py

@@ -0,0 +1,43 @@
+"""Archive utilities for packing/unpacking workspace directories."""
+
+import io
+import tarfile
+from pathlib import Path
+
+
+def pack(directory: Path) -> bytes:
+    """Pack a directory into a gzipped tar archive.
+
+    Args:
+        directory: Local directory to archive. Must exist.
+
+    Returns:
+        The tar.gz archive as bytes.
+    """
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+        for entry in sorted(directory.rglob("*")):
+            if entry.is_file():
+                arcname = str(entry.relative_to(directory))
+                tar.add(str(entry), arcname=arcname)
+    return buf.getvalue()
+
+
+def unpack(data: bytes, directory: Path) -> None:
+    """Unpack a gzipped tar archive into a directory.
+
+    Args:
+        data: The tar.gz archive bytes.
+        directory: Target directory. Created if it doesn't exist.
+    """
+    directory.mkdir(parents=True, exist_ok=True)
+    buf = io.BytesIO(data)
+    with tarfile.open(fileobj=buf, mode="r:gz") as tar:
+        # Security: prevent path traversal
+        for member in tar.getmembers():
+            member_path = Path(directory / member.name).resolve()
+            if not str(member_path).startswith(str(directory.resolve())):
+                raise ValueError(
+                    f"Archive member {member.name!r} would escape target directory"
+                )
+        tar.extractall(path=str(directory), filter="data")

temporal_workdir/_temporal.py

@@ -0,0 +1,112 @@
+"""Temporal-specific integration for Workspace."""
+
+from __future__ import annotations
+
+import contextvars
+import functools
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, TypeVar
+
+import temporalio.activity
+
+from temporal_workdir._workspace import Workspace
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+_current_workspace_path: contextvars.ContextVar[Path | None] = contextvars.ContextVar(
+    "_current_workspace_path", default=None
+)
+
+
+def workspace(
+    remote_url_template: str,
+    key_fn: Callable[..., dict[str, str]] | None = None,
+    **workspace_kwargs: Any,
+) -> Callable[[F], F]:
+    """Decorator that provides a :class:`Workspace` to a Temporal activity.
+
+    The workspace path is available via :func:`get_workspace_path` inside
+    the activity body. The workspace is pulled before execution and pushed
+    after successful completion.
+
+    Template variables in ``remote_url_template`` are resolved from
+    :func:`temporalio.activity.info` and, optionally, from ``key_fn``.
+
+    Built-in template variables (from ``activity.info()``):
+
+    - ``{workflow_id}`` — stable across retries
+    - ``{activity_id}`` — unique per scheduling within a workflow
+    - ``{activity_type}`` — the activity name
+    - ``{task_queue}``
+
+    Example::
+
+        @workspace("gs://bucket/{workflow_id}/{activity_type}")
+        @activity.defn
+        async def process(input: ProcessInput) -> Output:
+            ws = get_workspace_path()
+            data = json.loads((ws / "config.json").read_text())
+            ...
+
+        # With key_fn for custom template vars:
+        @workspace(
+            "gs://bucket/{workflow_id}/{component}",
+            key_fn=lambda input: {"component": input.component_name},
+        )
+        @activity.defn
+        async def process(input: ProcessInput) -> Output:
+            ...
+
+    Args:
+        remote_url_template: URL template with ``{var}`` placeholders.
+        key_fn: Optional function that receives the activity's positional
+            arguments and returns a dict of additional template variables.
+        **workspace_kwargs: Extra keyword arguments forwarded to
+            :class:`Workspace` (e.g., ``cleanup="keep"``).
+    """
+
+    def decorator(fn: F) -> F:
+        @functools.wraps(fn)
+        async def wrapper(*args: Any, **kwargs: Any) -> Any:
+            info = temporalio.activity.info()
+            template_vars: dict[str, str] = {
+                "workflow_id": info.workflow_id or "",
+                "activity_id": info.activity_id,
+                "activity_type": info.activity_type,
+                "task_queue": info.task_queue,
+            }
+            if key_fn is not None:
+                template_vars.update(key_fn(*args))
+
+            remote_url = remote_url_template.format(**template_vars)
+
+            async with Workspace(remote_url, **workspace_kwargs) as ws:
+                token = _current_workspace_path.set(ws.path)
+                try:
+                    return await fn(*args, **kwargs)
+                finally:
+                    _current_workspace_path.reset(token)
+
+        return wrapper  # type: ignore[return-value]
+
+    return decorator
+
+
+def get_workspace_path() -> Path:
+    """Get the workspace path for the currently executing activity.
+
+    Call this from inside an activity decorated with :func:`workspace`.
+
+    Returns:
+        The local workspace :class:`~pathlib.Path`.
+
+    Raises:
+        RuntimeError: If called outside a workspace-decorated activity.
+    """
+    path = _current_workspace_path.get(None)
+    if path is None:
+        raise RuntimeError(
+            "get_workspace_path() called outside a workspace-decorated activity"
+        )
+    return path

temporal_workdir/_workspace.py

@@ -0,0 +1,130 @@
+"""Core Workspace class for syncing file trees with remote storage."""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Literal
+from urllib.parse import urlparse
+
+import fsspec
+
+from temporal_workdir._archive import pack, unpack
+
+
+class Workspace:
+    """Sync a local directory with a remote storage location.
+
+    A Workspace maps a remote URL (the "key") to a local directory. On entry,
+    the remote archive is downloaded and unpacked. On clean exit, the local
+    directory is packed and uploaded back.
+
+    Works with any storage backend supported by fsspec (GCS, S3, Azure, local
+    filesystem, etc.). The backend is auto-detected from the URL scheme.
+
+    Usage::
+
+        async with Workspace("gs://bucket/state/component-x") as ws:
+            data = json.loads((ws.path / "component.json").read_text())
+            (ws.path / "output.csv").write_text("a,b\\n1,2")
+            # On clean exit: local dir archived and uploaded to remote
+
+    Args:
+        remote_url: Remote storage URL. The scheme determines the fsspec
+            backend (``gs://`` for GCS, ``s3://`` for S3, ``file://`` for
+            local, etc.). An ``.tar.gz`` suffix is appended automatically
+            for the archive file.
+        local_path: Local directory to use as the working copy. If ``None``,
+            a temporary directory is created.
+        cleanup: What to do with the local directory after push.
+            ``"auto"`` deletes it, ``"keep"`` leaves it in place.
+        storage_options: Extra keyword arguments passed to
+            ``fsspec.filesystem()``. Use for authentication, project IDs, etc.
+    """
+
+    def __init__(
+        self,
+        remote_url: str,
+        local_path: Path | None = None,
+        cleanup: Literal["auto", "keep"] = "auto",
+        **storage_options: object,
+    ) -> None:
+        self._remote_url = remote_url.rstrip("/")
+        self._archive_url = self._remote_url + ".tar.gz"
+        self._cleanup = cleanup
+        self._storage_options = storage_options
+
+        parsed = urlparse(self._archive_url)
+        self._protocol = parsed.scheme or "file"
+        # fsspec expects path without scheme for most backends
+        self._remote_path = (
+            parsed.netloc + parsed.path if parsed.netloc else parsed.path
+        )
+
+        self._fs = fsspec.filesystem(self._protocol, **storage_options)
+
+        if local_path is not None:
+            self._local_path = local_path
+            self._owns_tempdir = False
+        else:
+            self._local_path = Path(tempfile.mkdtemp(prefix="temporal-workdir-"))
+            self._owns_tempdir = True
+
+    @property
+    def path(self) -> Path:
+        """The local working directory.
+
+        Read and write files here freely. Changes are pushed to remote storage
+        when the context manager exits cleanly.
+        """
+        return self._local_path
+
+    async def pull(self) -> None:
+        """Download and unpack the remote archive to the local directory.
+
+        If no archive exists at the remote URL, the local directory is left
+        empty (first run). Existing local files are removed before unpacking.
+        """
+        if not self._fs.exists(self._remote_path):
+            self._local_path.mkdir(parents=True, exist_ok=True)
+            return
+
+        data = self._fs.cat_file(self._remote_path)
+        # Clear local dir before unpacking to avoid stale files
+        if self._local_path.exists():
+            shutil.rmtree(self._local_path)
+        unpack(data, self._local_path)
+
+    async def push(self) -> None:
+        """Pack the local directory and upload to remote storage.
+
+        If the local directory is empty, the remote archive is deleted
+        (if it exists) to keep storage clean.
+        """
+        files = list(self._local_path.rglob("*"))
+        if not any(f.is_file() for f in files):
+            # Empty workspace — remove remote archive if it exists
+            if self._fs.exists(self._remote_path):
+                self._fs.rm(self._remote_path)
+            return
+
+        data = pack(self._local_path)
+        self._fs.pipe_file(self._remote_path, data)
+
+    async def __aenter__(self) -> Workspace:
+        """Pull remote state and return the workspace."""
+        await self.pull()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: object,
+    ) -> None:
+        """Push local state on clean exit, then optionally clean up."""
+        if exc_type is None:
+            await self.push()
+        if self._cleanup == "auto" and self._owns_tempdir:
+            shutil.rmtree(self._local_path, ignore_errors=True)

temporal_workdir-0.1.0.dist-info/METADATA

@@ -0,0 +1,113 @@
+Metadata-Version: 2.4
+Name: temporal-workdir
+Version: 0.1.0
+Summary: Remote-backed workspace sync for Temporal activities
+Project-URL: Homepage, https://github.com/saeedseyfi/temporal-workdir
+Project-URL: Repository, https://github.com/saeedseyfi/temporal-workdir
+Author-email: Saeed Seyfi <me@saeedseyfi.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: distributed,fsspec,temporal,workflow,workspace
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Requires-Dist: fsspec>=2024.1.0
+Requires-Dist: temporalio>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pyright>=1.1; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.5.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Workspace Sync for Temporal Activities
+
+Sync a local directory with remote storage before and after a Temporal activity. Enables file-based activities to work across distributed workers where disk is not shared.
+
+## Problem
+
+Temporal activities that read/write files on local disk break when you scale to multiple worker instances. Each worker has its own disk. This module syncs a remote storage location to a local temp directory before the activity runs, and pushes changes back after.
+
+## Install
+
+```bash
+pip install temporal-workdir
+
+# With a specific cloud backend:
+pip install temporal-workdir gcsfs    # Google Cloud Storage
+pip install temporal-workdir s3fs     # Amazon S3
+pip install temporal-workdir adlfs    # Azure Blob Storage
+```
+
+## Usage
+
+### As a context manager (generic, works anywhere)
+
+```python
+from temporal_workdir import Workspace
+
+async with Workspace("gs://my-bucket/pipeline/component-x") as ws:
+    # ws.path is a local Path — read and write files normally
+    data = json.loads((ws.path / "component.json").read_text())
+    (ws.path / "result.csv").write_text("col1,col2\nval1,val2")
+    # On clean exit: local dir is archived and uploaded
+    # On exception: no upload (remote state unchanged)
+```
+
+### As a Temporal activity decorator
+
+```python
+from temporalio import activity
+from temporal_workdir import workspace, get_workspace_path
+
+@workspace("gs://my-bucket/{workflow_id}/{activity_type}")
+@activity.defn
+async def extract(input: ExtractInput) -> ExtractOutput:
+    ws = get_workspace_path()
+    # Template vars resolved from activity.info()
+    source = (ws / "source.json").read_text()
+    (ws / "output.csv").write_text(process(source))
+    return ExtractOutput(success=True)
+```
+
+### Custom template variables
+
+```python
+@workspace(
+    "gs://my-bucket/{workflow_id}/components/{component}",
+    key_fn=lambda input: {"component": input.component_name},
+)
+@activity.defn
+async def register(input: RegisterInput) -> RegisterOutput:
+    ws = get_workspace_path()
+    ...
+```
+
+## How It Works
+
+1. **Pull**: On entry, downloads `{remote_url}.tar.gz` and unpacks to a temp directory
+2. **Execute**: Your activity reads/writes files in the local directory
+3. **Push**: On clean exit, packs the directory into `tar.gz` and uploads
+
+If the archive doesn't exist yet (first run), the local directory starts empty. If the activity raises an exception, no push happens. Remote state is untouched.
+
+## Storage Backends
+
+Any backend supported by [fsspec](https://filesystem-spec.readthedocs.io/):
+
+| Scheme | Backend | Extra package |
+|--------|---------|--------------|
+| `gs://` | Google Cloud Storage | `gcsfs` |
+| `s3://` | Amazon S3 | `s3fs` |
+| `az://` | Azure Blob Storage | `adlfs` |
+| `file://` | Local filesystem | (none) |
+| `memory://` | In-memory (testing) | (none) |
+
+Pass backend-specific options as keyword arguments:
+
+```python
+Workspace("gs://bucket/key", project="my-gcp-project", token="cloud")
+```

temporal_workdir-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+temporal_workdir/__init__.py,sha256=XZIJNC0RxXmy6wUsO5ovYP10dilUPDNyqQ960GT2etw,657
+temporal_workdir/_archive.py,sha256=NDMj_3_yLW93ul_nxFXxZwsNHQK9daVasJCY3v0XyJM,1421
+temporal_workdir/_temporal.py,sha256=6AGuYJ4Q4YPuszKx-lgDJwa2EKGKVH3fWhvsY7FmiZ8,3714
+temporal_workdir/_workspace.py,sha256=-6wCo4BjeJ7SaHacDqaqvoJ7IoY7cTLWj5XWHjY2K5Y,4734
+temporal_workdir-0.1.0.dist-info/METADATA,sha256=JZYZLEdNU8Crv84t1P2SxvZHODNwZrJb3ETZxrrFLiI,3956
+temporal_workdir-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+temporal_workdir-0.1.0.dist-info/licenses/LICENSE,sha256=vifq4N7-y4oruUiHcZ67R5Z6VguCHMul7LGv8n3ZrcI,1068
+temporal_workdir-0.1.0.dist-info/RECORD,,

temporal_workdir-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

temporal_workdir-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Saeed Seyfi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.