datashare-python 0.1.3__tar.gz → 0.2.10__tar.gz

datashare_python-0.2.10/.gitignore

@@ -0,0 +1,21 @@
+.idea/
+# Python
+*.log
+venv
+*.egg-info
+.eggs
+__pycache__
+*.pytest_cache
+*.pyc
+build
+dist
+test/.env
+.cache
+tmp
+./*.csv
+.DS_Store
+
+# VS code
+.vscode
+# Doc
+site

datashare_python-0.2.10/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: datashare-python
+Version: 0.2.10
+Summary: Manage Python tasks and local resources in Datashare
+Project-URL: Homepage, https://icij.github.io/datashare-python/
+Project-URL: Documentation, https://icij.github.io/datashare-python/
+Project-URL: Repository, https://github.com/ICIJ/datashare-python
+Project-URL: Issues, https://github.com/ICIJ/datashare-python/issues
+Author-email: Clément Doumouro <cdoumouro@icij.org>, Clément Doumouro <clement.doumouro@gmail.com>, Lion Summerbell <lsummerbell@icij.org>
+Requires-Python: <4,>=3.11
+Requires-Dist: aiohttp~=3.11.9
+Requires-Dist: aiostream~=0.6.4
+Requires-Dist: alive-progress~=3.2.0
+Requires-Dist: icij-common[elasticsearch]~=0.7.3
+Requires-Dist: nest-asyncio~=1.6.0
+Requires-Dist: python-json-logger~=4.0.0
+Requires-Dist: temporalio~=1.23.0
+Requires-Dist: tomlkit>=0.14.0
+Requires-Dist: typer~=0.15.4
+Requires-Dist: worker-template

datashare_python-0.2.10/datashare_python/.gitignore

@@ -0,0 +1 @@
+worker-template.tar.gz

{datashare_python-0.1.3 → datashare_python-0.2.10}/datashare_python/__main__.py

@@ -1,7 +1,7 @@
-from datashare_python.cli import cli_app
+from .cli import cli_app
 
 
-def main():
+def main() -> None:
     cli_app()
 
 

datashare_python-0.2.10/datashare_python/cli/__init__.py

@@ -0,0 +1,52 @@
+import importlib.metadata
+import os
+from typing import Annotated
+
+import typer
+from icij_common.logging_utils import setup_loggers
+
+import datashare_python
+from datashare_python.cli.local import local_app
+from datashare_python.cli.project import project_app
+from datashare_python.cli.task import task_app
+from datashare_python.cli.utils import AsyncTyper
+from datashare_python.cli.worker import worker_app
+
+cli_app = AsyncTyper(
+    context_settings={"help_option_names": ["-h", "--help"]},
+    pretty_exceptions_enable=False,
+)
+cli_app.add_typer(local_app)
+cli_app.add_typer(project_app)
+cli_app.add_typer(task_app)
+cli_app.add_typer(worker_app)
+
+
+def version_callback(value: bool) -> None:  # noqa: FBT001
+    if value:
+        package_version = importlib.metadata.version(datashare_python.__name__)
+        print(package_version)
+        raise typer.Exit()
+
+
+def pretty_exc_callback(value: bool) -> None:  # noqa: FBT001
+    if not value:
+        os.environ["TYPER_STANDARD_TRACEBACK"] = "1"
+
+
+@cli_app.callback()
+def main(
+    version: Annotated[  # noqa: ARG001
+        bool | None,
+        typer.Option("--version", callback=version_callback, is_eager=True),
+    ] = None,
+    *,
+    pretty_exceptions: Annotated[  # noqa: ARG001
+        bool,
+        typer.Option(
+            "--pretty-exceptions", callback=pretty_exc_callback, is_eager=True
+        ),
+    ] = False,
+) -> None:
+    """Datashare Python CLI."""
+    setup_loggers(["__main__", datashare_python.__name__])

datashare_python-0.2.10/datashare_python/cli/local.py

@@ -0,0 +1,33 @@
+from typing import Annotated
+
+import typer
+
+from datashare_python.cli.utils import AsyncTyper
+from datashare_python.constants import DEFAULT_NAMESPACE, DEFAULT_TEMPORAL_ADDRESS
+from datashare_python.local_client import LocalClient
+
+_REGISTER_NAMESPACE_HELP = "register namespace"
+_TEMPORAL_URL_HELP = "address for temporal server"
+_NAMESPACE_HELP = "namespace name"
+_LOCAL = "local"
+
+local_app = AsyncTyper(name=_LOCAL)
+
+
+@local_app.async_command(help=_REGISTER_NAMESPACE_HELP)
+async def register_namespace(
+    namespace: Annotated[
+        str, typer.Option("--namespace", "-n", help=_NAMESPACE_HELP)
+    ] = DEFAULT_NAMESPACE,
+    temporal_address: Annotated[
+        str, typer.Option("--temporal-address", "-a", help=_TEMPORAL_URL_HELP)
+    ] = DEFAULT_TEMPORAL_ADDRESS,
+) -> None:
+    """Create namespace
+
+    :param namespace: namespace
+    :param temporal_address: target host
+    """
+    client = LocalClient()
+
+    await client.register_namespace(temporal_address, namespace)

datashare_python-0.2.10/datashare_python/cli/project.py

@@ -0,0 +1,34 @@
+import logging
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from datashare_python.template import init_project
+
+from .utils import AsyncTyper, eprint
+
+_INIT_PROJECT_HELP = "initialize a new worker project from a template"
+_INIT_PROJECT_NAME_HELP = "name of the new worker package"
+_INIT_PROJECT_PATH_HELP = "path where project will be created"
+
+_PROJECT = "project"
+
+project_app = AsyncTyper(name=_PROJECT)
+
+logger = logging.getLogger(__name__)
+
+
+@project_app.async_command(help=_INIT_PROJECT_HELP)
+async def init(
+    name: Annotated[str, typer.Argument(help=_INIT_PROJECT_NAME_HELP)],
+    path: Annotated[
+        Path | None, typer.Option("--path", "-p", help=_INIT_PROJECT_PATH_HELP)
+    ] = None,
+) -> None:
+    if path is None:
+        path = Path(".")
+    eprint(f"Initializing {name} worker project in {path.absolute()}...")
+    init_project(name, path)
+    eprint(f"Project {name} initialized !")
+    print(path)

datashare_python-0.1.3/datashare_python/cli/tasks.py → datashare_python-0.2.10/datashare_python/cli/task.py

@@ -4,15 +4,14 @@ import logging
 import sys
 from pathlib import Path
 from traceback import FrameSummary, StackSummary
-from typing import Annotated, Any, Optional
+from typing import Annotated, Any
 
 import typer
 from alive_progress import alive_bar
-from icij_worker import TaskState
-from icij_worker.objects import READY_STATES, Task, TaskError
 
 from datashare_python.cli.utils import AsyncTyper, eprint
 from datashare_python.constants import PYTHON_TASK_GROUP
+from datashare_python.objects import READY_STATES, Task, TaskError, TaskState
 from datashare_python.task_client import DatashareTaskClient
 
 logger = logging.getLogger(__name__)
@@ -40,16 +39,16 @@ async def start(
     name: Annotated[str, typer.Argument(help=_NAME_HELP)],
     args: Annotated[TaskArgs, typer.Argument(help=_ARGS_HELP)] = None,
     group: Annotated[
-        Optional[str],
+        str | None,
         typer.Option("--group", "-g", help=_GROUP_HELP),
     ] = PYTHON_TASK_GROUP.name,
     ds_address: Annotated[
         str, typer.Option("--ds-address", "-a", help=_DS_URL_HELP)
     ] = DEFAULT_DS_ADDRESS,
     ds_api_key: Annotated[
-        Optional[str], typer.Option("--ds-api-key", "-k", help=_DS_API_KEY_HELP)
+        str | None, typer.Option("--ds-api-key", "-k", help=_DS_API_KEY_HELP)
     ] = None,
-):
+) -> None:
     match args:
         case str():
             as_path = Path(name)
@@ -76,16 +75,16 @@ async def watch(
         str, typer.Option("--ds-address", "-a", help=_DS_URL_HELP)
     ] = DEFAULT_DS_ADDRESS,
     ds_api_key: Annotated[
-        Optional[str], typer.Option("--ds-api-key", "-k", help=_DS_API_KEY_HELP)
+        str | None, typer.Option("--ds-api-key", "-k", help=_DS_API_KEY_HELP)
     ] = None,
     polling_interval_s: Annotated[
         float, typer.Option("--polling-interval-s", "-p", help=_POLLING_INTERVAL_S_HELP)
     ] = 1.0,
-):
+) -> None:
     client = DatashareTaskClient(ds_address, api_key=ds_api_key)
     async with client:
         task = await client.get_task(task_id)
-        if task.state is READY_STATES:
+        if task.state in READY_STATES:
             await _handle_ready(task, client, already_done=True)
         await _handle_alive(task, client, polling_interval_s)
     print(task_id)
@@ -98,19 +97,19 @@ async def result(
         str, typer.Option("--ds-address", "-a", help=_DS_URL_HELP)
     ] = DEFAULT_DS_ADDRESS,
     ds_api_key: Annotated[
-        Optional[str], typer.Option("--ds-api-key", "-k", help=_DS_API_KEY_HELP)
+        str | None, typer.Option("--ds-api-key", "-k", help=_DS_API_KEY_HELP)
     ] = None,
 ) -> Any:
     client = DatashareTaskClient(ds_address, api_key=ds_api_key)
     async with client:
         res = await client.get_task_result(task_id)
-        if isinstance(res, (dict, list)):
+        if isinstance(res, dict | list):
             res = json.dumps(res, indent=2)
         print(res)
 
 
 async def _handle_ready(
-    task: Task, client: DatashareTaskClient, already_done: bool = False
+    task: Task, client: DatashareTaskClient, *, already_done: bool = False
 ) -> None:
     match task.state:
         case TaskState.ERROR:
@@ -126,27 +125,26 @@ async def _handle_ready(
             raise ValueError(f"Unexpected task state {task.state}")
 
 
-async def _handle_error(task, client: DatashareTaskClient):
+async def _handle_error(task: Task, client: DatashareTaskClient) -> None:
     error = await client.get_task_error(task.id)
     eprint(
-        f"Task({task.id}) failed with the following"
-        f" error:\n\n{_format_error(error)}"
+        f"Task({task.id}) failed with the following error:\n\n{_format_error(error)}"
     )
     eprint(f"Task({task.id}) ❌")
     raise typer.Exit(code=1)
 
 
-async def _handle_cancelled(task):
+async def _handle_cancelled(task: Task) -> None:
     eprint(f"Task({task.id}) was cancelled !")
     eprint(f"Task({task.id}) 🛑")
     raise typer.Exit(code=1)
 
 
-async def _handle_already_done(task):
+async def _handle_already_done(task: Task) -> None:
     eprint(f"Task({task.id}) ✅ is already completed !")
 
 
-async def _handle_done(task):
+async def _handle_done(task: Task) -> None:
     eprint(f"Task({task.id}) 🛬")
     eprint(f"Task({task.id}) ✅")
 
@@ -166,7 +164,7 @@ async def _handle_alive(
             task = await client.get_task(task.id)
             task_state = task.state
             progress = task.progress or 0.0
-            bar(progress)  # pylint: disable=not-callable
+            bar(progress)
             await asyncio.sleep(polling_interval_s)
     if task_state in READY_STATES:
         await _handle_ready(task, client)

{datashare_python-0.1.3 → datashare_python-0.2.10}/datashare_python/cli/utils.py

@@ -1,16 +1,18 @@
 import asyncio
 import concurrent.futures
 import sys
+from collections.abc import Callable
 from functools import wraps
+from typing import Any
 
 import typer
 
 
 class AsyncTyper(typer.Typer):
-    def async_command(self, *args, **kwargs):
-        def decorator(async_func):
+    def async_command(self, *args, **kwargs) -> Callable[[Callable], Callable]:
+        def decorator(async_func: Callable) -> Callable:
             @wraps(async_func)
-            def sync_func(*_args, **_kwargs):
+            def sync_func(*_args, **_kwargs) -> Any:
                 res = asyncio.run(async_func(*_args, **_kwargs))
                 return res
 
@@ -20,14 +22,14 @@ class AsyncTyper(typer.Typer):
         return decorator
 
 
-def eprint(*args, **kwargs):
+def eprint(*args, **kwargs) -> None:
     print(*args, file=sys.stderr, **kwargs)
 
 
 def _to_concurrent(
     fut: asyncio.Future, loop: asyncio.AbstractEventLoop
 ) -> concurrent.futures.Future:
-    async def wait():
+    async def wait() -> None:
         await fut
 
     return asyncio.run_coroutine_threadsafe(wait(), loop)

datashare_python-0.2.10/datashare_python/cli/worker.py

@@ -0,0 +1,114 @@
+import logging
+from typing import Annotated
+
+import typer
+
+from datashare_python.constants import DEFAULT_NAMESPACE, DEFAULT_TEMPORAL_ADDRESS
+from datashare_python.discovery import discover_activities, discover_workflows
+from datashare_python.types_ import TemporalClient
+from datashare_python.worker import datashare_worker
+
+from .utils import AsyncTyper
+
+_START_WORKER_HELP = "start a datashare worker"
+
+_LIST_WORKFLOWS_HELP = "list registered workflows"
+_LIST_WORKFLOW_NAMES_HELP = "workflow names filters (supports regexes)"
+
+_LIST_ACTIVITIES_HELP = "list registered activities"
+_LIST_ACTIVITY_NAMES_HELP = "activity names filters (supports regexes)"
+
+_START_WORKER_WORKFLOWS_HELP = "workflow names run by the worker (supports regexes)"
+_START_WORKER_ACTIVITIES_HELP = "activity names run by the worker (supports regexes)"
+_WORKER_QUEUE_HELP = "worker task queue"
+_WORKER_MAX_ACTIVITIES_HELP = (
+    "maximum number of concurrent activities/tasks"
+    " concurrently run by the worker. Defaults to 1 to encourage horizontal scaling."
+)
+_TEMPORAL_NAMESPACE_HELP = "worker temporal namespace"
+
+_TEMPORAL_URL_HELP = "address for temporal server"
+_NAMESPACE_HELP = "namespace name"
+_WORKER = "worker"
+
+worker_app = AsyncTyper(name=_WORKER)
+
+logger = logging.getLogger(__name__)
+
+
+@worker_app.async_command(help=_LIST_WORKFLOWS_HELP)
+async def list_workflows(
+    names: Annotated[list[str], typer.Argument(help=_LIST_WORKFLOW_NAMES_HELP)],
+) -> None:
+    workflows = [wf_name for wf_name, _ in discover_workflows(names)]
+    if not workflows:
+        out = """Couldn't find any registered workflow 🤔.
+Make sure your workflow plugins correctly expose workflow entry points, refer to the \
+documentation to learn how to do so."""
+        print(out)
+        return
+    workflows = "\n".join(f"- {wf}" for wf in workflows)
+    out = f"Found {len(workflows)} registered workflows:\n{workflows}"
+    print(out)
+
+
+@worker_app.async_command(help=_LIST_ACTIVITIES_HELP)
+async def list_activities(
+    names: Annotated[list[str], typer.Argument(help=_LIST_ACTIVITY_NAMES_HELP)],
+) -> None:
+    activities = [act_name for act_name, _ in discover_activities(names)]
+    if not activities:
+        out = """Couldn't find any registered activity 🤔.
+    Make sure your activity plugins correctly expose activity entry points, refer \
+to the documentation to learn how to do so."""
+        print(out)
+        return
+    activities = "\n".join(f"- {act}" for act in activities)
+    out = f"Found {len(activities)} registered activities:\n{activities}"
+    print(out)
+
+
+@worker_app.async_command(help=_START_WORKER_HELP)
+async def start(
+    workflows: Annotated[list[str], typer.Option(help=_START_WORKER_WORKFLOWS_HELP)],
+    activities: Annotated[list[str], typer.Option(help=_START_WORKER_ACTIVITIES_HELP)],
+    queue: Annotated[str, typer.Option("--queue", "-q", help=_WORKER_QUEUE_HELP)],
+    temporal_address: Annotated[
+        str, typer.Option("--temporal-address", "-a", help=_TEMPORAL_URL_HELP)
+    ] = DEFAULT_TEMPORAL_ADDRESS,
+    namespace: Annotated[
+        str, typer.Option("--temporal-namespace", "-ns", help=_TEMPORAL_NAMESPACE_HELP)
+    ] = DEFAULT_NAMESPACE,
+    max_concurrent_activities: Annotated[
+        int, typer.Option("--max-activities", help=_WORKER_MAX_ACTIVITIES_HELP)
+    ] = 1,
+) -> None:
+    wf_names, wfs = zip(*discover_workflows(workflows), strict=False)
+    registered = ""
+    if wf_names:
+        n_wfs = len(wf_names)
+        registered += (
+            f"- {n_wfs} workflow{'s' if n_wfs > 1 else ''}: {','.join(wf_names)}"
+        )
+    act_names, acts = zip(*discover_activities(activities), strict=False)
+    if act_names:
+        if registered:
+            registered += "\n"
+        i = len(act_names)
+        registered += f"- {i} activit{'ies' if i > 1 else 'y'}: {','.join(act_names)}"
+    if not acts and not wfs:
+        raise ValueError("Couldn't find any registered activity or workflow.")
+    logger.info("Starting datashare worker running:\n%s", registered)
+    client = await TemporalClient.connect(temporal_address, namespace=namespace)
+    worker = datashare_worker(
+        client,
+        workflows=wfs,
+        activities=acts,
+        task_queue=queue,
+        max_concurrent_activities=max_concurrent_activities,
+    )
+    try:
+        await worker.run()
+    except Exception as e:  # noqa: BLE001
+        await worker.shutdown()
+        raise e

datashare_python-0.2.10/datashare_python/config.py

@@ -0,0 +1,93 @@
+from typing import ClassVar
+
+from icij_common.es import ESClient
+from icij_common.pydantic_utils import ICIJSettings
+from pydantic import Field, PrivateAttr
+from pydantic_settings import SettingsConfigDict
+from temporalio.contrib.pydantic import pydantic_data_converter
+
+import datashare_python
+
+from .objects import BaseModel
+from .task_client import DatashareTaskClient
+from .types_ import TemporalClient
+from .utils import LogWithWorkerIDMixin
+
+_ALL_LOGGERS = [datashare_python.__name__]
+
+DS_WORKER_SETTINGS_CONFIG = SettingsConfigDict(
+    env_prefix="DS_WORKER_",
+    env_nested_delimiter="__",
+    nested_model_default_partial_update=True,
+)
+
+
+class ESClientConfig(BaseModel):
+    address: str = "http://localhost:9200"
+    default_page_size: int = 1000
+    keep_alive: str = "10m"
+    max_concurrency: int = 5
+    max_retries: int = 0
+    max_retry_wait_s: int | float = 60
+    timeout_s: int | float = 60 * 5
+
+    def to_es_client(self, api_key: str | None = None) -> ESClient:
+        client = ESClient(
+            hosts=[self.address],
+            pagination=self.default_page_size,
+            max_concurrency=self.max_concurrency,
+            keep_alive=self.keep_alive,
+            timeout=self.timeout_s,
+            max_retries=self.max_retries,
+            max_retry_wait_s=self.max_retry_wait_s,
+            api_key=api_key,
+        )
+        client.transport._verified_elasticsearch = True
+        return client
+
+
+class DatashareClientConfig(BaseModel):
+    api_key: str | None = None
+    url: str = "http://datashare:8080"
+
+    def to_task_client(self) -> DatashareTaskClient:
+        return DatashareTaskClient(self.url, self.api_key)
+
+
+class TemporalClientConfig(BaseModel):
+    host: str = "temporal:7233"
+    namespace: str = "datashare-default"
+    _client: TemporalClient | None = PrivateAttr(default=None)
+
+    async def to_client(self) -> TemporalClient:
+        if self._client is None:
+            self._client = await TemporalClient.connect(
+                target_host=self.host,
+                namespace=self.namespace,
+                data_converter=pydantic_data_converter,
+            )
+        return self._client
+
+    # For the lru_cache
+    def __hash__(self) -> int:
+        return id(self)
+
+
+class WorkerConfig(ICIJSettings, LogWithWorkerIDMixin, BaseModel):
+    model_config = DS_WORKER_SETTINGS_CONFIG
+
+    loggers: ClassVar[list[str]] = Field(_ALL_LOGGERS, frozen=True)
+    log_level: str = Field(default="INFO")
+
+    datashare: DatashareClientConfig = DatashareClientConfig()
+    elasticsearch: ESClientConfig = ESClientConfig()
+    temporal: TemporalClientConfig = TemporalClientConfig()
+
+    def to_es_client(self) -> ESClient:
+        return self.elasticsearch.to_es_client(self.datashare.api_key)
+
+    def to_task_client(self) -> DatashareTaskClient:
+        return self.datashare.to_task_client()
+
+    async def to_temporal_client(self) -> TemporalClient:
+        return await self.temporal.to_client()