cmem-plugin-git 1.0.0__py3-none-any.whl

cmem_plugin_git/__init__.py

@@ -0,0 +1 @@
+"""cmem-plugin-git"""

cmem_plugin_git/autocompletion.py

@@ -0,0 +1,86 @@
+"""Parameter types that ask the repository what it holds"""
+
+from typing import Any, ClassVar
+
+from cmem_plugin_base.dataintegration.context import PluginContext
+from cmem_plugin_base.dataintegration.types import Autocompletion, StringParameterType
+
+from cmem_plugin_git.remote import GitRemote
+
+ROOT_LABEL = "/ (top of the repository)"
+
+
+def _matching(completions: list[Autocompletion], query_terms: list[str]) -> list[Autocompletion]:
+    """Keep the completions that match every term the user typed."""
+    if not query_terms:
+        return completions
+    return [
+        completion
+        for completion in completions
+        if all(
+            term.lower() in (completion.label or completion.value).lower() for term in query_terms
+        )
+    ]
+
+
+def _remote(values: list[Any]) -> GitRemote | None:
+    """Build a remote from the parameter values an autocompletion depends on."""
+    url, username, token = values[0], values[1], values[2]
+    if not str(url).strip():
+        return None
+    password = token if isinstance(token, str) else token.decrypt()
+    return GitRemote(url=str(url), token=str(password), username=str(username))
+
+
+class RefParameterType(StringParameterType):
+    """Autocomplete the branches and tags a repository advertises"""
+
+    allow_only_autocompleted_values: bool = False
+    autocomplete_value_with_labels: bool = True
+    autocompletion_depends_on_parameters: ClassVar[list[str]] = ["url", "username", "token"]
+
+    def autocomplete(
+        self,
+        query_terms: list[str],
+        depend_on_parameter_values: list[Any],
+        context: PluginContext,
+    ) -> list[Autocompletion]:
+        """Return the branches and tags matching all query terms."""
+        _ = context
+        remote = _remote(depend_on_parameter_values)
+        if remote is None:
+            return []
+        completions = [
+            Autocompletion(value=ref.name, label=f"{ref.name} ({ref.kind})")
+            for ref in remote.list_refs()
+        ]
+        return _matching(completions, query_terms)
+
+
+class FolderParameterType(StringParameterType):
+    """Autocomplete the folders a repository holds at the selected revision"""
+
+    allow_only_autocompleted_values: bool = False
+    autocomplete_value_with_labels: bool = True
+    autocompletion_depends_on_parameters: ClassVar[list[str]] = [
+        "url",
+        "username",
+        "token",
+        "ref",
+    ]
+
+    def autocomplete(
+        self,
+        query_terms: list[str],
+        depend_on_parameter_values: list[Any],
+        context: PluginContext,
+    ) -> list[Autocompletion]:
+        """Return the folders matching all query terms."""
+        _ = context
+        remote = _remote(depend_on_parameter_values)
+        if remote is None:
+            return []
+        snapshot = remote.snapshot(str(depend_on_parameter_values[3]))
+        completions = [Autocompletion(value="", label=ROOT_LABEL)]
+        completions += [Autocompletion(value=folder, label=folder) for folder in snapshot.folders()]
+        return _matching(completions, query_terms)

cmem_plugin_git/download.py

@@ -0,0 +1,272 @@
+"""Download files from a git repository"""
+
+import tempfile
+from collections.abc import Callable, Iterator, Sequence
+from contextlib import suppress
+from pathlib import Path
+
+from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport
+from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginAction
+from cmem_plugin_base.dataintegration.entity import Entities, Entity
+from cmem_plugin_base.dataintegration.parameter.password import Password
+from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin
+from cmem_plugin_base.dataintegration.ports import FixedNumberOfInputs, FixedSchemaPort
+from cmem_plugin_base.dataintegration.typed_entities.file import FileEntitySchema, LocalFile
+
+from cmem_plugin_git.parameters import (
+    URL_DESCRIPTION,
+    connection_parameters,
+    list_schema,
+    preview_of,
+    remote_of,
+    selection_parameters,
+)
+from cmem_plugin_git.remote import (
+    DEFAULT_USERNAME,
+    LFS_POINTER_PREFIX,
+    RepositoryFile,
+    Selection,
+    Snapshot,
+    iter_warnings,
+)
+
+ACCESS_DESCRIPTION = (
+    "An access token that may read the repository. Leave it empty to read a public repository."
+)
+BATCH_SIZE = 50
+"""How many files' content is asked for in one request.
+
+A server that hands out single objects answers one request per batch rather than
+one per file, and a batch bounds how much content is held in memory at once.
+"""
+
+URL_DESCRIPTION_WITH_INPUT = (
+    f"{URL_DESCRIPTION} Leave it empty to take the repository, and the files to read from it, "
+    "from the connected input instead - which is how **List Git files** drives this task."
+)
+SELECTION_NOTE = (
+    "Ignored when the Repository URL is empty and the input drives file selection instead."
+)
+REF_LABEL = "Revision"
+REF_DESCRIPTION = (
+    "The branch, tag or commit id to read. Leave it empty to read the repository's default "
+    f"branch. {SELECTION_NOTE}"
+)
+
+
+@Plugin(
+    label="Download Git files",
+    plugin_id="cmem_plugin_git-Download",
+    description="Download files from a git repository and hand them to the next task.",
+    documentation="""
+This workflow task reads files from a git repository at one revision and hands them
+to the next task in the workflow. It works on the objects of the repository rather
+than on a checkout, so only the files it selects are transferred.
+
+Files leave the task as file entities that any file consuming task can read. They are
+written into a temporary folder of the container the task runs in, and they are not
+deleted afterwards, because the next task reads them only when it gets to them.
+
+This task works in one of two mutually exclusive ways, decided by whether a
+Repository URL is configured. Set it, and the task fetches on its own, guided by
+the folder, expression and subfolder settings, with the input closed. Leave it
+empty, and the input opens instead: it supplies both the repository and the
+files to read from it, and the folder and expression settings go unused. This is
+how **List Git files** drives this task after the workflow has filtered its
+output.
+
+The repository is reached over HTTP(S), which is the only transport these tasks speak;
+an SSH address is refused. Reading needs a token with read access to the repository and
+no more than that: the `read_repository` scope on GitLab, or a fine grained GitHub token
+granting `Contents: Read-only`. A public repository needs no token at all. The user name sent
+alongside the token matters on GitLab and not on GitHub, which is why its default is
+the value GitLab accepts.
+
+Submodules and files stored with Git LFS are reported as skipped rather than
+downloaded, because the repository holds no content for them, only a reference. A
+symbolic link is delivered under its own name with the content of the file it points
+at; one that points outside the repository is skipped. Reading a commit id rather
+than a branch or a tag works only where the server allows fetching an object it does
+not advertise.
+""",
+    icon=Icon(package=__package__, file_name="git-download.svg"),
+    actions=[
+        PluginAction(
+            name="preview",
+            label="Preview results (max. 10)",
+            description="List the first files this configuration selects.",
+        )
+    ],
+    parameters=[
+        *connection_parameters(
+            REF_LABEL,
+            REF_DESCRIPTION,
+            ACCESS_DESCRIPTION,
+            url_description=URL_DESCRIPTION_WITH_INPUT,
+            url_optional=True,
+        ),
+        *selection_parameters(note=SELECTION_NOTE),
+    ],
+)
+class DownloadGitFiles(WorkflowPlugin):
+    """Git Workflow Plugin: download files"""
+
+    # A plugin constructor takes one argument per PluginParameter, so its arity is
+    # fixed by the plugin's configuration surface, not by a style choice here.
+    def __init__(  # noqa: PLR0913, PLR0917
+        self,
+        url: str = "",
+        token: str | Password = "",
+        ref: str = "",
+        username: str = DEFAULT_USERNAME,
+        path: str = "",
+        regex: str = "",
+        no_subfolder: bool = False,
+    ) -> None:
+        self.url = url
+        self.token = token
+        self.ref = ref
+        self.username = username
+        self.path = path
+        self.regex = regex
+        self.no_subfolder = no_subfolder
+        # A Repository URL configured here means this task fetches on its own, so the
+        # input is closed; leaving it empty means the repository and the files both
+        # come from the input, so the input is required instead.
+        self.input_ports = (
+            FixedNumberOfInputs([])
+            if self.url.strip()
+            else FixedNumberOfInputs([FixedSchemaPort(schema=list_schema())])
+        )
+        self.output_port = FixedSchemaPort(schema=FileEntitySchema())
+
+    def _snapshot(self) -> Snapshot:
+        """Fetch the revision's directory listing."""
+        return remote_of(self.url, self.token, self.username).snapshot(self.ref)
+
+    def _selection(self, snapshot: Snapshot) -> Selection:
+        """Decide which files to read, from the folder, expression and subfolder parameters."""
+        return snapshot.select(path=self.path, regex=self.regex, no_subfolder=self.no_subfolder)
+
+    def _from_input(
+        self, inputs: Sequence[Entities]
+    ) -> tuple[Selection, Callable[[list[str]], dict[str, bytes]]]:
+        """Build the selection and a blob reader from what a connected task sent
+
+        Used when this task's own Repository URL is left empty: the files to fetch,
+        and the repository to fetch them from, both come from the input instead of
+        from this task's own parameters. Fetching each file by the blob id it
+        already carries, rather than re-resolving a revision, means there is no
+        window in which the branch could have moved between the two tasks.
+        """
+        if not inputs:
+            raise ValueError(
+                "No Repository URL is configured and nothing is connected to the input, "
+                "so there is nothing to read."
+            )
+        entities = list(inputs[0].entities)
+        if not entities:
+            return Selection(), lambda _blob_ids: {}
+        url = str(entities[0].values[-1][0])
+        remote = remote_of(url, self.token, self.username)
+        files = [_file_from_entity(entity) for entity in entities]
+        return Selection(files=files), remote.fetch_blobs
+
+    def preview(self) -> str:
+        """Show what this configuration selects."""
+        if not self.url.strip():
+            return (
+                "This configuration takes its repository and file selection from the "
+                "connected input, so it cannot be previewed before the workflow runs."
+            )
+        selection = self._selection(self._snapshot())
+        return preview_of([file.path for file in selection.files], selection.warnings)
+
+    def execute(self, inputs: Sequence[Entities], context: ExecutionContext) -> Entities:
+        """Run the workflow task."""
+        read: Callable[[list[str]], dict[str, bytes]]
+        if self.url.strip():
+            snapshot = self._snapshot()
+            selection = self._selection(snapshot)
+            read = snapshot.read
+            self.log.info(f"Downloading {len(selection.files)} file(s) from {self.url}")
+        else:
+            selection, read = self._from_input(inputs)
+            self.log.info(f"Downloading {len(selection.files)} file(s) from the connected input")
+        directory = Path(tempfile.mkdtemp(prefix="cmem-plugin-git-"))
+        files = []
+        cancelled = False
+        for batch in _batched(selection.files, BATCH_SIZE):
+            if cancelled:
+                break
+            contents = read([file.blob_id for file in batch])
+            for file in batch:
+                with suppress(AttributeError):
+                    if context.workflow.status() == "Canceling":  # type: ignore[union-attr]
+                        cancelled = True
+                        break
+                content = contents[file.blob_id]
+                if content.startswith(LFS_POINTER_PREFIX):
+                    selection.warnings.append(
+                        f"'{file.path}' holds a Git LFS pointer instead of content and was skipped."
+                    )
+                    continue
+                files.append(_write(directory, file, content))
+                context.report.update(
+                    ExecutionReport(
+                        entity_count=len(files),
+                        operation="read",
+                        operation_desc="files downloaded",
+                    )
+                )
+        if cancelled:
+            selection.warnings.append(
+                "The workflow was cancelled, so only part of the selection was downloaded."
+            )
+        schema = FileEntitySchema()
+        entities = [schema.to_entity(file) for file in files]
+        context.report.update(
+            ExecutionReport(
+                entity_count=len(entities),
+                operation="read",
+                operation_desc="files downloaded",
+                warnings=list(iter_warnings(selection.warnings)),
+                sample_entities=Entities(entities=iter(entities[:10]), schema=schema),
+            )
+        )
+        return Entities(entities=iter(entities), schema=schema)
+
+
+def _file_from_entity(entity: Entity) -> RepositoryFile:
+    """Rebuild the file a list task described, from its entity
+
+    The inverse of list.py's ``_entity()``: values are read positionally, in the
+    order ``list_schema()`` declares them. The URL carried alongside is read by
+    the caller instead, since it names the repository rather than one file of it.
+    """
+    path, name, size, mode, blob_id, commit_id, _url = entity.values
+    return RepositoryFile(
+        path=path[0],
+        name=name[0],
+        size=int(size[0]) if size else None,
+        mode=mode[0],
+        blob_id=blob_id[0],
+        commit_id=commit_id[0],
+    )
+
+
+def _batched(files: list[RepositoryFile], size: int) -> Iterator[list[RepositoryFile]]:
+    """Cut the selection into groups that are fetched together."""
+    for start in range(0, len(files), size):
+        yield files[start : start + size]
+
+
+def _write(directory: Path, file: RepositoryFile, content: bytes) -> LocalFile:
+    """Write one downloaded file into the task's temporary folder."""
+    target = directory / file.name
+    counter = 0
+    while target.exists():
+        counter += 1
+        target = directory / f"{target.stem}_{counter}{target.suffix}"
+    target.write_bytes(content)
+    return LocalFile(str(target))

cmem_plugin_git/git-download.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg focusable="false" preserveAspectRatio="xMidYMid meet" xmlns="http://www.w3.org/2000/svg" fill="currentColor" width="32" height="32" viewBox="0 0 32 32" aria-hidden="true" class="eccgui-icon"><circle cx="8" cy="6" r="3"></circle><circle cx="8" cy="26" r="3"></circle><rect x="7" y="6" width="2" height="20"></rect><rect x="8" y="15" width="7" height="2"></rect><circle cx="17" cy="16" r="3"></circle><rect x="26" y="6" width="2" height="13"></rect><path d="M22 17h10l-5 8z"></path></svg>

cmem_plugin_git/git-list.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg focusable="false" preserveAspectRatio="xMidYMid meet" xmlns="http://www.w3.org/2000/svg" fill="currentColor" width="32" height="32" viewBox="0 0 32 32" aria-hidden="true" class="eccgui-icon"><circle cx="8" cy="6" r="3"></circle><circle cx="8" cy="26" r="3"></circle><rect x="7" y="6" width="2" height="20"></rect><rect x="8" y="15" width="7" height="2"></rect><circle cx="17" cy="16" r="3"></circle><rect x="23" y="6" width="8" height="2"></rect><rect x="23" y="15" width="8" height="2"></rect><rect x="23" y="24" width="8" height="2"></rect></svg>

cmem_plugin_git/git-upload.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg focusable="false" preserveAspectRatio="xMidYMid meet" xmlns="http://www.w3.org/2000/svg" fill="currentColor" width="32" height="32" viewBox="0 0 32 32" aria-hidden="true" class="eccgui-icon"><circle cx="8" cy="6" r="3"></circle><circle cx="8" cy="26" r="3"></circle><rect x="7" y="6" width="2" height="20"></rect><rect x="8" y="15" width="7" height="2"></rect><circle cx="17" cy="16" r="3"></circle><rect x="26" y="13" width="2" height="13"></rect><path d="M27 6l5 8H22z"></path></svg>

cmem_plugin_git/list.py

@@ -0,0 +1,161 @@
+"""List the files of a git repository"""
+
+from collections.abc import Sequence
+
+from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport
+from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginAction
+from cmem_plugin_base.dataintegration.entity import Entities, Entity
+from cmem_plugin_base.dataintegration.parameter.password import Password
+from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin
+from cmem_plugin_base.dataintegration.ports import (
+    FixedNumberOfInputs,
+    FixedSchemaPort,
+)
+
+from cmem_plugin_git.parameters import (
+    connection_parameters,
+    list_schema,
+    preview_of,
+    remote_of,
+    selection_parameters,
+)
+from cmem_plugin_git.remote import DEFAULT_USERNAME, RepositoryFile, Selection, iter_warnings
+
+ACCESS_DESCRIPTION = (
+    "An access token that may read the repository. Leave it empty to read a public repository."
+)
+REF_LABEL = "Revision"
+REF_DESCRIPTION = (
+    "The branch, tag or commit id to read. Leave it empty to read the repository's default branch."
+)
+
+
+@Plugin(
+    label="List Git files",
+    plugin_id="cmem_plugin_git-List",
+    description="List the files of a git repository, without transferring their content.",
+    documentation="""
+This workflow task lists the files a git repository holds at one revision. It reads
+the repository over HTTP(S) and transfers the directory listing only, never the
+content of a file.
+
+Each file leaves the task as one entity with its repository path, its name, its size
+in bytes, its git file mode, the id of its content, the id of the commit that was
+read, and this task's own Repository URL. The task takes no input.
+
+The size is the one value that may be missing. A git tree records a name, a mode and
+an object id, but not a length, so a file's size is only known once its content has
+been transferred. Against a server that supports partial fetching - GitHub and GitLab
+both do - this task deliberately does not transfer content, and the size stays empty;
+against a server that does not, the content arrives anyway and the size is filled in.
+
+It is usually the first task of a chain: filter its output in the workflow, then
+connect it to **Download Git files**, which then fetches exactly the files that
+survived the filtering. Leave Download's own Repository URL empty for this -
+it then reads the URL from this task's output as well, rather than needing it
+configured a second time.
+
+The repository is reached over HTTP(S), which is the only transport these tasks speak;
+an SSH address is refused. Reading needs a token with read access to the repository and
+no more than that: the `read_repository` scope on GitLab, or a fine grained GitHub token
+granting `Contents: Read-only`. A public repository needs no token at all. The user name sent
+alongside the token matters on GitLab and not on GitHub, which is why its default is
+the value GitLab accepts.
+
+Submodules and files stored with Git LFS are reported as skipped rather than listed,
+because neither holds content the repository can hand out. A symbolic link is listed
+under its own name, carrying the size and the content id of the file it points at;
+one that points outside the repository is skipped. Reading a commit id rather than a
+branch or a tag works only where the server allows fetching an object it does not
+advertise.
+
+The task reads one revision and nothing around it. It answers no question about
+history: when a file last changed, who changed it, or how two revisions differ are
+all outside what these tasks do, because none of it can be had without fetching the
+history the design deliberately leaves on the server.
+""",
+    icon=Icon(package=__package__, file_name="git-list.svg"),
+    actions=[
+        PluginAction(
+            name="preview",
+            label="Preview results (max. 10)",
+            description="List the first files this configuration selects.",
+        )
+    ],
+    parameters=[
+        *connection_parameters(REF_LABEL, REF_DESCRIPTION, ACCESS_DESCRIPTION),
+        *selection_parameters(),
+    ],
+)
+class ListGitFiles(WorkflowPlugin):
+    """Git Workflow Plugin: list files"""
+
+    # A plugin constructor takes one argument per PluginParameter, so its arity is
+    # fixed by the plugin's configuration surface, not by a style choice here.
+    def __init__(  # noqa: PLR0913, PLR0917
+        self,
+        url: str,
+        token: str | Password = "",
+        ref: str = "",
+        username: str = DEFAULT_USERNAME,
+        path: str = "",
+        regex: str = "",
+        no_subfolder: bool = False,
+    ) -> None:
+        self.url = url
+        self.token = token
+        self.ref = ref
+        self.username = username
+        self.path = path
+        self.regex = regex
+        self.no_subfolder = no_subfolder
+        self.input_ports = FixedNumberOfInputs([])
+        self.output_port = FixedSchemaPort(schema=list_schema())
+
+    def _select(self) -> Selection:
+        """Fetch the revision's directory listing and apply the selection."""
+        remote = remote_of(self.url, self.token, self.username)
+        snapshot = remote.snapshot(self.ref)
+        return snapshot.select(path=self.path, regex=self.regex, no_subfolder=self.no_subfolder)
+
+    def preview(self) -> str:
+        """Show what this configuration selects."""
+        selection = self._select()
+        return preview_of([file.path for file in selection.files], selection.warnings)
+
+    def execute(self, inputs: Sequence[Entities], context: ExecutionContext) -> Entities:
+        """Run the workflow task."""
+        _ = inputs
+        self.log.info(f"Listing files of {self.url}")
+        selection = self._select()
+        entities = [_entity(file, self.url) for file in selection.files]
+        context.report.update(
+            ExecutionReport(
+                entity_count=len(entities),
+                operation="read",
+                operation_desc="files listed",
+                warnings=list(iter_warnings(selection.warnings)),
+            )
+        )
+        return Entities(entities=iter(entities), schema=list_schema())
+
+
+def _entity(file: RepositoryFile, url: str) -> Entity:
+    """Turn one listed file into an entity.
+
+    ``url`` is this task's own repository URL, carried along so that
+    **Download Git files** can read the repository it is being asked to fetch
+    from without needing the same URL configured a second time.
+    """
+    return Entity(
+        uri=f"urn:git:{file.commit_id}:{file.path}",
+        values=[
+            [file.path],
+            [file.name],
+            [] if file.size is None else [str(file.size)],
+            [file.mode],
+            [file.blob_id],
+            [file.commit_id],
+            [url],
+        ],
+    )