projspec 0.0.2__py3-none-any.whl

projspec/__init__.py

@@ -0,0 +1,7 @@
+from projspec._version import __version__  # noqa: F401
+from projspec.proj import Project, ProjectSpec
+import projspec.content
+import projspec.artifact
+from projspec.utils import get_cls
+
+__all__ = ["Project", "ProjectSpec", "get_cls"]

projspec/__main__.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""Simple example executable for this library"""
+
+import json
+
+import click
+
+import projspec.proj
+
+# TODO: allow subcommands to execute artifacts, list known types and set config
+
+
+@click.command()
+@click.option(
+    "--types",
+    default="ALL",
+    help='Type names to scan for (comma-separated list in camel or snake case); defaults to "ALL"',
+)
+@click.argument("path", default=".")
+@click.option("--walk", is_flag=True, help="To descend into all child directories")
+@click.option("--summary", is_flag=True, help="Show abbreviated output")
+@click.option(
+    "--make", help="(Re)Create the first artifact found matching this type name"
+)
+@click.option(
+    "--storage_options",
+    default="",
+    help="storage options dict for the given URL, as JSON",
+)
+def main(path, types, walk, summary, make, storage_options):
+    if types in {"ALL", ""}:
+        types = None
+    else:
+        types = types.split(",")
+    if storage_options:
+        storage_options = json.loads(storage_options)
+    else:
+        storage_options = None
+    proj = projspec.Project(
+        path, storage_options=storage_options, types=types, walk=walk
+    )
+    if make:
+        art: projspec.artifact.BaseArtifact
+        for art in proj.artifacts:
+            if art.snake_name() == projspec.utils.camel_to_snake(make):
+                print("Launching:", art)
+                art.remake()
+                return
+        print("No such artifact found")
+    elif summary:
+        print(proj.text_summary())
+    else:
+        print(proj)
+
+
+if __name__ == "__main__":
+    main()

projspec/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.0.2'
+__version_tuple__ = version_tuple = (0, 0, 2)
+
+__commit_id__ = commit_id = None

projspec/artifact/__init__.py

@@ -0,0 +1,18 @@
+"""Things that a project can do or make"""
+
+from projspec.artifact.base import BaseArtifact, FileArtifact
+from projspec.artifact.installable import CondaPackage, Wheel
+from projspec.artifact.process import Process
+from projspec.artifact.python_env import EnvPack, CondaEnv, VirtualEnv, LockFile
+
+__all__ = [
+    "BaseArtifact",
+    "FileArtifact",
+    "CondaPackage",
+    "Wheel",
+    "Process",
+    "EnvPack",
+    "CondaEnv",
+    "VirtualEnv",
+    "LockFile",
+]

projspec/artifact/base.py

@@ -0,0 +1,114 @@
+import logging
+import subprocess
+from typing import Literal
+
+import fsspec.implementations.local
+
+from projspec.proj import Project
+from projspec.utils import camel_to_snake, is_installed
+
+logger = logging.getLogger("projspec")
+registry = {}
+
+
+class BaseArtifact:
+    def __init__(self, proj: Project, cmd: list[str] | None = None):
+        self.proj = proj
+        self.cmd = cmd
+        self.proc = None
+
+    def _is_clean(self) -> bool:
+        return self.proc is None  # in general, more complex
+
+    def _is_done(self) -> bool:
+        return self.proc is not None  # in general, more complex
+
+    def _check_runner(self):
+        return self.cmd[0] in is_installed
+
+    @property
+    def state(self) -> Literal["clean", "done", "pending"]:
+        if self._is_clean():
+            return "clean"
+        elif self._is_done():
+            return "done"
+        else:
+            return "pending"
+
+    def make(self, *args, **kwargs):
+        """Create the artifact and any runtime it depends on"""
+        if not isinstance(self.proj.fs, fsspec.implementations.local.LocalFileSystem):
+            # Later, will implement download-and-make, although some tools
+            # can already do this themselves.
+            raise RuntimeError("Can't run local command on remote project")
+        logger.debug(" ".join(self.cmd))
+        # this default implementation does not store any state
+        self._make(*args, **kwargs)
+
+    def _make(self, *args, **kwargs):
+        subprocess.check_call(self.cmd, cwd=self.proj.url, **kwargs)
+
+    def remake(self):
+        """Recreate the artifact and any runtime it depends on"""
+        self.clean()
+        self.make()
+
+    def clean(self):
+        """Remove artifact"""
+        # this default implementation leaves nothing to clean
+        pass
+
+    def __repr__(self):
+        return f"{type(self).__name__}, '{' '.join(self.cmd)}', {self.state}"
+
+    def _repr2(self):
+        return f"{' '.join(self.cmd)}, {self.state}"
+
+    @classmethod
+    def __init_subclass__(cls, **kwargs):
+        sn = cls.snake_name()
+        if sn in registry:
+            raise RuntimeError()
+        registry[sn] = cls
+
+    @classmethod
+    def snake_name(cls):
+        return camel_to_snake(cls.__name__)
+
+    def to_dict(self, compact=True):
+        """Distil the instance to JSON compatible dict
+
+        compact: if True, will produce condensed output, perhaps justa  string.
+        """
+        if compact:
+            return self._repr2()
+        dic = {
+            k: v
+            for k, v in self.__dict__.items()
+            if not k.startswith("_") and k not in ("proj", "proc")
+        }
+        dic["klass"] = ["artifact", self.snake_name()]
+        dic["proc"] = None
+        return dic
+
+
+def get_cls(name: str) -> type[BaseArtifact]:
+    """Find an artifact class by snake-case name."""
+    return registry[name]
+
+
+class FileArtifact(BaseArtifact):
+    """Specialised artifacts, where the output is one or more files"""
+
+    # TODO: account for outputs to a directory/glob pattern, so we can
+    #   apply to wheel; or unknown output location, e.g., conda-build.
+
+    def __init__(self, proj: Project, fn: str, **kw):
+        self.fn = fn
+        super().__init__(proj, **kw)
+
+    def _is_done(self) -> bool:
+        return self.proj.fs.exists(self.fn)
+
+    def _is_clean(self) -> bool:
+        return not self.proj.fs.exists(self.fn)

projspec/artifact/installable.py

@@ -0,0 +1,68 @@
+import logging
+import os.path
+import subprocess
+
+from projspec.artifact import BaseArtifact
+
+logger = logging.getLogger("projspec")
+
+
+class Wheel(BaseArtifact):
+    """An installable python wheel file
+
+    Note that in general there may be a set of wheels for different platforms.
+    The actual name of the wheel file depends on platform, vcs config
+    and maybe other factors. We just check if the dist/ directory is
+    populated.
+
+    This output is intended to be _local_ - pushing to a remote location (e.g., pypi)
+    is call publishing.
+    """
+
+    def _is_done(self) -> bool:
+        return True
+
+    def _is_clean(self) -> bool:
+        files = self.proj.fs.glob(f"{self.proj.url}/dist/*.whl")
+        return len(files) == 0
+
+    def clean(self):
+        files = self.proj.fs.glob(f"{self.proj.url}/dist/*.whl")
+        self.proj.fs.rm(files)
+
+
+class CondaPackage(BaseArtifact):
+    """An installable python wheel file
+
+    Note that in general, there may be a set of wheels for different platforms.
+    The actual name of the wheel file depends on the platform, vcs config
+    and maybe other factors. We just check if the dist/ directory is
+    populated.
+
+    This output is intended to be _local_ - pushing to a remote location (e.g., pypi)
+    is call publishing.
+    """
+
+    def __init__(self, path=None, name=None, **kwargs):
+        super().__init__(**kwargs)
+        self.path: str | None = path
+        self.name = name
+
+    def _make(self, *args, **kwargs):
+        import re
+
+        logger.debug(" ".join(self.cmd))
+        out = subprocess.check_output(self.cmd).decode("utf-8")
+        if fn := re.match(r"'(.*?\.conda)'\n", out):
+            if os.path.exists(fn.group(1)):
+                self.path = fn.group(1)
+
+    def _is_done(self) -> bool:
+        return True
+
+    def _is_clean(self) -> bool:
+        return self.path is None or not self.proj.fs.glob(self.path)
+
+    def clean(self):
+        if self.path is not None:
+            self.proj.fs.rm(self.path)

projspec/artifact/linter.py

@@ -0,0 +1,15 @@
+from projspec import Project
+from projspec.artifact import BaseArtifact
+
+# ruff, isort, mypy ...
+
+
+class PreCommit(BaseArtifact):
+    """Typically used as a git hook, this lists a set of linters that a project uses."""
+
+    # recognised by the presence o .pre-commit-config.yaml, but we don't need
+    # to parse it in order to run.
+
+    def __init__(self, proj: Project, cmd=None):
+        # ignore cmd: this should always be the same
+        super().__init__(proj, cmd=["pre-commit", "run", "-a"])

projspec/artifact/process.py

@@ -0,0 +1,25 @@
+import subprocess
+
+from projspec.artifact import BaseArtifact
+
+
+class Process(BaseArtifact):
+    """A simple process where we know nothing about what it does, only if it's running.
+
+    Can include batch jobs and long-running services.
+    """
+
+    def _make(self):
+        if self.proc is None:
+            self.proc = subprocess.Popen(self.cmd, **self.kw)
+
+    def _is_done(self) -> bool:
+        return self.proc is not None and self.proc.poll() is None
+
+    def clean(
+        self,
+    ):
+        if self.proc is not None:
+            self.proc.terminate()
+            self.proc.wait()
+            self.proc = None

projspec/artifact/python_env.py

@@ -0,0 +1,50 @@
+"""Python runtimes
+
+Note that for actually running python processes. There is also an implicit
+runtime from either the env that the process is running in (i.e., the PATH),
+or sys.executable.
+"""
+
+import json
+import subprocess
+from functools import cache
+
+from projspec.artifact import FileArtifact
+
+
+class CondaEnv(FileArtifact):
+    """Path to a project conda-built env
+
+    Contains both python itself and any other binaries, as well as linked
+    libraries.
+
+    In the case of a project having an environment.yaml with a named output,
+    the path may be outside the project tree.
+    """
+
+    @staticmethod
+    @cache
+    def envs() -> list[str]:
+        """Global conda env root paths"""
+        # pixi also has global envs
+        out = subprocess.check_output(["conda", "env", "list", "--json"])
+        return json.loads(out.decode())["envs"]
+
+
+class VirtualEnv(FileArtifact):
+    """Path to a project virtual environment
+
+    Some tools like pipenv put these environments in a global location.
+    """
+
+
+class EnvPack(FileArtifact):
+    """Archival form of a python environment
+
+    - conda-pack: https://conda.github.io/conda-pack/
+    - pixi-pack: https://pixi.sh/latest/deployment/pixi_pack/
+    """
+
+
+class LockFile(FileArtifact):
+    """File containing exact environment specification"""

projspec/content/__init__.py

@@ -0,0 +1,25 @@
+from projspec.content.base import BaseContent
+from projspec.content.data import FrictionlessData, IntakeCatalog
+from projspec.content.env_var import EnvironmentVariables
+from projspec.content.environment import Environment, Stack, Precision
+from projspec.content.executable import Command
+from projspec.content.license import License
+
+# from projspec.content.linter
+from projspec.content.metadata import DescriptiveMetadata
+from projspec.content.package import PythonPackage
+
+
+__all__ = [
+    "BaseContent",
+    "FrictionlessData",
+    "IntakeCatalog",
+    "EnvironmentVariables",
+    "Command",
+    "License",
+    "DescriptiveMetadata",
+    "PythonPackage",
+    "Environment",
+    "Stack",
+    "Precision",
+]

projspec/content/base.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass, field
+
+from projspec.artifact import BaseArtifact
+from projspec.proj.base import Project
+from projspec.utils import Enum, camel_to_snake
+
+registry = {}
+
+
+@dataclass
+class BaseContent:
+    proj: Project = field(repr=False)
+    artifacts: set[BaseArtifact] = field(repr=False)
+
+    def _repr2(self):
+        return {
+            k: (v.name if isinstance(v, Enum) else v)
+            for k, v in self.__dict__.items()
+            if not k.startswith("_") and k not in ("proj", "artifacts")
+        }
+
+    @classmethod
+    def __init_subclass__(cls, **kwargs):
+        sn = cls.snake_name()
+        if sn in registry:
+            raise RuntimeError()
+        registry[sn] = cls
+
+    @classmethod
+    def snake_name(cls):
+        return camel_to_snake(cls.__name__)
+
+    def to_dict(self, compact=False):
+        if compact:
+            return self._repr2()
+        dic = {
+            k: getattr(self, k)
+            for k in self.__dataclass_fields__
+            if k not in ("proj", "artifacts")
+        }
+        dic["artifacts"] = []
+        dic["klass"] = ["content", self.snake_name()]
+        for k in list(dic):
+            if isinstance(dic[k], Enum):
+                dic[k] = dic[k].value
+        return dic

projspec/content/data.py

@@ -0,0 +1,24 @@
+"""Contents specifying datasets"""
+
+from projspec.content import BaseContent
+
+
+class FrictionlessData(BaseContent):
+    """A datapackage spec, as defined by frictionlessdata
+
+    This lists loadable tabular files with defined schema, typically from formats such as
+    JSON, CSV, and parquet.
+
+    See https://specs.frictionlessdata.io/data-package/
+    """
+
+    # typically in a datapackage.json spec
+
+
+class IntakeCatalog(BaseContent):
+    """A catalog of data assets, including basic properties (location) and how to load/process them.
+
+    See https://intake.readthedocs.io/en/latest/
+    """
+
+    # typically in a catalog.yaml free-floating file

projspec/content/env_var.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass, field
+
+from projspec.content.base import BaseContent
+
+
+@dataclass
+class EnvironmentVariables(BaseContent):
+    """A set of environment variable key/value pairs, typically used with new processes."""
+
+    variables: dict[str, str | None] = field(default_factory=dict)

projspec/content/environment.py

@@ -0,0 +1,41 @@
+from dataclasses import dataclass, field
+from enum import auto
+
+from projspec.content import BaseContent
+from projspec.utils import Enum
+
+
+class Stack(Enum):
+    """The type of environment by packaging tech"""
+
+    PIP = auto()
+    CONDA = auto()
+
+
+class Precision(Enum):
+    """Type of environment definition by the amount of precision"""
+
+    # TODO: categories may be refined, e.g., whether items include architecture or hash
+    SPEC = auto()
+    LOCK = auto()
+
+
+@dataclass
+class Environment(BaseContent):
+    """Definition of a python runtime environment"""
+
+    stack: Stack
+    precision: Precision
+    packages: list[str]
+    # This may be empty for loose specs; may include endpoints or index URLs.
+    channels: list[str] = field(default_factory=list)
+
+    def _repr2(self):
+        out = {
+            k: (v.name if isinstance(v, Enum) else v)
+            for k, v in self.__dict__.items()
+            if not k.startswith("_") and k not in ("proj", "artifacts")
+        }
+        if not self.channels:
+            out.pop("channels", None)
+        return out

projspec/content/executable.py

@@ -0,0 +1,15 @@
+"""Executable contents produce artifacts"""
+
+from dataclasses import dataclass
+
+from projspec.content import BaseContent
+
+
+@dataclass
+class Command(BaseContent):
+    """The simplest runnable thing; we don't know what it does/outputs."""
+
+    cmd: list[str] | str
+
+    def _repr2(self):
+        return " ".join(self.cmd) if isinstance(self.cmd, list) else self.cmd

projspec/content/license.py

@@ -0,0 +1,16 @@
+from projspec.content import BaseContent
+
+
+class License(BaseContent):
+    """A legal description of what the given project (code and other assets) can be used for.
+
+    This could be one of the typical open-source permissive licenses (see https://spdx.org/licenses/),
+    specified either just by its name or by a link. Some projects will have custom or restrictive
+    conditions on their replication and use.
+    """
+
+    # https://opensource.org/licenses
+
+    shortname: str  # aka SPDX
+    fullname: str
+    url: str  # relative in the project or remote HTTP

projspec/content/metadata.py

@@ -0,0 +1,9 @@
+from dataclasses import field
+
+from projspec.content import BaseContent
+
+
+class DescriptiveMetadata(BaseContent):
+    """Miscellaneous descriptive information"""
+
+    meta: dict[str, str] = field(default_factory=dict)

projspec/content/package.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+from projspec.content import BaseContent
+
+
+@dataclass
+class PythonPackage(BaseContent):
+    """Importable python directory, i.e., containing an __init__.py file."""
+
+    package_name: str

projspec/html.py

@@ -0,0 +1,45 @@
+def dict_to_html(data: dict, title="Data", open_level=2) -> str:
+    """
+    Convert a nested dictionary to expandable HTML using <details> tags.
+
+    Args:
+        data: The dictionary to convert
+        title: Title for the details element
+        open_level: whether to set elements as expanded; yes if > 0, and will
+            decrement for inner levels.
+
+    Returns:
+        String containing HTML with expandable details elements
+    """
+    # With help from Claude Sonnet 4.
+    if not isinstance(data, dict):
+        return f"<span>{data}</span>"
+
+    if not data:
+        return ""
+    open = "open" if open_level > 0 else "closed"
+
+    html = [
+        f'<details {open} style="margin-left: 20px; margin-bottom: 10px;"><summary style="cursor: pointer; color: #2c5aa0; padding: 5px;"><strong>{title}</strong></summary>'
+    ]
+
+    for key, value in data.items():
+        if isinstance(value, dict):
+            html.append(dict_to_html(value, key, open_level - 1))
+        elif isinstance(value, (list, tuple)):
+            html.append(
+                f'<details style="margin-left: 20px; margin-bottom: 10px;"><summary style="cursor: pointer; color: #2c5aa0; padding: 5px;"><strong>{key}</strong></summary>'
+            )
+            for i, item in enumerate(value):
+                if isinstance(item, dict):
+                    html.append(dict_to_html(item, f"{key}[{i}]", open_level - 1))
+                else:
+                    html.append(f'<div style=" margin: 5px 0;"> {item}</div>')
+            html.append("</details>")
+        else:
+            html.append(
+                f'<div style=" margin: 5px 0;"><strong>{key}:</strong> {value}</div>'
+            )
+
+    html.append("</details>")
+    return "".join(html)

projspec/proj/__init__.py

@@ -0,0 +1,31 @@
+from projspec.proj.base import ParseFailed, Project, ProjectSpec
+from projspec.proj.conda_package import CondaRecipe, RattlerRecipe
+from projspec.proj.conda_project import CondaProject
+from projspec.proj.documentation import RTD, MDBook
+from projspec.proj.git import GitRepo
+from projspec.proj.pixi import Pixi
+from projspec.proj.poetry import Poetry
+from projspec.proj.pyscript import PyScript
+from projspec.proj.python_code import PythonCode, PythonLibrary
+from projspec.proj.rust import Rust, RustPython
+from projspec.proj.uv import Uv
+
+__all__ = [
+    "ParseFailed",
+    "Project",
+    "ProjectSpec",
+    "CondaRecipe",
+    "CondaProject",
+    "GitRepo",
+    "MDBook",
+    "Poetry",
+    "RattlerRecipe",
+    "Pixi",
+    "PyScript",
+    "PythonCode",
+    "PythonLibrary",
+    "RTD",
+    "Rust",
+    "RustPython",
+    "Uv",
+]