moat-src 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

moat/src/_main.py

@@ -1,11 +1,12 @@
 # command line interface
 # pylint: disable=missing-module-docstring
+from __future__ import annotations
 
 import io
 import logging
 import subprocess
 import sys
-from collections import defaultdict
+from collections import defaultdict, deque
 from configparser import RawConfigParser
 from pathlib import Path
 
@@ -15,89 +16,322 @@ import tomlkit
 from anyio import run_process
 from moat.util import P, add_repr, attrdict, make_proc, yload, yprint
 from packaging.requirements import Requirement
+from attrs import define,field
+from shutil import rmtree,copyfile,copytree
+from contextlib import suppress
 
 logger = logging.getLogger(__name__)
 
+PACK=Path("packaging")
+ARCH=subprocess.check_output(["dpkg","--print-architecture"]).decode("utf-8").strip()
 
-class Repo(git.Repo):
-    """Amend git.Repo with submodule and tag caching"""
+def dash(n:str) -> str:
+    """
+    moat.foo.bar > foo-bar
+    foo.bar > ext-foo-bar
+    """
+    if n in ("main","moat"):
+        return "main"
+    if "." not in n:  # also applies to single-name packages
+        return n
+
+    if not n.startswith("moat."):
+        return "ext-"+n.replace("-",".")
+    return n.replace(".","-")[5:]
+
+def undash(n:str) -> str:
+    """
+    foo-bar > moat.foo.bar
+    ext-foo-bar > foo.bar
+    """
+    if "." in n:
+        return n
+
+    if n in ("main","moat"):
+        return "moat"
+    if n.startswith("ext-"):
+        return n.replace("-",".")[4:]
+    return "moat."+n.replace("-",".")
+
+class ChangedError(RuntimeError):
+    def __init__(subsys,tag,head):
+        self.subsys = subsys
+        self.tag = tag
+        self.head = head
+    def __str__(self):
+        s = self.subsys or "Something"
+        if head is None:
+            head="HEAD"
+        else:
+            head = head.hexsha[:9]
+        return f"{s} changed between {tag.name} and {head}"
+
+class _Common:
+    _last_tag:str = None
+
+    def next_tag(self,major:bool=False,minor:bool=False):
+        tag = self.last_tag()[0]
+        try:
+            n = [ int(x) for x in tag.split('.') ]
+            if len(n) != 3:
+                raise ValueError(n)
+        except ValueError:
+            raise ValueError(f"Tag {tag} not in major#.minor#.patch# form.") from None
+
+        if major:
+            n = [n[0]+1,0,0]
+        elif minor:
+            n = [n[0],n[1]+1,0]
+        else:
+            n = [n[0],n[1],n[2]+1]
+        return ".".join(str(x) for x in n)
+
+@define
+class Package(_Common):
+    _repo:Repo = field(repr=False)
+    name:str = field()
+    under:str = field(init=False,repr=False)
+    path:Path = field(init=False,repr=False)
+    files:set(Path) = field(init=False,factory=set,repr=False)
+    subs:dict[str,Package] = field(factory=dict,init=False,repr=False)
+    hidden:bool = field(init=False,repr=False)
+
+    def __init__(self, repo, name):
+        self.__attrs_init__(repo,name)
+        self.under = name.replace(".","_")
+        self.path = Path(*name.split("."))
+        self.hidden = not (PACK/self.dash).exists()
+
+    @property
+    def dash(self):
+        return dash(self.name)
+
+    def __eq__(self, other):
+        return self.name==other.name
+
+    def __hash__(self):
+        return hash(self.name)
+
+    @property
+    def mdash(self):
+        d=dash(self.name)
+        if d.startswith("ext-"):
+            return d[4:]
+        else:
+            return "moat-"+d
+
+    def populate(self, path:Path, real=None):
+        """
+        Collect this package's file names.
+        """
+        self.path = path
+        for fn in path.iterdir():
+            if fn.name == "__pycache__":
+                continue
+            if (sb := self.subs.get(fn.name,None)) is not None:
+                sb.populate(fn, real=self if sb.hidden else None)
+            else:
+                (real or self).files.add(fn)
+
+    def copy(self) -> None:
+        """
+        Copies the current version of this subsystem to its packaging area.
+        """
+        if not self.files:
+            raise ValueError(f"No files in {self.name}?")
+        p = Path("packaging")/self.dash
+        with suppress(FileNotFoundError):
+            rmtree(p/"moat")
+        dest = p/self.path
+        dest.mkdir(parents=True)
+        for f in self.files:
+            pf=p/f
+            pf.parent.mkdir(parents=True,exist_ok=True)
+            if f.is_dir():
+                copytree(f, pf, symlinks=False)
+            else:
+                copyfile(f, pf, follow_symlinks=True)
+        licd = p/"LICENSE.txt"
+        if not licd.exists():
+            copyfile("LICENSE.txt", licd)
+
+    def last_tag(self, unchanged:bool=False) -> Tag|None:
+        """
+        Return the most-recent tag for this subrepo
+        """
+        try:
+            tag,commit = self._repo.versions[self.dash]
+        except KeyError:
+            raise KeyError(f"No version for {self.dash} found") from None
+        if unchanged and self.has_changes(commit):
+            raise ChangedError(subsys,t,ref)
+        return tag,commit
+
+    def has_changes(self, tag:Commit=None) -> bool:
+        """
+        Test whether the given subsystem (or any subsystem)
+        changed between the head and the @tag commit
+        """
+        if tag is None:
+            tag,commit = self.last_tag()
+        else:
+            commit = tag
+        head = self._repo.head.commit
+        for d in head.diff(commit):
+            if self._repo.repo_for(d.a_path) != self.name and self._repo.repo_for(d.b_path) != self.name:
+                continue
+            return True
+        return False
+
+
+class Repo(git.Repo,_Common):
+    """Amend git.Repo with tag caching and pseudo-submodule splitting"""
 
     moat_tag = None
-    submod = None
+    _last_tag=None
 
     def __init__(self, *a, **k):
         super().__init__(*a, **k)
-        self._subrepo_cache = {}
         self._commit_tags = defaultdict(list)
         self._commit_topo = {}
 
+        self._repos = {}
+        self._make_repos()
+
         for t in self.tags:
             self._commit_tags[t.commit].append(t)
 
         p = Path(self.working_dir)
         mi = p.parts.index("moat")
         self.moat_name = "-".join(p.parts[mi:])
+        with open("versions.yaml") as f:
+            self.versions = yload(f)
 
-    def subrepos(self, recurse=True, depth=True, same=True):
-        """List subrepositories (and cache them)."""
+    def write_tags(self):
+        with open("versions.yaml","w") as f:
+            yprint(self.versions,f)
+        self.index.add("versions.yaml")
 
-        if same and recurse and not depth:
-            yield self
 
-        if "/lib/" not in self.working_dir:
-            for r in self.submodules:
-                try:
-                    res = self._subrepo_cache[r.path]
-                except KeyError:
-                    try:
-                        p = Path(self.working_dir) / r.path
-                        self._subrepo_cache[r.path] = res = Repo(p)
-                    except git.exc.InvalidGitRepositoryError:
-                        logger.info("%s: invalid, skipping.", p)
-                        continue
-                    res.submod = r
-                if recurse:
-                    yield from res.subrepos(depth=depth)
-                else:
-                    yield res
+    def last_tag(self, unchanged:bool=False) -> Tag|None:
+        """
+        Return the most-recent tag for this repo
+        """
+        if self._last_tag is not None:
+            return self._last_tag,self._last_tag
+        for c in self._repo.commits(self.head.commit):
+            t = self.tagged(c)
+            if t is None:
+                continue
 
-        if same and recurse and depth:
-            yield self
+            self._last_tag = t
+            if unchanged and self.has_changes(c):
+                raise ChangedError(subsys,t)
+            return t,t
+
+        raise ValueError(f"No tags found")
+
+    def part(self, name):
+        return self._repos[dash(name)]
+
+    @property
+    def _repo(self):
+        return self
+
+    @property
+    def parts(self):
+        return self._repos.values()
+
+    def tags_of(self, c:Commit) -> Sequence[Tag]:
+        return self._commit_tags[c]
+
+    def _add_repo(self, name):
+        dn = dash(name)
+        pn = undash(name)
+        if dn in self._repos:
+            return self._repos[dn]
+
+        p = Package(self,pn)
+        self._repos[dn] = p
+        if "." in pn:
+            par,nam = pn.rsplit(".",1)
+            pp = self._add_repo(par)
+            pp.subs[nam] = p
+        return p
+
+    def _make_repos(self) -> dict:
+        """Collect subrepos"""
+        for fn in Path("packaging").iterdir():
+            if fn.name == "main":
+                continue
+            if not fn.is_dir() or "." in fn.name:
+                continue
+            self._add_repo(str(fn.name))
+
+        self._repos["main"].populate(Path("moat"))
+
+    def repo_for(self, path:Path|str) -> str:
+        """
+        Given a file path, returns the subrepo in question
+        """
+        name = "moat"
+        sc = self._repos["main"]
+        path=Path(path)
+        try:
+            if path.parts[0] == "packaging":
+                return path.parts[1].replace("-",".")
+        except KeyError:
+            return name
+
+        if path.parts[0] != "moat":
+            return None
+
+        for p in path.parts[1:]:
+            if p in sc.subs:
+                name += "."+p
+                sc = sc.subs[p]
+            else:
+                break
+        return name
 
     def commits(self, ref=None):
-        """Iterate over topo sort of commits following @ref, or HEAD"""
+        """Iterate over topo sort of commits following @ref, or HEAD.
+
+        WARNING: this code does not do a true topological breadth-first
+        search. Doesn't matter much for simple merges that are based on
+        a mostly-current checkout, but don't expect correctness when branches
+        span tags.
+        """
         if ref is None:
             ref = self.head.commit
-        try:
-            res = self._commit_topo[ref]
-        except KeyError:
-            visited = set()
-            res = []
-
-            def _it(c):
-                return iter(sorted(c.parents, key=lambda x: x.committed_date))
-
-            work = [(ref, _it(ref))]
-
-            while work:
-                c, gen = work.pop()
-                visited.add(c)
-                for n in gen:
-                    if n not in visited:
-                        work.append((c, gen))
-                        work.append((n, _it(n)))
-                        break
-                else:
-                    res.append(c)
-            self._commit_topo[ref] = res
 
-        n = len(res)
-        while n:
-            n -= 1
-            yield res[n]
+        visited = set()
+        work = deque([ref])
+        while work:
+            ref = work.popleft()
+            if ref in visited:
+                continue
+            visited.add(ref)
+            yield ref
+            work.extend(ref.parents)
+
+    def has_changes(self, tag:Tag=None) -> bool:
+        """
+        Test whether any subsystem changed since the "tagged" commit
+
+        """
+        if tag is None:
+            tag,commit = self.last_tag()
+        head = self._repo.head.commit
+        for d in head.diff(tag):
+            if self.repo_for(d.a_path) == "moat" and self.repo_for(d.b_path) == "moat":
+                continue
+            return True
+        return False
 
-    def tagged(self, c=None) -> str:
-        """Return a commit's tag.
+
+    def tagged(self, c:Commit=None) -> Tag|None:
+        """Return a commit's tag name.
         Defaults to the head commit.
         Returns None if no tag, raises ValueError if more than one is found.
         """
@@ -106,9 +340,17 @@ class Repo(git.Repo):
         if c not in self._commit_tags:
             return None
         tt = self._commit_tags[c]
+
+        tt = [t for t in tt if "/" not in t.name]
+
+        if not tt:
+            return None
         if len(tt) > 1:
-            raise ValueError(f"{self.working_dir}: multiple tags: {tt}")
-        return tt[0]
+            if subsys is not None:
+                raise ValueError(f"Multiple tags for {subsys}: {tt}")
+            raise ValueError(f"Multiple tags: {tt}")
+        return tt[0].name
+
 
 
 @click.group(short_help="Manage MoaT itself")
@@ -116,7 +358,7 @@ async def cli():
     """
     This collection of commands is useful for managing and building MoaT itself.
     """
-    pass  # pylint: disable=unnecessary-pass
+    pass
 
 
 def fix_deps(deps: list[str], tags: dict[str, str]) -> bool:
@@ -132,17 +374,22 @@ def fix_deps(deps: list[str], tags: dict[str, str]) -> bool:
     return work
 
 
-def run_tests(repo: Repo) -> bool:
-    """Run tests (i.e., 'tox') in this repository."""
+def run_tests(pkg: str|None, *opts) -> bool:
+    """Run subtests for subpackage @pkg."""
+
+    if pkg is None:
+        tests = Path("tests")
+    else:
+        tests = dash(pkg).replace("-","_")
+        tests = Path("tests")/tests
 
-    proj = Path(repo.working_dir) / "Makefile"
-    if not proj.is_file():
-        # No Makefile. Assume it's OK.
+    if not Path(tests):
+        # No tests. Assume it's OK.
         return True
     try:
-        print("\n*** Testing:", repo.working_dir)
+        print("\n*** Testing:", pkg)
         # subprocess.run(["python3", "-mtox"], cwd=repo.working_dir, check=True)
-        subprocess.run(["make", "test"], cwd=repo.working_dir, check=True)
+        subprocess.run(["python3","-mpytest", *opts, tests], stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, check=True)
     except subprocess.CalledProcessError:
         return False
     else:
@@ -269,8 +516,9 @@ def encomma(proj, path):
     """list > comma-delimited string"""
     _mangle(proj, path, lambda x: ",".join(x))  # pylint: disable=unnecessary-lambda
 
+
 def apply_hooks(repo, force=False):
-    h = Path(repo.git_dir)/"hooks"
+    h = Path(repo.git_dir) / "hooks"
     drop = set()
     seen = set()
     for f in h.iterdir():
@@ -281,12 +529,12 @@ def apply_hooks(repo, force=False):
     for f in drop:
         f.unlink()
 
-    pt = (Path(__file__).parent / "_hooks")
+    pt = Path(__file__).parent / "_hooks"
     for f in pt.iterdir():
         if not force:
             if f.name in seen:
                 continue
-        t = h/f.name
+        t = h / f.name
         d = f.read_text()
         t.write_text(d)
         t.chmod(0o755)
@@ -382,8 +630,9 @@ def apply_templates(repo):
             txi = "\n" + txi.replace("\n\t", "\n ")
             proj["tool"]["tox"] = dict(
                 legacy_tox_ini=tomlkit.items.String.from_raw(
-                    txi, type_=tomlkit.items.StringType.MLB
-                )
+                    txi,
+                    type_=tomlkit.items.StringType.MLB,
+                ),
             )
 
         projp = Path(repo.working_dir) / "pyproject.toml"
@@ -445,386 +694,438 @@ def path_():
     print(Path(__file__).parent / "_templates")
 
 
-@cli.command(
-    epilog="""\
-By default, changes amend the HEAD commit if the text didn't change.
-"""
-)
-@click.option("-A", "--amend", is_flag=True, help="Fix previous commit (DANGER)")
-@click.option("-N", "--no-amend", is_flag=True, help="Don't fix prev commit even if same text")
-@click.option("-D", "--no-dirty", is_flag=True, help="don't check for dirtiness (DANGER)")
-@click.option("-C", "--no-commit", is_flag=True, help="don't commit")
-@click.option("-s", "--skip", type=str, multiple=True, help="skip this repo")
-@click.option("--hooks", is_flag=True, help="only update hooks")
-@click.option("--HOOKS", "fhooks", is_flag=True, help="force-update hooks")
-@click.option(
-    "-m",
-    "--message",
-    type=str,
-    help="commit message if changed",
-    default="Update from MoaT template",
-)
-@click.option("-o", "--only", type=str, multiple=True, help="affect only this repo")
-async def setup(no_dirty, no_commit, skip, only, message, amend, no_amend, hooks, fhooks):
-    """
-    Set up projects using templates.
-
-    Default: amend if the text is identical and the prev head isn't tagged.
-    """
-    repo = Repo()
-    skip = set(skip)
-    if only:
-        repos = (Repo(x) for x in only)
-    else:
-        repos = (x for x in repo.subrepos(depth=True) if x.moat_name[5:] not in skip)
+@cli.command()
+@click.option("-s", "--show", is_flag=True, help="Show all tags")
+@click.option("-r", "--run", is_flag=True, help="Update all stale tags")
+def tags(show,run):
+    repo = Repo(None)
 
-    for r in repos:
-        apply_hooks(r, fhooks)
-        if hooks or fhooks:
-            print(r.working_dir)
-            continue
+    if show:
+        if run:
+            raise click.UsageError("Can't display and change the tag at the same time!")
 
-        if not is_clean(r, not no_dirty):
-            if not no_dirty:
+        for r in repo.parts:
+            try:
+                tag,commit = r.last_tag()
+            except ValueError:
+                print(f"{r.dash} -")
                 continue
+            if r.has_changes(commit):
+                print(f"{r.dash} {tag} STALE")
+            else:
+                print(f"{r.dash} {tag}")
+        return
 
-        proj = Path(r.working_dir) / "pyproject.toml"
-        if proj.is_file():
-            apply_templates(r)
-        else:
-            logger.info("%s: no pyproject.toml file. Skipping.", r.working_dir)
+    if repo.is_dirty(index=True, working_tree=True, untracked_files=True, submodules=False):
+        print("Repo is dirty. Not tagging globally.", file=sys.stderr)
+        return
+
+    changed=False
+    for r in repo.parts:
+        tag,commit = r.last_tag()
+        if not r.has_changes(commit):
+            print(repo.dash,tag,"UNCHANGED")
             continue
 
-        if no_commit:
+        tag = r.next_tag()
+        print(repo.dash,tag)
+        if not run:
             continue
 
-        if r.is_dirty(index=True, working_tree=True, untracked_files=False, submodules=True):
-            if no_amend or r.tagged():
-                a = False
-            elif amend:
-                a = True
-            else:
-                a = r.head.commit.message == message
-
-            for rr in r.subrepos(recurse=False):
-                # This part updates the supermodule's SHA entries so they
-                # point to the submodule#s current HEAD, assuming it is
-                # clean
-                if not rr.is_dirty(index=True, working_tree=True, submodules=True):
-                    rrp = rr.submod.path
-                    rri = rr.head.commit.hexsha
-                    ri = r.index.entries[(rrp, 0)].hexsha
-
-                    if rri == ri:
-                        continue
-                    sn = git.objects.submodule.base.Submodule(
-                        r,
-                        rr.head.commit.binsha,
-                        name=rr.submod.name,
-                        path=rr.submod.path,
-                        mode=rr.submod.mode,
-                    )
-                    print("Submodule update:", rrp)
-                    r.index.add([sn])  # doesn't work, SIGH
-            if a:
-                p = r.head.commit.parents
-            else:
-                p = (r.head.commit,)
+        repo.versions[r.dash] = (tag,repo.head.commit.hexsha[:9])
+        changed=True
 
-            r.index.commit(message, parent_commits=p)
+    if changed:
+        repo.write_tags()
 
 
 @cli.command()
-@click.option("-P", "--no-pypi", is_flag=True, help="don't push to PyPi")
-@click.option("-D", "--no-deb", is_flag=True, help="don't debianize")
-@click.option("-d", "--deb", type=str, help="Debian archive to push to (from dput.cfg)")
-@click.option("-o", "--only", type=str, multiple=True, help="affect only this repo")
-@click.option("-s", "--skip", type=str, multiple=True, help="skip this repo")
-async def publish(no_pypi, no_deb, skip, only, deb):
+@click.option("-r", "--run", is_flag=True, help="actually do the tagging")
+@click.option("-m", "--minor", is_flag=True, help="create a new minor version")
+@click.option("-M", "--major", is_flag=True, help="create a new major version")
+@click.option("-s", "--subtree", type=str, help="Tag this partial module")
+@click.option("-v", "--tag", "force", type=str, help="Use this explicit tag value")
+@click.option("-q", "--query","--show","show", is_flag=True, help="Show the latest tag")
+@click.option("-f", "--force","FORCE", is_flag=True, help="replace an existing tag")
+def tag(run,minor,major,subtree,force,FORCE,show):
     """
-    Publish modules to PyPi and/or Debian.
+    Tag the repository (or a subtree).
     """
-    repo = Repo(None)
-    skip = set(skip)
-    if only:
-        repos = (Repo(repo, x) for x in only)
-    else:
-        repos = (x for x in repo.subrepos() if x.moat_name[5:] not in skip)
-
-    if not no_deb:
-        for r in repos:
-            p = Path(r.working_dir) / "pyproject.toml"
-            if not p.is_file():
-                continue
-            print(r.working_dir)
-            args = ["-d", deb] if deb else []
-            subprocess.run(["merge-to-deb"] + args, cwd=r.working_dir, check=True)
+    if minor and major:
+        raise click.UsageError("Can't change both minor and major!")
+    if force and (minor or major):
+        raise click.UsageError("Can't use an explicit tag with changing minor or major!")
+    if FORCE and (minor or major):
+        raise click.UsageError("Can't use an explicit tag with changing minor or major!")
+    if show and (run or force or minor or major):
+        raise click.UsageError("Can't display and change the tag at the same time!")
 
-    if not no_pypi:
-        for r in repos:
-            p = Path(r.working_dir) / "pyproject.toml"
-            if not p.is_file():
-                continue
-            print(r.working_dir)
-            subprocess.run(["make", "pypi"], cwd=r.working_dir, check=True)
+    repo = Repo(None)
 
+    if subtree:
+        r = repo.part(subtree)
+    else:
+        r = repo
 
-async def fix_main(repo):
-    """
-    Set "main" references to the current HEAD.
+    if show:
+        tag,commit = r.last_tag()
+        if r.has_changes(commit):
+            print(f"{tag} STALE")
+        else:
+            print(tag)
+        return
 
-    Repos with a non-detached head are skipped.
-    Reports an error if HEAD is not a direct descendant.
-    """
+    if force:
+        tag = force
+    elif FORCE:
+        tag,_ = r.last_tag()
+    else:
+        tag = r.next_tag(major,minor)
 
-    async def _fix(r):
-        if not r.head.is_detached:
-            if r.head.ref.name not in {"main", "moat"}:
-                print(f"{r.working_dir}: Head is {r.head.ref.name !r}", file=sys.stderr)
-            return
-        if "moat" in r.refs:
-            m = r.refs["moat"]
+    if run or subtree:
+        if subtree:
+            repo.versions[r.dash] = (tag,repo.head.commit.hexsha[:9])
+            repo.write_tags()
         else:
-            m = r.refs["main"]
-        if m.commit != r.head.commit:
-            ch = await run_process(
-                ["git", "-C", r.working_dir, "merge-base", m.commit.hexsha, r.head.commit.hexsha],
-                input=None,
-                stderr=sys.stderr,
-            )
-            ref = ch.stdout.decode().strip()
-            if ref != m.commit.hexsha:
-                print(f"{r.working_dir}: need merge", file=sys.stderr)
-                return
-            m.set_reference(ref=r.head.commit, logmsg="fix_main")
-        r.head.set_reference(ref=m, logmsg="fix_main 2")
-
-    for r in repo.subrepos():
-        await _fix(r)
+            git.TagReference.create(repo,tag, force=FORCE)
+        print(f"{tag}")
+    else:
+        print(f"{tag} DRY_RUN")
 
 
 @cli.command()
-async def fixref():
+@click.option("-P", "--no-pypi", is_flag=True, help="don't push to PyPi")
+@click.option("-D", "--no-deb", is_flag=True, help="don't debianize")
+@click.option("-d", "--deb", type=str, help="Debian archive to push to (from dput.cfg)")
+@click.option("-o", "--only", type=str, multiple=True, help="affect only this package")
+@click.option("-s", "--skip", type=str, multiple=True, help="skip this package")
+async def publish(no_pypi, no_deb, skip, only, deb):
     """
-    Reset 'main' ref to HEAD
-
-    Submodules frequently have detached HEADs. This command resets "main"
-    to them, but only if that is a fast-forward operation.
+    Publish modules to PyPi and/or Debian.
 
-    An error message is printed if the head doesn't point to "main", or if
-    the merge wouldn't be a fast-forward operation.
+    MoaT modules can be given as shorthand, i.e. with dashes and excluding
+    the "moat-" prefix.
     """
     repo = Repo(None)
-    await fix_main(repo)
+    if only and skip:
+        raise click.UsageError("You can't both include and exclude packages.")
 
+    if only:
+        repos = (repo.subrepo(x) for x in only)
+    else:
+        s = set()
+        for sk in skip:
+            s += set(sk.split(","))
+        repos = (x for x in repo.parts if dash(x.name) not in sk)
 
-@cli.command()
-@click.option("-r", "--remote", type=str, help="Remote. Default: all.", default="--all")
-@click.pass_obj
-async def push(obj, remote):
-    """Push the current state"""
+    deb_args = "-b -us -uc".split()
 
-    repo = Repo(None)
-    for r in repo.subrepos():
-        try:
-            cmd = ["git", "-C", r.working_dir, "push", "--tags"]
-            if not obj.debug:
-                cmd.append("-q")
-            elif obj.debug > 1:
-                cmd.append("-v")
-            cmd.append(remote)
-            await run_process(cmd, input=None, stdout=sys.stdout, stderr=sys.stderr)
-
-        except subprocess.CalledProcessError as exc:
-            print("  Error in", r.working_dir, file=sys.stderr)
-            sys.exit(exc.returncode)
+    for r in repos:
+        t,c = r.last_tag
+        if r.has_changes(c):
+            print(f"Error: changes in {r.name} since tag {t.name}")
+            continue
 
+        print(f"Processing {r.name}, tag: {t.name}")
+        r.copy()
+        rd=PACK/r.dash
 
-@cli.command()
-@click.option("-r", "--remote", type=str, help="Remote to fetch. Default: probably 'origin'.")
-@click.option("-b", "--branch", type=str, default=None, help="Branch to merge.")
-@click.pass_obj
-async def pull(obj, remote, branch):
-    """Fetch updates"""
+        if not no_deb:
+            p = rd / "debian"
+            if not p.is_dir():
+                continue
+            subprocess.run(["debuild"] + deb_args, cwd=rd, check=True)
 
-    repo = Repo(None)
-    for r in repo.subrepos():
-        try:
-            cmd = ["git", "-C", r.working_dir, "fetch", "--recurse-submodules=no", "--tags"]
-            if not obj.debug:
-                cmd.append("-q")
-            elif obj.debug > 1:
-                cmd.append("-v")
-            if remote is not None:
-                cmd.append(remote)
-            await run_process(cmd, input=None, stdout=sys.stdout, stderr=sys.stderr)
-
-            cmd = ["git", "-C", r.working_dir, "merge", "--ff"]
-            if not obj.debug:
-                cmd.append("-q")
-            elif obj.debug > 1:
-                cmd.append("-v")
-            if remote is not None:
-                if branch is None:
-                    branch = "moat" if "moat" in r.refs else "main"
-                cmd.append(f"{remote}/{branch}")
-            await run_process(cmd, input=None, stdout=sys.stdout, stderr=sys.stderr)
-
-        except subprocess.CalledProcessError as exc:
-            print("  Error in", r.working_dir, file=sys.stderr)
-            sys.exit(exc.returncode)
+    if not no_pypi:
+        for r in repos:
+            p = Path(r.working_dir) / "pyproject.toml"
+            if not p.is_file():
+                continue
+            print(r.working_dir)
+            subprocess.run(["make", "pypi"], cwd=r.working_dir, check=True)
 
 
-@cli.command()
-@click.option("-T", "--no-test", is_flag=True, help="Skip testing")
+@cli.command(epilog="""
+The default for building Debian packages is '--no-sign --build=binary'.
+'--no-sign' is dropped when you use '--deb'.
+The binary-only build is currently unconditional.
+
+The default for uploading to Debian via 'dput' is '--unchecked ext';
+it is dropped when you use '--dput'.
+""")
+@click.option("-f", "--no-dirty", is_flag=True, help="don't check for dirtiness (DANGER)")
+@click.option("-F", "--no-tag", is_flag=True, help="don't check for tag uptodate-ness (DANGER)")
+@click.option("-D", "--no-deb", is_flag=True, help="don't build Debian packages")
+@click.option("-C", "--no-commit", is_flag=True, help="don't commit the result")
+@click.option("-V", "--no-version", is_flag=True, help="don't update dependency versions in pyproject files")
+@click.option("-P", "--no-pypi", is_flag=True, help="don't push to PyPI")
+@click.option("-T", "--no-test", is_flag=True, help="don't run tests")
+@click.option("-o", "--pytest", "pytest_opts", type=str,multiple=True, help="Options for pytest")
+@click.option("-d", "--deb", "deb_opts", type=str,multiple=True, help="Options for debuild")
+@click.option("-p", "--dput", "dput_opts", type=str,multiple=True, help="Options for dput")
+@click.option("-r", "--run", is_flag=True, help="actually do the tagging")
+@click.option("-s", "--skip", "skip_", type=str,multiple=True, help="skip these repos")
+@click.option("-m", "--minor", is_flag=True, help="create a new minor version")
+@click.option("-M", "--major", is_flag=True, help="create a new major version")
+@click.option("-t", "--tag", "forcetag", type=str, help="Use this explicit tag value")
 @click.option(
     "-v",
     "--version",
     type=(str, str),
     multiple=True,
-    help="Update external dep version",
+    help="Update external dependency",
 )
-@click.option("-C", "--no-commit", is_flag=True, help="don't commit")
-@click.option("-D", "--no-dirty", is_flag=True, help="don't check for dirtiness (DANGER)")
-@click.option("-c", "--cache", is_flag=True, help="don't re-test if unchanged")
-async def build(version, no_test, no_commit, no_dirty, cache):
+@click.argument("parts", nargs=-1)
+async def build(no_commit, no_dirty, no_test, no_tag, no_pypi, parts, dput_opts, pytest_opts, deb_opts, run, version, no_version, no_deb, skip_, major,minor,forcetag):
     """
     Rebuild all modified packages.
     """
-    bad = False
     repo = Repo(None)
+
     tags = dict(version)
     skip = set()
-    heads = attrdict()
-
-    if repo.is_dirty(index=True, working_tree=True, untracked_files=False, submodules=False):
-        print("Please commit top-level changes and try again.")
-        return
-
-    if cache:
-        cache = Path(".tested.yaml")
-        try:
-            heads = yload(cache, attr=True)
-        except FileNotFoundError:
-            pass
-
-    for r in repo.subrepos():
-        if not is_clean(r, not no_dirty):
-            bad = True
-            if not no_dirty:
-                skip.add(r)
-                continue
-
-        if not no_test and heads.get(r.moat_name, "") != r.commit().hexsha and not run_tests(r):
-            print("FAIL", r.moat_name)
-            bad = True
-            break
+    for s in skip_:
+        for sn in s.split(","):
+            skip.add(dash(sn))
+    parts = set(dash(s) for s in parts)
+    debversion={}
+
+    if no_tag and not no_version:
+        print("Warning: not updating moat versions in pyproject files", file=sys.stderr)
+    if minor and major:
+        raise click.UsageError("Can't change both minor and major!")
+    if forcetag and (minor or major):
+        raise click.UsageError("Can't use an explicit tag with changing minor or major!")
+
+    if forcetag is None:
+        forcetag = repo.next_tag(major,minor)
+
+    full = False
+    if parts:
+        repos = [ repo.part(x) for x in parts ]
+    else:
+        if not skip:
+            full = True
+        repos = [ x for x in repo.parts if not x.hidden and x.dash not in skip and not (PACK/x.dash/"SKIP").exists() ]
 
-        if r.is_dirty(index=True, working_tree=True, untracked_files=True, submodules=False):
-            print("DIRTY", r.moat_name)
-            if r.moat_name != "src":
-                bad = True
+    for name in PACK.iterdir():
+        if name.suffix != ".changes":
             continue
-
-        heads[r.moat_name] = r.commit().hexsha
-        t = r.tagged(r.head.commit)
-        if t is not None:
-            if r.is_dirty(index=False, working_tree=False, untracked_files=False, submodules=True):
-                t = None
-
-        if t is None:
-            for c in r.commits():
-                t = r.tagged(c)
-                if t is not None:
-                    break
-            else:
-                print("NOTAG", t, r.moat_name)
-                bad = True
-                continue
-            print("UNTAGGED", t, r.moat_name)
-            xt, t = t.name.rsplit(".", 1)
-            t = f"{xt}.{int(t)+1}"
-            # t = r.create_tag(t)
-            # do not create the tag yet
+        name=name.stem
+        name,vers,_ = name.split("_")
+        if name.startswith("moat-"):
+            name = name[5:]
         else:
-            print("TAG", t, r.moat_name)
-        tags[r.moat_name] = t
-
-    if cache:
-        with cache.open("w") as f:
-            # always write cache file
-            yprint(heads, stream=f)
-    if bad:
-        print("No work done. Fix and try again.")
-        return
-
-    dirty = set()
+            name = "ext-"+name
+        debversion[name]=vers.rsplit("-",1)[0]
+
+    
+    # Step 0: basic check
+    if not no_dirty:
+        if repo.is_dirty(index=False, working_tree=True, untracked_files=True, submodules=False):
+            if not run:
+                print("*** Repository is not clean.", file=sys.stderr)
+            else:
+                print("Please commit changes and try again.", file=sys.stderr)
+                return
 
-    check = True
+    # Step 1: check for changed files since last tagging
+    if not no_tag:
+        err = set()
+        for r in repos:
+            try:
+                tag,commit = r.last_tag()
+            except KeyError:
+                rd = PACK/r.dash
+                p = rd / "pyproject.toml"
+                if not p.is_file():
+                    continue
+                raise
+            tags[r.mdash] = tag
+            if r.has_changes(commit):
+                err.add(r.dash)
+        if err:
+            if not run:
+                print("*** Untagged changes:", file=sys.stderr)
+                print("***", *err, file=sys.stderr)
+            else:
+                print("Untagged changes:", file=sys.stderr)
+                print(*err, file=sys.stderr)
+                print("Please tag (moat src tag -s PACKAGE) and try again.", file=sys.stderr)
+                return
 
-    while check:
-        check = False
+    # Step 2: run tests
+    if not no_test:
+        fails = set()
+        for p in parts:
+            if not run_tests(p, *pytest_opts):
+                fails.add(p.name)
+        if fails:
+            if not run:
+                print(f"*** Tests failed:", *fails, file=sys.stderr)
+            else:
+                print(f"Failed tests:", *fails, file=sys.stderr)
+                print(f"Fix and try again.", file=sys.stderr)
+                return
 
-        # Next: fix versioned dependencies
-        for r in repo.subrepos():
-            if r in skip:
-                continue
-            p = Path(r.working_dir) / "pyproject.toml"
-            if not p.is_file():
-                # bad=True
-                print("Skip:", r.working_dir)
-                continue
-            with p.open("r") as f:
-                pr = tomlkit.load(f)
+    # Step 3: set version and fix versioned dependencies
+    for r in repos:
+        rd = PACK/r.dash
+        p = rd / "pyproject.toml"
+        if not p.is_file():
+            # bad=True
+            print("Skip:", r.name, file=sys.stderr)
+            continue
+        with p.open("r") as f:
+            pr = tomlkit.load(f)
+            pr["project"]["version"] = r.last_tag()[0]
 
-            work = False
+        if not no_version:
             try:
                 deps = pr["project"]["dependencies"]
             except KeyError:
                 pass
             else:
-                work = fix_deps(deps, tags) | work
+                fix_deps(deps, tags)
             try:
                 deps = pr["project"]["optional_dependencies"]
             except KeyError:
                 pass
             else:
                 for v in deps.values():
-                    work = fix_deps(v, tags) | work
-
-            if work:
-                p.write_text(pr.as_string())
-                r.index.add(p)
-            if work or r.is_dirty(index=False, working_tree=False, submodules=True):
-                dirty.add(r)
-                t = tags[r.moat_name]
-                if not isinstance(t, str):
-                    xt, t = t.name.rsplit(".", 1)
-                    t = f"{xt}.{int(t)+1}"
-                    tags[r.moat_name] = t
-                check = True
-
-    if bad:
-        print("Partial work done. Fix and try again.")
-        return
+                    fix_deps(v, tags)
+
+        p.write_text(pr.as_string())
+        repo.index.add(p)
 
-    if not no_commit:
-        for r in repo.subrepos(depth=True):
-            if not r.is_dirty(
-                index=True, working_tree=True, untracked_files=False, submodules=True
-            ):
+    # Step 3: copy to packaging dir
+    for r in repos:
+        r.copy()
+        
+    # Step 4: build Debian package
+    if not no_deb:
+        if not deb_opts:
+            deb_opts = ["--no-sign"]
+
+        for r in repos:
+            rd=PACK/r.dash
+            p = rd / "debian"
+            if not p.is_dir():
                 continue
+            try:
+                res = subprocess.run(["dpkg-parsechangelog","-l","debian/changelog","-S","version"], cwd=rd, check=True, stdout=subprocess.PIPE)
+                tag = res.stdout.strip().decode("utf-8").rsplit("-",1)[0]
+                ltag = r.last_tag()[0]
+                if tag != ltag:
+                    subprocess.run(["debchange", "--distribution","unstable", "--newversion",ltag+"-1",f"New release for {forcetag}"] , cwd=rd, check=True)
+                    repo.index.add(p/"changelog")
+
+                if debversion.get(r.dash,"") != ltag:
+                    subprocess.run(["debuild", "--build=binary"] + deb_opts, cwd=rd, check=True)
+            except subprocess.CalledProcessError:
+                if not run:
+                    print("*** Failure packaging",r.name,file=sys.stderr)
+                else:
+                    print("Failure packaging",r.name,file=sys.stderr)
+                    return
 
-            if r in dirty:
-                r.index.commit("Update MoaT requirements")
+    # Step 5: build PyPI package
+    if not no_pypi:
+        err=set()
+        up=set()
+        for r in repos:
+            rd=PACK/r.dash
+            p = rd / "pyproject.toml"
+            if not p.is_file():
+                continue
+            tag = r.last_tag()[0]
+            name = r.dash
+            if name.startswith("ext-"):
+                name=name[4:]
+            else:
+                name="moat-"+r.dash
+
+            targz = rd/"dist"/f"{r.under}-{tag}.tar.gz"
+            done = rd/"dist"/f"{r.under}-{tag}.done"
+            if targz.is_file():
+                if not done.exists():
+                    up.add(r)
+            else:
+                try:
+                    subprocess.run(["python3", "-mbuild", "-snw"], cwd=rd, check=True)
+                except subprocess.CalledProcessError:
+                    err.add(r.name)
+                else:
+                    up.add(r)
+        if err:
+            if not run:
+                print("*** Build errors:", file=sys.stderr)
+                print("***", *err, file=sys.stderr)
+            else:
+                print("Build errors:", file=sys.stderr)
+                print(*err, file=sys.stderr)
+                print("Please fix and try again.", file=sys.stderr)
+                return
+        
+        # Step 6: upload PyPI package
+        if run:
+            err=set()
+            for r in up:
+                rd=PACK/r.dash
+                p = rd / "pyproject.toml"
+                if not p.is_file():
+                    continue
+                tag = r.last_tag()[0]
+                name = r.dash
+                if name.startswith("ext-"):
+                    name=name[4:]
+                else:
+                    name="moat-"+r.dash
+                targz = Path("dist")/f"{r.under}-{tag}.tar.gz"
+                whl = Path("dist")/f"{r.under}-{tag}-py3-none-any.whl"
+                try:
+                    res = subprocess.run(["twine", "upload", str(targz), str(whl)], cwd=rd, check=True)
+                except subprocess.CalledProcessError:
+                    err.add(r.name)
+                else:
+                    done = rd/"dist"/f"{r.under}-{tag}.done"
+                    done.touch()
+            if err:
+                print("Upload errors:", file=sys.stderr)
+                print(*err, file=sys.stderr)
+                print("Please fix(?) and try again.", file=sys.stderr)
+                return
 
-            for rr in r.subrepos(recurse=False):
-                r.git.add(rr.working_dir)
-            r.index.commit("Submodule Update")
+    # Step 7: upload Debian package
+    if run and not no_deb:
+        err = set()
+        if not dput_opts:
+            dput_opts = ["-u","ext"]
+        for r in repos:
+            ltag = r.last_tag()[0]
+            if not (PACK/r.dash/"debian").is_dir():
+                continue
+            changes = PACK/f"{r.mdash}_{ltag}-1_{ARCH}.changes"
+            done = PACK/f"{r.mdash}_{ltag}-1_{ARCH}.done"
+            if done.exists():
+                continue
+            try:
+                subprocess.run(["dput", *dput_opts, str(changes)], check=True)
+            except subprocess.CalledProcessError:
+                err.add(r.name)
+            else:
+                done.touch()
+        if err:
+            print("Upload errors:", file=sys.stderr)
+            print(*err, file=sys.stderr)
+            print("Please fix(?) and try again.", file=sys.stderr)
+            return
 
-            t = tags[r.moat_name]
-            if isinstance(t, str):
-                r.create_tag(t)
+    # Step 8: commit the result
+    if run and not no_commit:
+        repo.write_tags()
+        repo.index.commit(f"Build version {forcetag}")
+        git.TagReference.create(repo, forcetag)
 
 
 add_repr(tomlkit.items.String)