pygitpub 0.0.63__tar.gz → 0.0.65__tar.gz

{pygitpub-0.0.63 → pygitpub-0.0.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pygitpub
-Version: 0.0.63
+Version: 0.0.65
 Summary: Pygitpub will help you work with github
 Project-URL: Homepage, https://github.com/veltzer/pygitpub
 Project-URL: Bug Tracker, https://github.com/veltzer/pygitpub/issues
@@ -41,9 +41,7 @@ project website: https://veltzer.github.io/pygitpub
 
 author: Mark Veltzer
 
-version: 0.0.63
-
-![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)
+version: 0.0.65
 
 ## github
 

{pygitpub-0.0.63 → pygitpub-0.0.65}/README.md

@@ -9,9 +9,7 @@ project website: https://veltzer.github.io/pygitpub
 
 author: Mark Veltzer
 
-version: 0.0.63
-
-![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)
+version: 0.0.65
 
 ## github
 

{pygitpub-0.0.63 → pygitpub-0.0.65}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pygitpub"
-version = "0.0.63"
+version = "0.0.65"
 requires-python = ">=3.14"
 authors = [
 	{ name = "Mark Veltzer", email = "mark.veltzer@gmail.com" }
@@ -41,8 +41,7 @@ dependencies = [
 
 [dependency-groups]
 dev = [
-    "mypy",
-    "pylint",
+    "pyrefly",
     "pytest",
     "ruff",
     "sphinx",
@@ -83,24 +82,28 @@ exclude = [
     "*.md",
 ]
 
-[tool.mypy]
-python_version = "3.14"
-warn_return_any = true
-warn_unused_configs = true
-mypy_path = "src:python:scripts"
-explicit_package_bases = true
-namespace_packages = true
-
-# Third-party libraries without type stubs; `foo.*` also matches `foo` itself.
-[[tool.mypy.overrides]]
-module = [
+[tool.pyrefly]
+# Type checking (rsconstruct's pyrefly processor). preset = "legacy" is
+# pyrefly's mypy-compatible behavior; "." on the search path is mypy's
+# implicit current directory.
+preset = "legacy"
+python-version = "3.14"
+search-path = [
+    "src",
+    ".",
+]
+# Third-party libraries without type information.
+ignore-missing-imports = [
     "github.*",
     "lupa.*",
     "pyapikey.*",
     "pylogconf.*",
     "pytconf.*",
 ]
-ignore_missing_imports = true
+
+[tool.pyrefly.errors]
+no-any-return = "error"
+
 [tool.hatch.envs.default]
 installer = "uv"
 

{pygitpub-0.0.63 → pygitpub-0.0.65}/src/pygitpub/configs.py

@@ -14,11 +14,8 @@ AFFILIATIONS = [
 
 class ConfigGithub(Config):
     """
-    Paramters for your github account
+    Parameters for your github account
     """
-    username = ParamCreator.create_str(
-        help_string="What is your github username?",
-    )
     apikey = ParamCreator.create_str(
         help_string="What is the name of your apikey (see pyapikey)?",
     )
@@ -63,6 +60,26 @@ class ConfigAlgo(Config):
     )
 
 
+# The ways the runs_show_failing and runs_show_not_success endpoints can
+# query github. The first is the default.
+RUNS_APIS = [
+    "graphql",
+    "rest",
+]
+
+
+class ConfigRuns(Config):
+    """
+    Parameters to control how workflow runs are queried
+    """
+    api = ParamCreator.create_choice(
+        choice_list=RUNS_APIS,
+        help_string="graphql: check the default branch head commit of all repos in parallel batches (fast);"
+                    " rest: walk every workflow's last run on any branch (slow, one call per workflow)",
+        default="graphql",
+    )
+
+
 class ConfigOutput(Config):
     """
     Parameters to control output

{pygitpub-0.0.63 → pygitpub-0.0.65}/src/pygitpub/main.py

@@ -6,6 +6,9 @@ import json
 import os
 import subprocess
 import sys
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
 
 import github
 import pyapikey
@@ -15,7 +18,7 @@ from pytconf.pydoc import get_first_line
 from pytconf.registry import the_registry
 
 import pygitpub.static
-from pygitpub.configs import AFFILIATIONS, ConfigAlgo, ConfigGithub, ConfigOutput
+from pygitpub.configs import AFFILIATIONS, ConfigAlgo, ConfigGithub, ConfigOutput, ConfigRuns
 from pygitpub.utils.lua import config_path, load_lua_file
 from pygitpub.utils.misc import get_all_git_repos
 
@@ -31,7 +34,7 @@ def get_repo_folder(repo) -> str:
 
 def get_affiliation() -> str:
     """ validate the affiliation parameter and return it in the form github wants """
-    values = [x.strip() for x in ConfigAlgo.affiliation.split(",")]
+    values: list[str] = [x.strip() for x in ConfigAlgo.affiliation.split(",")]
     values = [x for x in values if x != ""]
     if not values:
         raise ValueError("affiliation must not be empty")
@@ -41,9 +44,13 @@ def get_affiliation() -> str:
     return ",".join(values)
 
 
-def yield_repos():
+def get_github(**kwargs: Any) -> github.Github:
     apikey = pyapikey.get_key(ConfigGithub.apikey)
-    g = github.Github(login_or_token=apikey)
+    return github.Github(auth=github.Auth.Token(apikey), **kwargs)
+
+
+def yield_repos():
+    g = get_github()
     # github filters by affiliation server side, so repos we are not interested
     # in (org repos, by default) are never fetched in the first place
     affiliation = get_affiliation()
@@ -92,7 +99,8 @@ def fix_metadata() -> None:
             print(f"in {folder}...")
             print(f"description_short is [{description_short}]")
             print(f"repo.description is [{repo.description}]")
-            repo.edit(description=description_short)
+            if not ConfigAlgo.dryrun:
+                repo.edit(description=description_short)
         if "KEYWORDS" not in project:
             print(f"{folder} no keywords")
             continue
@@ -101,32 +109,36 @@ def fix_metadata() -> None:
             print(f"in {folder}...")
             print(f"keywords is [{keywords}]")
             print(f"repo.get_topics() is [{repo.get_topics()}]")
-            repo.replace_topics(keywords)
+            if not ConfigAlgo.dryrun:
+                repo.replace_topics(keywords)
 
 
 @register_endpoint(
     description="Set the correct website for the project on all repos where it is not correct",
     configs=[
         ConfigGithub,
+        ConfigOutput,
         ConfigAlgo,
     ],
 )
 def homepage_fix() -> None:
     for repo in yield_repos():
+        owner = repo.owner.login
+        # the user site repo is served at the root of the site, usually under
+        # a custom domain, so the pattern below does not apply to it
+        if repo.name.lower() == f"{owner.lower()}.github.io":
+            if ConfigOutput.verbose:
+                print(f"skipping [{repo.name}] because it is the user site")
+            continue
         if repo.has_pages:
-            right_homepage = f"https://{ConfigGithub.username}.github.io/{repo.name}/"
-            if repo.homepage != right_homepage:
-                print(f"{repo.name} is wrong!")
-                print(f"homepage: {repo.homepage}")
-                print(f"right_homepage: {right_homepage}")
-                print(f"patching [{repo.name}]...")
-                repo.edit(repo.name, homepage=right_homepage)
+            right_homepage = f"https://{owner}.github.io/{repo.name}/"
         else:
-            right_homepage = f"https://github.com/{ConfigGithub.username}/{repo.name}/"
-            if repo.homepage != right_homepage:
-                print(f"{repo.name} is wrong!")
-                print(f"homepage: {repo.homepage}")
-                print(f"right_homepage: {right_homepage}")
+            right_homepage = f"https://github.com/{owner}/{repo.name}/"
+        if repo.homepage != right_homepage:
+            print(f"{repo.name} is wrong!")
+            print(f"homepage: {repo.homepage}")
+            print(f"right_homepage: {right_homepage}")
+            if not ConfigAlgo.dryrun:
                 print(f"patching [{repo.name}]...")
                 repo.edit(repo.name, homepage=right_homepage)
 
@@ -195,20 +207,23 @@ def repos_list() -> None:
     ],
 )
 def cleanup() -> None:
-    # pylint: disable=too-many-branches,too-many-statements
     for repo in yield_repos():
         # 1. Cleanup workflow runs
         for workflow in repo.get_workflows():
             existing = 0
             for run in workflow.get_runs():
                 if ConfigOutput.verbose:
-                    print(f"inspecting [workflow run] {repo.name} {workflow.name} {run.conclusion}")
+                    print(f"inspecting [workflow run] {repo.name} {workflow.name} {run.status} {run.conclusion}")
+                # github refuses to delete a run that is queued or still
+                # running, so leave those alone and don't count them
+                if run.status != "completed":
+                    continue
                 delete_it = False
-                # if its a pages build delete it unless its in mid work (run.conclusion is None)
-                if workflow.name == "pages-build-deployment" and run.conclusion is not None:
+                # if its a pages build delete it
+                if workflow.name == "pages-build-deployment":
                     delete_it = True
-                # if its not on the master branch, delete it
-                if run.head_branch != "master":
+                # if its not on the default branch, delete it
+                if run.head_branch != repo.default_branch:
                     delete_it = True
                 # if its not a paged build and it failed then delete it
                 if workflow.name != "pages-build-deployment" and run.conclusion in (
@@ -219,6 +234,8 @@ def cleanup() -> None:
                     delete_it = True
                 if delete_it:
                     print(f"deleting [workflow run] {repo.name} {workflow.name} {run.head_branch} {run.conclusion} {run.url}")
+                    if ConfigAlgo.dryrun:
+                        continue
                     try:
                         run.delete()
                     except github.GithubException as e:
@@ -242,12 +259,13 @@ def cleanup() -> None:
 
             if delete_it:
                 print(f"deleting [deployment] {repo.name} {deployment.id}")
+                if ConfigAlgo.dryrun:
+                    continue
                 try:
                     deployment.create_status("inactive")
                 except github.GithubException as e:
                     print(f"Failed to deactivate deployment: {e}")
                 try:
-                    # pylint: disable=protected-access
                     status, _, _ = deployment._requester.requestJson("DELETE", deployment.url)
                     if status != 204:
                         print(f"Failed to delete deployment: HTTP {status}")
@@ -259,6 +277,8 @@ def cleanup() -> None:
         for release in repo.get_releases():
             if existing_releases >= 4:
                 print(f"deleting [release] {repo.name} {release.title}")
+                if ConfigAlgo.dryrun:
+                    continue
                 try:
                     release.delete_release()
                 except github.GithubException as e:
@@ -292,8 +312,108 @@ def runs_show_running() -> None:
     for repo in yield_repos():
         for workflow in repo.get_workflows():
             for run in workflow.get_runs():
-                if run.conclusion is None:
-                    print(f"{repo.name}: {workflow.name} {run.conclusion}")
+                if run.status in ("in_progress", "queued"):
+                    print(f"{repo.name}: {workflow.name} {run.status}")
+
+
+# repos per GraphQL query; measured fastest at around 10
+GRAPHQL_BATCH_SIZE = 10
+# GraphQL queries in flight at once, well below github's concurrency limits
+GRAPHQL_WORKERS = 26
+
+GRAPHQL_ROLLUP_FIELD = """
+r{index}: repository(owner: {owner}, name: {name}) {{
+  defaultBranchRef {{ target {{ ... on Commit {{ statusCheckRollup {{ state }} }} }} }}
+}}
+"""
+
+GRAPHQL_SUITES_QUERY = """
+query($owner: String!, $name: String!) {
+  repository(owner: $owner, name: $name) {
+    defaultBranchRef {
+      target {
+        ... on Commit {
+          checkSuites(first: 50) {
+            nodes {
+              status
+              conclusion
+              workflowRun { workflow { name } }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+"""
+
+
+def graphql_show_runs(bad_state: Callable[[str], bool], bad_conclusion: Callable[[str], bool]) -> None:
+    """
+    Print the workflows that are not green on the default branch head commit
+    of every repo.
+
+    Resolving statusCheckRollup is slow on github's side (~4.5s per 50 repos)
+    and cursor pagination would run the pages one after another, so the repos
+    are split into small batches, each batch is one query with an aliased
+    `repository` field per repo, and the batches run in parallel. Repos whose
+    rollup state is bad then get one more query each to name the workflows.
+    Only the head commit of the default branch is looked at, so a failed run
+    on another branch, or a scheduled run on an older commit, does not show.
+    """
+    # the default client sleeps a second between POSTs, and every GraphQL
+    # query is a POST, which would serialize the parallel queries
+    g = get_github(
+        pool_size=GRAPHQL_WORKERS,
+        seconds_between_requests=None,
+        seconds_between_writes=None,
+    )
+
+    def graphql(query: str, variables: dict[str, str]) -> dict[str, Any]:
+        _headers, data = g.requester.graphql_query(query, variables)
+        result: dict[str, Any] = data["data"]
+        return result
+
+    def get_states(batch: list[str]) -> list[str | None]:
+        fields = []
+        for index, full_name in enumerate(batch):
+            owner, name = full_name.split("/", 1)
+            # json.dumps gives a correctly quoted GraphQL string literal
+            fields.append(GRAPHQL_ROLLUP_FIELD.format(index=index, owner=json.dumps(owner), name=json.dumps(name)))
+        data = graphql("query {" + "".join(fields) + "}", {})
+        states: list[str | None] = []
+        for index in range(len(batch)):
+            ref = data[f"r{index}"]["defaultBranchRef"]
+            rollup = ref and ref["target"].get("statusCheckRollup")
+            states.append(rollup and rollup["state"])
+        return states
+
+    def get_bad_workflows(full_name: str) -> list[tuple[str, str]]:
+        owner, name = full_name.split("/", 1)
+        data = graphql(GRAPHQL_SUITES_QUERY, {"owner": owner, "name": name})
+        workflows = []
+        for suite in data["repository"]["defaultBranchRef"]["target"]["checkSuites"]["nodes"]:
+            # github creates an empty, never-run suite for every installed
+            # app; without a workflow run behind it, it tells us nothing
+            if suite["workflowRun"] is None:
+                continue
+            # an unfinished suite has no conclusion yet; report its status
+            conclusion = suite["conclusion"] or suite["status"]
+            if bad_conclusion(conclusion):
+                workflows.append((suite["workflowRun"]["workflow"]["name"], conclusion.lower()))
+        return workflows
+
+    full_names = [repo.full_name for repo in yield_repos()]
+    batches = [full_names[i:i + GRAPHQL_BATCH_SIZE] for i in range(0, len(full_names), GRAPHQL_BATCH_SIZE)]
+    with ThreadPoolExecutor(GRAPHQL_WORKERS) as executor:
+        states = [state for result in executor.map(get_states, batches) for state in result]
+        # repos without any checks have no state and are skipped, as the rest
+        # api path skips workflows that never ran
+        bad = [full_name for full_name, state in zip(full_names, states) if state is not None and bad_state(state)]
+        for full_name, workflows in zip(bad, executor.map(get_bad_workflows, bad)):
+            name = full_name.split("/", 1)[1]
+            for workflow, conclusion in workflows:
+                print(f"{name}: {workflow} {conclusion}")
 
 
 @register_endpoint(
@@ -301,9 +421,16 @@ def runs_show_running() -> None:
     configs=[
         ConfigGithub,
         ConfigAlgo,
+        ConfigRuns,
     ],
 )
 def runs_show_failing() -> None:
+    if ConfigRuns.api == "graphql":
+        graphql_show_runs(
+            lambda state: state in ("FAILURE", "ERROR"),
+            lambda conclusion: conclusion == "FAILURE",
+        )
+        return
     for repo in yield_repos():
         for workflow in repo.get_workflows():
             last_run = next(iter(workflow.get_runs()), None)
@@ -316,9 +443,16 @@ def runs_show_failing() -> None:
     configs=[
         ConfigGithub,
         ConfigAlgo,
+        ConfigRuns,
     ],
 )
 def runs_show_not_success() -> None:
+    if ConfigRuns.api == "graphql":
+        graphql_show_runs(
+            lambda state: state != "SUCCESS",
+            lambda conclusion: conclusion != "SUCCESS",
+        )
+        return
     for repo in yield_repos():
         for workflow in repo.get_workflows():
             last_run = next(iter(workflow.get_runs()), None)
@@ -345,7 +479,6 @@ def pull_all() -> None:
                     [
                         "git",
                         "pull",
-                        # "--tags",
                     ],
                     cwd=folder,
                 )
@@ -365,7 +498,7 @@ def pull_all() -> None:
 
 
 @register_endpoint(
-    description="Pull all projects from github",
+    description="Clone all projects from github",
     configs=[
         ConfigGithub,
         ConfigAlgo,
@@ -378,12 +511,9 @@ def clone_all() -> None:
         project = repo.name
         folder = get_repo_folder(repo)
         all_repo_folders.add(folder)
-        # print(f"considering [{project}] from [{repo.ssh_url}]...")
         if os.path.isfile(folder):
-            # print(f"skipping [{folder}] as it is not to be cloned...")
             continue
         if os.path.isdir(folder):
-            # print(f"skipping [{folder}] as it is already cloned...")
             continue
         if ConfigAlgo.dryrun:
             continue
@@ -397,7 +527,6 @@ def clone_all() -> None:
             ],
             cwd=base_dir,
             stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
         )
     if ConfigAlgo.show_extra:
         disk_repos = get_all_git_repos(base_dir)
@@ -421,7 +550,7 @@ def workflows_run() -> None:
         for workflow in repo.get_workflows():
             print(f"{repo.name}: {workflow.name}...", end="")
             sys.stdout.flush()
-            ret = workflow.create_dispatch(ref="master")
+            ret = workflow.create_dispatch(ref=repo.default_branch)
             print(f"{ret}")
 
 

{pygitpub-0.0.63 → pygitpub-0.0.65}/src/pygitpub/static.py

@@ -1,5 +1,5 @@
 """ version which can be consumed from within the module """
-VERSION_STR = "0.0.63"
+VERSION_STR = "0.0.65"
 DESCRIPTION = "Pygitpub will help you work with github"
 APP_NAME = "pygitpub"
 LOGGER_NAME = "pygitpub"

{pygitpub-0.0.63 → pygitpub-0.0.65}/src/pygitpub/utils/misc.py

@@ -1,25 +1,8 @@
 """ misc.py """
 
 import glob
-import logging
-import os
 import os.path
 
-from pygitpub import LOGGER_NAME
-
-
-def get_logger():
-    return logging.getLogger(LOGGER_NAME)
-
-
-def get_number_of_files(folder: str) -> int:
-    count = 0
-    for _root, _directories, files in os.walk(folder):
-        count += len(files)
-    return count
-
-
-
 
 def get_all_git_repos(base_dir: str) -> set[str]:
     """ we use */.git under base_dir with recursive=False (which is the default)