ghpy 0.2.99__py3-none-any.whl

ghpy/Connections.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from http.client import HTTPSConnection
+from queue import Empty, LifoQueue
+from threading import Semaphore
+from typing import TYPE_CHECKING, BinaryIO
+
+from .PooledResponse import PooledResponse
+
+
+MAX_CONNECTIONS = 4
+
+
+class Connections:
+	def __init__(self, host: str, maxConnections: int = MAX_CONNECTIONS):
+		self.host = host
+		self.connections: LifoQueue[HTTPSConnection] = LifoQueue()
+		self.semaphore = Semaphore(maxConnections)
+
+	def request(self, method: str, url: str, headers: dict[str, str],
+			data: bytes | BinaryIO | None) -> PooledResponse:
+		connection = self._acquire()
+		try:
+			connection.request(method, url, headers=headers, body=data)
+			response = connection.getresponse()
+		except OSError:
+			self._close(connection)
+			raise
+		return PooledResponse(response, self, connection)
+
+	def takeBack(self, connection: HTTPSConnection) -> None:
+		self.connections.put(connection)
+		self.semaphore.release()
+
+	def _acquire(self) -> HTTPSConnection:
+		self.semaphore.acquire()
+		try:
+			return self.connections.get_nowait()
+		except Empty:
+			return HTTPSConnection(self.host)
+
+	def _close(self, connection: HTTPSConnection) -> None:
+		connection.close()
+		self.semaphore.release()

ghpy/PooledResponse.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from http.client import HTTPResponse, HTTPSConnection
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+	from .Connections import Connections
+
+
+class PooledResponse:
+	def __init__(self, response: HTTPResponse, connections: Connections,
+			connection: HTTPSConnection):
+		self.response = response
+		self.connections = connections
+		self.connection = connection
+
+	def __enter__(self) -> "PooledResponse":
+		return self
+
+	def __exit__(self, excType, excValue, traceback) -> None:
+		self.close()
+
+	def close(self) -> None:
+		if not self.connection:
+			return
+		try:
+			self.response.read()
+			self.response.close()
+		finally:
+			self.connections.takeBack(self.connection)
+			self.connection = None
+
+	def read(self, amount: int = -1) -> bytes:
+		return self.response.read(amount)
+
+	def getheader(self, name: str, default: str | None = None) -> str | None:
+		return self.response.getheader(name, default)
+
+	@property
+	def headers(self):
+		return self.response.headers
+
+	@property
+	def reason(self) -> str:
+		return self.response.reason
+
+	@property
+	def status(self) -> int:
+		return self.response.status

ghpy/Release.py

@@ -0,0 +1,149 @@
+import logging
+import math
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any, Callable, Iterable, TypeVar
+from urllib.parse import quote
+
+from .ReleaseAssets import ReleaseAssets
+from .RequestFactory import RequestFactory
+from .errors.AssetNotFoundError import AssetNotFoundError
+from .errors.AssetTimeoutError import AssetTimeoutError
+from .errors.DownloadError import DownloadError
+from .errors.HTTPError import HTTPError
+from .errors.OperationError import OperationError
+from .errors.UploadError import UploadError
+
+
+RELEASE_ASSET_URL = "https://api.github.com/repos/{owner}/{repo}/releases/assets/{id}"
+RELEASE_UPLOAD_URL = "https://uploads.github.com/repos/{owner}/{repo}/releases/{id}/assets"
+MAX_WORKERS = 4
+
+T = TypeVar("T")
+
+logger = logging.getLogger(__name__)
+
+
+def parallel(items: Iterable[T], function: Callable[[T], None]) -> None:
+	failed = False
+	with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+		futures = [executor.submit(function, item) for item in items]
+		for future in as_completed(futures):
+			if error := future.exception():
+				logger.error("%s", error)
+				failed = True
+	if failed:
+		raise OperationError("Operation failed")
+
+
+class Release:
+	def __init__(self, owner: str, repo: str, data: dict[str, Any], token: str,
+			requests: RequestFactory, assets: ReleaseAssets):
+		self.owner = owner
+		self.repo = repo
+		self.id = data["databaseId"]
+		self.tag = data["tagName"]
+		self.isDraft = data["isDraft"]
+		self.isPrivate = data["isPrivate"]
+		self.token = token
+		self.requests = requests
+		self.assets = assets
+
+	def uploadAssets(self, paths: list[Path], clobber: bool = False) -> None:
+		names = [path.name for path in paths]
+		if len(names) != len(set(names)):
+			raise ValueError("Upload files must have unique names")
+
+		assets = self.assets.find([]) if clobber else {}
+		parallel(paths, lambda path: self.uploadAsset(path, assets.get(path.name)))
+
+	def uploadAsset(self, path: Path, asset: dict[str, Any] | None = None) -> None:
+		if asset is not None:
+			self.assets.deleteAsset(asset["id"])
+
+		url = RELEASE_UPLOAD_URL.format(owner=self.owner, repo=self.repo, id=self.id)
+		url += f"?name={quote(path.name)}"
+
+		file = response = None
+		try:
+			file = path.open("rb")
+			headers = {
+				"Authorization": f"Bearer {self.token}",
+				"Accept": "application/vnd.github+json",
+				"Content-Type": "application/octet-stream",
+				"Content-Length": str(path.stat().st_size),
+			}
+			response = self.requests.post(url, headers=headers, data=file)
+		except HTTPError as error:
+			raise UploadError(path.name, error.status) from error
+		finally:
+			file and file.close()
+			response and response.close()
+
+		logger.info("Uploaded %s", path.name)
+
+	def downloadAssets(self, patterns: list[str]) -> None:
+		assets = self.assets.find(patterns)
+		if not assets:
+			raise AssetNotFoundError(patterns)
+
+		parallel(assets.items(), self.downloadAsset)
+
+	def downloadAsset(self, asset: tuple[str, dict[str, Any]]) -> None:
+		name, data = asset
+		url = (
+			RELEASE_ASSET_URL.format(owner=self.owner, repo=self.repo, id=data["id"])
+			if self.isPrivate
+			else data["browser_download_url"]
+		)
+		headers = {
+			"Authorization": f"Bearer {self.token}",
+			"Accept": "application/octet-stream"
+		}
+
+		file = response = None
+		try:
+			response = self.requests.get(url, headers=headers)
+			file = Path(name).open("wb")
+			while chunk := response.read(1024 * 1024):
+				file.write(chunk)
+		except HTTPError as error:
+			raise DownloadError(name, error.status) from error
+		finally:
+			file and file.close()
+			response and response.close()
+
+		logger.info("Downloaded %s", name)
+
+	def awaitAssets(self, names: list[str]) -> None:
+		deadline = time.monotonic() + self.requests.timeout
+		parallel(self.assets.awaitAssets(names, deadline),
+				lambda asset: self.awaitAsset(asset, deadline))
+
+	def awaitAsset(self, asset: tuple[str, dict[str, Any]],
+			deadline: float) -> None:
+		name, data = asset
+		logger.info("Waiting for %s", name)
+		url = (
+			RELEASE_ASSET_URL.format(owner=self.owner, repo=self.repo, id=data["id"])
+			if self.isPrivate else data["browser_download_url"]
+		)
+		headers = {
+			"Authorization": f"Bearer {self.token}",
+			"Accept": "application/octet-stream",
+		}
+		remaining = math.ceil(deadline - time.monotonic())
+		if remaining <= 0:
+			raise AssetTimeoutError(name)
+
+		response = None
+		try:
+			response = self.requests.get(url, headers=headers, retryStatuses={404},
+					timeout=remaining)
+		except TimeoutError as error:
+			raise AssetTimeoutError(name) from error
+		finally:
+			response and response.close()
+
+		logger.info("Available %s", name)

ghpy/ReleaseAssets.py

@@ -0,0 +1,80 @@
+import json
+import logging
+import time
+
+from typing import Any, Iterator
+
+from fnmatch import fnmatch
+from .errors.AssetTimeoutError import AssetTimeoutError
+from .RequestFactory import RequestFactory
+
+PAGE_SIZE = 100
+MAX_ASSETS = 1000
+MAX_PAGES = MAX_ASSETS // PAGE_SIZE
+
+RELEASE_ASSETS_URL = "https://api.github.com/repos/{owner}/{repo}/releases/{id}/assets"
+RELEASE_ASSET_URL = "https://api.github.com/repos/{owner}/{repo}/releases/assets/{id}"
+
+logger = logging.getLogger(__name__)
+
+
+class ReleaseAssets:
+	def __init__(self, requests: RequestFactory, token: str, owner: str, repo: str,
+			releaseId: int):
+		self.owner = owner
+		self.repo = repo
+		self.releaseId = releaseId
+		self.token = token
+		self.requests = requests
+
+	def find(self, patterns: list[str]) -> dict[str, Any]:
+		assets: dict[str, Any] = {}
+		headers = {
+			"Authorization": f"Bearer {self.token}",
+			"Accept": "application/vnd.github+json",
+		}
+
+		for page in range(1, MAX_PAGES + 1):
+			url = RELEASE_ASSETS_URL.format(owner=self.owner, repo=self.repo,
+					id=self.releaseId) + f"?per_page={PAGE_SIZE}&page={page}"
+			with self.requests.get(url, headers=headers) as response:
+				pageAssets = json.load(response)
+				assets.update({asset["name"]: asset for asset in pageAssets
+					if not patterns or any(
+						fnmatch(asset["name"], pattern) for pattern in patterns
+					)})
+			if len(pageAssets) < PAGE_SIZE:
+				break
+
+		return assets
+
+	def awaitAssets(self, names: list[str],
+			deadline: float) -> Iterator[tuple[str, dict[str, Any]]]:
+		pending = set(names)
+
+		for retry in range(self.requests.maxRetries + 1):
+			assets = self.find(list(pending))
+			for name, asset in assets.items():
+				if name not in pending:
+					continue
+				pending.remove(name)
+				yield name, asset
+			if not pending:
+				return
+			if retry >= self.requests.maxRetries or time.monotonic() >= deadline:
+				break
+			delay = min(self.requests.delay + retry * self.requests.increment,
+				self.requests.maxDelay)
+			time.sleep(delay)
+
+		raise AssetTimeoutError(list(pending))
+
+	def deleteAsset(self, assetId: int) -> None:
+		url = RELEASE_ASSET_URL.format(owner=self.owner, repo=self.repo, id=assetId)
+		headers = {
+			"Authorization": f"Bearer {self.token}",
+			"Accept": "application/vnd.github+json",
+		}
+		with self.requests.delete(url, headers=headers):
+			pass
+

ghpy/Releases.py

@@ -0,0 +1,54 @@
+import json
+
+from .Release import Release
+from .ReleaseAssets import ReleaseAssets
+from .RequestFactory import RequestFactory
+from .errors.GraphQLError import GraphQLError
+from .errors.ReleaseNotFoundError import ReleaseNotFoundError
+
+
+GET_RELEASE_QUERY = """
+query($owner: String!, $repo: String!, $tag: String!) {
+	repository(owner: $owner, name: $repo) {
+		isPrivate
+		release(tagName: $tag) {
+			databaseId
+			tagName
+			isDraft
+		}
+	}
+}
+"""
+
+
+class Releases:
+	def __init__(self, token: str, requests: RequestFactory):
+		self.token = token
+		self.requests = requests
+
+	def getRelease(self, owner: str, repo: str, tag: str) -> Release:
+		headers = {
+			"Authorization": f"Bearer {self.token}",
+			"Content-Type": "application/json",
+		}
+		data = json.dumps({"query": GET_RELEASE_QUERY,
+			"variables": {"owner": owner, "repo": repo, "tag": tag}
+		}).encode()
+
+		with self.requests.post("https://api.github.com/graphql", headers=headers,
+				data=data) as response:
+			body = response.read()
+			result = json.loads(body)
+
+		if "errors" in result:
+			raise GraphQLError(result)
+
+		repository = result["data"]["repository"]
+		releaseData = repository["release"]
+		if releaseData is None:
+			raise ReleaseNotFoundError(tag)
+
+		releaseData["isPrivate"] = repository["isPrivate"]
+		releaseId = releaseData["databaseId"]
+		assets = ReleaseAssets(self.requests, self.token, owner, repo, releaseId)
+		return Release(owner, repo, releaseData, self.token, self.requests, assets)

ghpy/Request.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import socket
+import time
+
+from typing import TYPE_CHECKING
+from urllib.parse import urljoin
+
+from .PooledResponse import PooledResponse
+from .errors.HTTPError import HTTPError
+from .errors.InvalidRedirectError import InvalidRedirectError
+from .errors.TooManyRedirectsError import TooManyRedirectsError
+from .netlib import sameOrigin, stripHeaders
+
+from typing import BinaryIO
+
+if TYPE_CHECKING:
+	from RequestFactory import RequestFactory
+
+
+MAX_CONNECTION_ERRORS = 3
+MAX_DELAY = 300
+MAX_REDIRECTS = 5
+MAX_RETRIES = 8
+RETRY_DELAY = 1
+RETRY_INCREMENT = 1
+
+REDIRECT_PRESERVE_METHOD = {307, 308}
+REDIRECT_TO_GET = {301, 302, 303}
+REDIRECT_STATUSES = REDIRECT_PRESERVE_METHOD | REDIRECT_TO_GET
+RETRY_STATUSES = {408, 425, 429, 500, 502, 503, 504}
+
+SENSITIVE_HEADERS = {"authorization", "proxy-authorization"}
+
+
+class Request:
+	def __init__(self, requests: RequestFactory, method: str, url: str,
+			headers: dict[str, str], data: bytes | BinaryIO | None, timeout: int = 30,
+			maxRetries: int = MAX_RETRIES, delay: int = RETRY_DELAY,
+			increment: int = RETRY_INCREMENT, maxDelay: int = MAX_DELAY,
+			retryStatuses: set[int] | None = None):
+		self.requests = requests
+		self.method = method
+		self.url = url
+		self.headers = headers
+		self.data = data
+		self.timeout = timeout
+		self.maxRetries = maxRetries
+		self.delay = delay
+		self.increment = increment
+		self.maxDelay = maxDelay
+		self.retryStatuses = RETRY_STATUSES | (retryStatuses or set())
+
+		self.retry = 0
+		self.connectionErrors = 0
+		self.redirects = 0
+		self.spent = 0
+
+	def execute(self) -> PooledResponse:
+		while True:
+			try:
+				connections = self.requests.getConnections(self.url)
+				if hasattr(self.data, "seek"):
+					self.data.seek(0)
+				response = connections.request(self.method, self.url,
+						self.headers, self.data)
+				self.connectionErrors = 0
+				if self._redirect(response):
+					continue
+				if not self._shouldRetry(response):
+					if response.status >= 400:
+						raise self._httpError(response)
+					return response
+				response.close()
+				self._wait(response.headers)
+			except OSError as error:
+				self.connectionErrors += 1
+				if not self._couldRetry(error):
+					raise
+				self._wait()
+
+	def _redirect(self, response: PooledResponse):
+		if response.status not in REDIRECT_STATUSES:
+			return False
+		response.close()
+
+		location = response.headers.get("Location")
+		if not location:
+			raise InvalidRedirectError(self.url, response.status, response.reason)
+		if self.redirects >= MAX_REDIRECTS:
+			raise TooManyRedirectsError(self.url, response.status, response.reason)
+
+		url = self.url
+		self.url = urljoin(self.url, location)
+		if not sameOrigin(url, self.url):
+			self.headers = stripHeaders(self.headers, SENSITIVE_HEADERS)
+		if response.status in REDIRECT_TO_GET:
+			self.method = "GET"
+			self.headers.pop("Content-Length", None)
+			self.headers.pop("Content-Type", None)
+			self.data = None
+		self.redirects += 1
+		return True
+
+	def _shouldRetry(self, response: PooledResponse):
+		return response.status in self.retryStatuses or (
+			response.status == 403 and (
+				response.headers.get("Retry-After") or
+				response.headers.get("X-RateLimit-Remaining") == "0"
+			)
+		)
+
+	def _couldRetry(self, error: OSError):
+		return isinstance(error, (BrokenPipeError, ConnectionRefusedError,
+			ConnectionResetError, socket.gaierror)) and \
+			self.connectionErrors < MAX_CONNECTION_ERRORS
+
+	def _wait(self, headers: dict[str, str] | None = None):
+		if self.retry >= self.maxRetries:
+			raise TimeoutError(f"Timed out after {self.retry} retries")
+
+		delay = min(self.delay + self.retry * self.increment, self.maxDelay)
+		if headers:
+			delay = max(delay, int(headers.get("Retry-After", "0")))
+			if headers.get("X-RateLimit-Remaining") == "0":
+				reset = int(headers.get("X-RateLimit-Reset", "0"))
+				delay = max(delay, reset - int(time.time()))
+
+		if self.spent + delay > self.timeout:
+			raise TimeoutError(f"Timed out after {self.retry} retries")
+
+		self.spent += delay
+		self.retry += 1
+		time.sleep(delay)
+
+	def _httpError(self, response: PooledResponse) -> HTTPError:
+		try:
+			body = response.read()
+			return HTTPError(self.url, response.status, response.reason,
+					response.headers, body)
+		finally:
+			response.close()

ghpy/RequestFactory.py

@@ -0,0 +1,53 @@
+from threading import Lock
+from typing import BinaryIO
+from urllib.parse import urlsplit
+
+from .Connections import Connections
+from .PooledResponse import PooledResponse
+from .Request import MAX_DELAY, MAX_RETRIES, RETRY_DELAY, RETRY_INCREMENT, Request
+
+DEFAULT_HEADERS = {"User-Agent": "ghpy"}
+
+
+class RequestFactory:
+	def __init__(self, timeout: int = 30, maxRetries: int = MAX_RETRIES,
+			delay: int = RETRY_DELAY, increment: int = RETRY_INCREMENT,
+			maxDelay: int = MAX_DELAY):
+		self.timeout = timeout
+		self.maxRetries = maxRetries
+		self.delay = delay
+		self.increment = increment
+		self.maxDelay = maxDelay
+		self.connections: dict[str, Connections] = {}
+		self.connectionsLock = Lock()
+
+	def get(self, url: str, headers: dict[str, str] | None = None,
+			retryStatuses: set[int] | None = None,
+			timeout: int | None = None) -> PooledResponse:
+		return self._request("GET", url, headers, retryStatuses=retryStatuses,
+				timeout=timeout)
+
+	def post(self, url: str, headers: dict[str, str] | None = None,
+			data: bytes | BinaryIO | None = None,
+			retryStatuses: set[int] | None = None) -> PooledResponse:
+		return self._request("POST", url, headers, data, retryStatuses)
+
+	def delete(self, url: str, headers: dict[str, str] | None = None,
+			retryStatuses: set[int] | None = None) -> PooledResponse:
+		return self._request("DELETE", url, headers, retryStatuses=retryStatuses)
+
+	def _request(self, method: str, url: str, headers: dict[str, str] | None = None,
+			data: bytes | BinaryIO | None = None, retryStatuses: set[int] | None = None,
+			timeout: int | None = None) -> PooledResponse:
+		requestHeaders = {**DEFAULT_HEADERS, **(headers or {})}
+		return Request(self, method, url, requestHeaders, data,
+				self.timeout if timeout is None else timeout,
+				self.maxRetries, self.delay, self.increment, self.maxDelay,
+				retryStatuses).execute()
+
+	def getConnections(self, url: str) -> Connections:
+		host = urlsplit(url).netloc
+		with self.connectionsLock:
+			if host not in self.connections:
+				self.connections[host] = Connections(host)
+			return self.connections[host]

ghpy/converters.py

@@ -0,0 +1,8 @@
+import argparse
+
+
+def repository(value: str) -> tuple[str, str]:
+	parts = value.split("/")
+	if len(parts) != 2 or not all(parts):
+		raise argparse.ArgumentTypeError("must be in OWNER/REPO format")
+	return tuple(parts)

ghpy/errors/AssetNotFoundError.py

@@ -0,0 +1,7 @@
+from .GhpyError import GhpyError
+
+
+class AssetNotFoundError(GhpyError):
+	def __init__(self, patterns: list[str]):
+		self.patterns = patterns
+		super().__init__(f"No assets found for: {', '.join(patterns)}")

ghpy/errors/AssetTimeoutError.py

@@ -0,0 +1,12 @@
+from .GhpyError import GhpyError
+
+
+class AssetTimeoutError(GhpyError):
+	def __init__(self, names: str | list[str], status: int = 0):
+		if isinstance(names, str):
+			names = [names]
+		self.names = names
+		message = "Timed out waiting for assets"
+		if status:
+			message += f" (HTTP {status})"
+		super().__init__(f"{message}: {', '.join(names)}")

ghpy/errors/DownloadError.py

@@ -0,0 +1,8 @@
+from .GhpyError import GhpyError
+
+
+class DownloadError(Exception):
+	def __init__(self, name: str, status: int):
+		self.name = name
+		self.status = status
+		super().__init__(f"Failed to download {name} (HTTP {status})")

ghpy/errors/GhpyError.py

@@ -0,0 +1,2 @@
+class GhpyError(Exception):
+	pass

ghpy/errors/GraphQLError.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from .GhpyError import GhpyError
+
+
+class GraphQLError(GhpyError):
+	def __init__(self, response: dict):
+		self.response = response
+		messages = [error["message"] for error in response["errors"]]
+		super().__init__(", ".join(messages))

ghpy/errors/HTTPError.py

@@ -0,0 +1,13 @@
+from http.client import HTTPMessage
+
+
+class HTTPError(Exception):
+	def __init__(self, url: str, status: int, reason: str,
+			headers: HTTPMessage, body: bytes):
+		self.url = url
+		self.status = status
+		self.code = status
+		self.reason = reason
+		self.headers = headers
+		self.body = body
+		super().__init__(f"HTTP {status} {reason}")

ghpy/errors/InvalidRedirectError.py

@@ -0,0 +1,6 @@
+from .RedirectError import RedirectError
+
+
+class InvalidRedirectError(RedirectError):
+	def __init__(self, url: str, status: int, reason: str):
+		super().__init__(url, status, reason, "Location header not found")

ghpy/errors/OperationError.py

@@ -0,0 +1,5 @@
+from .GhpyError import GhpyError
+
+
+class OperationError(GhpyError):
+	pass

ghpy/errors/RedirectError.py

@@ -0,0 +1,9 @@
+from .GhpyError import GhpyError
+
+
+class RedirectError(GhpyError):
+	def __init__(self, url: str, status: int, reason: str, message: str):
+		self.url = url
+		self.status = status
+		self.reason = reason
+		super().__init__(message)

ghpy/errors/ReleaseNotFoundError.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .GhpyError import GhpyError
+
+
+class ReleaseNotFoundError(GhpyError):
+	def __init__(self, tag: str):
+		self.tag = tag
+		super().__init__(f"Release {tag} not found")

ghpy/errors/TooManyRedirectsError.py

@@ -0,0 +1,6 @@
+from .RedirectError import RedirectError
+
+
+class TooManyRedirectsError(RedirectError):
+	def __init__(self, url: str, status: int, reason: str):
+		super().__init__(url, status, reason, "Too many redirects")