rgit 0.0.1__py3-none-any.whl

rgit/__init__.py

@@ -0,0 +1 @@
+from . import cli as _cli

rgit/__main__.py

@@ -0,0 +1,17 @@
+import sys, asyncio
+from . import cli
+
+
+def _smain(args):
+	asyncio.run(cli.main(args))
+
+
+def _ssmain():
+	try:
+		sys.exit(_smain(sys.argv[1:]))
+	except KeyboardInterrupt:
+		sys.stderr.write("\n")
+
+
+if __name__ == "__main__":
+	_ssmain()

rgit/cli/__init__.py

@@ -0,0 +1,74 @@
+import argparse, pathlib, sys
+from .. import configuration, constants
+from . import registry, scan, status, ignored
+
+
+async def main(args):
+	opts = _parse_args(args=args)
+	config_path = find_config_file(opts)
+	config = await configuration.load(config_file_path=config_path)
+	handler = registry.get_command_handler(opts.command)
+	if handler is not None:
+		handler_instance = handler()
+		await handler_instance.execute(opts=opts, config=config)
+	else:
+		print("external commands are not supported yet")
+
+
+def find_config_file(opts):
+	if opts.config_path:
+		return pathlib.Path(opts.config_path)
+
+	candidates = [
+		(pathlib.Path.home() / ("." + constants.SELF_NAME + ".json")),
+		(pathlib.Path.home() / ".config" / constants.SELF_NAME / "config.json"),
+	]
+
+	for c in candidates:
+		if c.exists():
+			return c
+
+	return None
+
+
+def _parse_args(args=None):
+	parser = argparse.ArgumentParser(
+		prog=constants.SELF_NAME,
+		description=None,
+		epilog=None
+	)
+
+	parser.add_argument(
+		"--config-path",
+		dest="config_path",
+		action="store",
+		metavar="PATH",
+		default=None,
+	)
+
+	parser.add_argument(
+		"--dont-show-progress",
+		dest="show_progress",
+		action="store_false",
+		default=sys.stderr.isatty(),
+		help="do not display display intermediary messages while executing",
+	)
+
+	subparsers = parser.add_subparsers(
+		title=None,
+		dest="command",
+		metavar="COMMAND",
+	)
+
+	for name, aliases, handler, disabled in registry.enumerate_command_handlers():
+		if disabled:
+			continue
+		handler.define_arguments(subparsers.add_parser(
+			name,
+			aliases=aliases,
+			help=handler.short_description()
+		))
+
+	opts = parser.parse_args(args)
+
+	return opts

rgit/cli/ignored.py

@@ -0,0 +1,239 @@
+import re, os, sys, pathlib
+from .registry import command
+from .. import git
+from ..tools import is_path_in, path_relative_to_or_unchanged, strict_int, add_status_msg, set_status_msg, draw_table
+
+
+@command("ignored")
+class Ignored(object):
+	@classmethod
+	def define_arguments(cls, parser):
+		parser.add_argument(
+			"--format", "-f",
+			dest="format",
+			action="store",
+			choices=["table", "json"],
+			default="table",
+		)
+		parser.add_argument(
+			"--group", "-g",
+			dest="groups",
+			action="append",
+			default=[],
+			help=(
+				"case-insensitive name of ignore group as defined in .gitignore files by putting "
+				"the group name immediately after \"#{{{\" which will include every pattern "
+				"until \"#}}}\""
+			),
+		)
+		parser.add_argument(
+			"--not-in-group", "-x",
+			dest="not_in_groups",
+			action="append",
+			default=[],
+			help="",
+		)
+		parser.add_argument(
+			"--list", "-l",
+			dest="show_lists_only",
+			action="append",
+			metavar="NAME",
+			default=None,
+			choices=["groups", "sources", "files"],
+			help="only show the specified lists"
+		)
+		parser.add_argument(
+			"folders",
+			nargs="*",
+			metavar="FOLDER",
+			help="only inspect repositories worktrees of which are in one of specified folders"
+		)
+	@classmethod
+	def short_description(cls):
+		return "show and manipulate ignored files in repositories"
+
+	def __init__(self):
+		self._config = None
+
+	async def execute(self, *, opts, config):
+		self._config = config
+
+		opts.folders = [pathlib.Path(f).resolve() for f in opts.folders]
+		opts.groups = set(opts.groups)
+		opts.not_in_groups = set(opts.not_in_groups)
+
+		ignore_group_reader = IgnoreGroupReader()
+
+		results = {}
+		for repo in self._config.repositories:
+			if await git.is_bare(repo):
+				add_status_msg(".")
+				continue
+
+			worktree_path = await git.toplevel(repo)
+			worktree_fspath = os.fspath(worktree_path)
+
+			if opts.folders and not any(is_path_in(f, worktree_path) for f in opts.folders):
+				add_status_msg("-")
+				continue
+
+			add_status_msg("*")
+
+			ignored_files = [
+				path
+				for _, path in filter(
+					lambda x: x[0] == "ignored",
+					await git.status(repo, "--ignored=matching")
+				)
+			]
+			if not ignored_files:
+				continue
+			stdout = await git.git(repo,
+				"check-ignore", "-z", "--verbose", "--non-matching", "--stdin",
+				stdin="\0".join(ignored_files),
+				returncode_ok=lambda returncode: returncode in (0, 1),
+				worktree=git.TOPLEVEL,
+				# Ignored files are reported relative to repo work-tree, which check-ignore will
+				# resolve using the current folder, so it must be the work-tree.
+				cwd=git.WORKTREE,
+			)
+			stdout = stdout.split("\0")
+			assert len(stdout) == 4 * len(ignored_files) + 1, (len(ignored_files), len(stdout))
+			assert stdout.pop(-1) == ""
+			ignored = map(lambda i: stdout[(i*4):(i*4+4)], range(len(stdout) // 4))
+
+			for ignore_file, ignore_file_line, ignore_pattern, path in ignored:
+				ignore_file = None if ignore_file == "" else worktree_path / ignore_file
+				ignore_file_line = None if ignore_file_line == "" else strict_int(ignore_file_line)
+				assert (ignore_file is None) == (ignore_file_line is None)
+				groups = ignore_group_reader.get_groups(ignore_file, ignore_file_line)
+				if groups is None:
+					group = "<failed to identify matching ignore pattern>"
+				elif len(groups) < 1:
+					group = "-"
+				else:
+					assert len(groups) <= 1
+					group = groups[0]
+				if opts.groups and group not in opts.groups:
+					continue
+				if group in opts.not_in_groups:
+					continue
+				ignore_file_fspath = os.fspath(ignore_file) if ignore_file is not None else None
+				results.setdefault(group, {}).setdefault(worktree_fspath, {})[path] = [
+					ignore_file_fspath, ignore_file_line, ignore_pattern
+				]
+		set_status_msg(None)
+
+		if opts.show_lists_only is not None:
+			lists_to_show = set(opts.show_lists_only)
+			at_least_one_list_shown = False
+			def show_list(thelist, *, sort=True):
+				if sort:
+					thelist = sorted(thelist)
+				nonlocal at_least_one_list_shown
+				if at_least_one_list_shown:
+					sys.stdout.write("\n")
+				else:
+					at_least_one_list_shown = True
+				if opts.format == "json":
+					import json
+					json.dump(thelist, sys.stdout, indent="\t")
+					sys.stdout.write("\n")
+				else:
+					# TODO If the file name contains unprintable chars (like "Icon\r") this will produce an incorrect list. Consider adding an escape flavor like "python", "shell", "c", etc.
+					sys.stdout.write("\n".join(thelist))
+					if thelist:
+						sys.stdout.write("\n")
+
+			if "groups" in lists_to_show:
+				lists_to_show.remove("groups")
+				show_list(list(results.keys()))
+
+			if "sources" in lists_to_show:
+				lists_to_show.remove("sources")
+				thelist = set()
+				for group in results.values():
+					for workdir in group.values():
+						for file in workdir.values():
+							thelist.add(f"{file[0]}:{file[1]}")
+				show_list(map(str, thelist))
+
+			if "files" in lists_to_show:
+				lists_to_show.remove("files")
+				thelist = set()
+				for group in results.values():
+					for workdir_path, workdir in group.items():
+						for file in workdir.keys():
+							thelist.add(os.path.join(workdir_path, file))
+				show_list(thelist)
+
+			if lists_to_show:
+				raise ValueError(f"unsupported lists {lists_to_show}")
+		else:
+			if opts.format == "json":
+				import json
+				fo = sys.stdout
+				json.dump(results, fo, indent="\t")
+				fo.write("\n")
+				fo.flush()
+			elif opts.format == "table":
+				for group, repos in results.items():
+					rows = [
+						["Work-tree Path", "File Path", "Source", "Pattern"]
+					]
+					for path, files in repos.items():
+						for file, (ignore_path, ignore_line, ignore_pattern) in files.items():
+							if file != repr(file)[1:-1]:
+								file = repr(file)
+							if ignore_path is not None:
+								ignore_path_relative = path_relative_to_or_unchanged(path, ignore_path)
+								ignore_rule_location = f"{ignore_path_relative}:{ignore_line}"
+							else:
+								ignore_rule_location = "<unknown>"
+							rows.append([
+								path, file, ignore_rule_location, ignore_pattern
+							])
+					draw_table(
+						rows,
+						title=group,
+						has_header=True,
+						fo=sys.stdout,
+					)
+			else:
+				raise ValueError(f"unsupported output format {repr(opts.format)}")
+
+
+class IgnoreGroupReader(object):
+	def __init__(self):
+		self._cache = {}
+
+	def get_groups(self, path, lineno):
+		if path is None or lineno is None:
+			return None
+		if path not in self._cache:
+			self._cache[path] = self._read_ignore_file(path)
+		ignore_file_map = self._cache[path]
+		assert 0 <= (lineno - 1) <= len(ignore_file_map), (lineno, len(ignore_file_map))
+		return ignore_file_map[lineno - 1]
+
+	@staticmethod
+	def _read_ignore_file(path):
+		result = []
+		start_marker = "#{{{"
+		end_marker = "#}}}"
+		group_name_pattern = re.compile(r"^[a-z0-9_]+$")
+		group_stack = []
+		with path.open("r") as fo:
+			for line_num, line in enumerate(fo, start=1):
+				if line.startswith(start_marker):
+					# TODO Instead of converting to lowercase, use case-insensitive mapping and report any inconsistencies.
+					group = line[len(start_marker):].strip().lower()
+					if not group_name_pattern.match(group):
+						raise ValueError(f"invalid group name - {repr(group)} in {repr(os.fspath(path))}")
+					group_stack.append(group)
+				elif line.startswith(end_marker):
+					group_stack.pop(-1)
+				if len(group_stack) > 1:
+					raise ValueError(f"nested groups are not supported: {os.fspath(path)}:{line_num}")
+				result.append(list(set(group_stack)))
+		return result

rgit/cli/registry.py

@@ -0,0 +1,22 @@
+_command_handlers = {}
+
+
+def command(*names, disabled=False):
+	def decorator(handler):
+		name = names[0]
+		if name in _command_handlers:
+			raise ValueError("command already registered")
+		_command_handlers[name] = (names[1:], handler, disabled)
+		return handler
+	return decorator
+
+
+def enumerate_command_handlers():
+	for name, (aliases, handler, disabled) in _command_handlers.items():
+		yield (name, aliases, handler, disabled)
+
+
+def get_command_handler(name):
+	if name not in _command_handlers:
+		return None
+	return _command_handlers[name][1]

rgit/cli/scan.py

@@ -0,0 +1,341 @@
+import os, sys, pathlib, collections, functools
+
+from ..tools import set_status_msg, add_status_msg
+from .registry import command
+from .. import git
+
+# pip install -U PyYAML>=6.0.1,<7.0.0
+import yaml
+
+
+@command("scan")
+class Scan(object):
+	@classmethod
+	def define_arguments(cls, parser):
+		parser.add_argument(
+			"starting_folders",
+			metavar="PATH",
+			nargs="*",
+			help="starting folders from where to conduct the search",
+		)
+		parser.add_argument(
+			"--output", "-o",
+			dest="output_path",
+			metavar="PATH",
+			action="store",
+			type=pathlib.Path,
+			default=None,
+			help="besides outputting found repositories on stdout, also write a YAML document at given path"
+		)
+		parser.add_argument(
+			"--ignore", "-i",
+			dest="scan_folders_ignore",
+			metavar="PATH",
+			action="append",
+			type=pathlib.Path,
+			default=[],
+			help="do not scan specified folders and their sub-folders",
+		)
+		parser.add_argument(
+			"--show-all",
+			dest="show_all",
+			action="store_true",
+			default=False,
+			help="report all found repositories, even those already added to configuration"
+		)
+		parser.add_argument(
+			"--skip-gitdirs",
+			dest="skip_gitdirs",
+			action="store_true",
+			default=False,
+			help="do not traverse into gitdirs of discovered repos"
+		)
+		parser.add_argument(
+			"--skip-worktrees",
+			dest="skip_worktrees",
+			action="store_true",
+			default=False,
+			help="do not traverse into worktrees of discovered repos"
+		)
+
+	@classmethod
+	def short_description(cls):
+		return "walk the filesystem to discover new repositories"
+
+	def __init__(self):
+		pass
+
+	async def execute(self, *, opts, config):
+		dirs_to_skip = {
+			*(opts.scan_folders_ignore),
+			*(config.scan_folders_ignore)
+		}
+		scan_folders = opts.starting_folders or list(config.scan_folders)
+		repositories_to_skip = set(config.repositories) if not opts.show_all else set()
+		repositories_found = set()
+		counter = 0
+
+		def should_ignore(gitdir, worktree=None):
+			"""Check if a repository should be ignored based on its gitdir or worktree location.
+
+			Args:
+				gitdir: Path to the git directory (.git folder)
+				worktree: Optional path to the worktree (None for bare repos or when not yet determined)
+
+			Returns:
+				True if the repository should be ignored, False otherwise
+
+			Note: We check both gitdir and worktree because repos can have custom worktrees
+			configured via core.worktree in git config. A repo might be in a non-ignored location
+			but have its worktree in an ignored folder, or vice versa.
+			"""
+			for ignored_folder in dirs_to_skip:
+				if gitdir.is_relative_to(ignored_folder):
+					return True
+				if worktree and worktree.is_relative_to(ignored_folder):
+					return True
+			return False
+
+		for starting_folder in scan_folders:
+			for root, dirs, files in os.walk(starting_folder, topdown=True):
+				root = pathlib.Path(root)
+				dirs.sort(key=lambda x: x.lower())
+
+				counter += 1
+				if counter >= 100:
+					if opts.show_progress:
+						add_status_msg(".")
+					counter = 0
+
+				repo = None
+
+				# Note that if traversing into gitdir-s, a regular non-bare repo will be detected
+				# twice - once for the worktree and once for the .git folder. Duplicates will be
+				# collapsed thought via the `repositories_found` set object.
+
+				if ".git" in dirs or ".git" in files:
+					# If the gitdir is ignored, there is no point in checking the worktree. The
+					# worktree could be overridden, but if this folder containing .git is ignored,
+					# the whole thing is considered ignored.
+					if should_ignore(root / ".git", worktree=root):
+						continue
+
+					try:
+						repo = git.Repo(gitdir=(root / ".git"), worktree=root)
+					except ValueError:
+						# Skip invalid git directories (e.g., corrupted or incomplete .git folders)
+						gitpath = root / ".git"
+						if opts.show_progress:
+							set_status_msg(None)
+						sys.stderr.write(f"WARNING: {gitpath.as_posix()!r} - invalid git repository\n")
+						repo = None
+					if opts.skip_gitdirs:
+						try:
+							dirs.remove(".git")
+						except ValueError:
+							pass
+					if opts.skip_worktrees:
+						re_add_gitdir = ".git" in dirs
+						del dirs[:]
+						if re_add_gitdir:
+							dirs.append(".git")
+				elif "refs" in dirs and "objects" in dirs and "HEAD" in files:
+					#TODO:bug: A repo might be set up such that the objects directory is elsewhere,
+					#  e.g. via GIT_OBJECT_DIRECTORY.
+					#  See https://github.com/git/git/blob/v2.42.0/setup.c#L345-L355
+
+					if should_ignore(root):
+						continue
+
+					try:
+						repo = git.Repo(gitdir=root)
+					except ValueError:
+						# Skip invalid git directories
+						gitpath = root
+						add_status_msg(f"WARNING: {gitpath.as_posix()} - invalid git repository\n")
+						repo = None
+					if opts.skip_gitdirs:
+						del dirs[:]
+
+				if repo is None:
+					continue
+
+				if repo in repositories_found:
+					continue
+
+				if should_ignore(repo.gitdir, repo.worktree):
+					continue
+
+				repositories_found.add(repo)
+				if opts.show_progress:
+					set_status_msg(None)
+				if repo.gitdir not in repositories_to_skip:
+					await self.report_new_repo(repo)
+
+		if opts.show_progress:
+			set_status_msg(None)
+
+		if opts.output_path is not None:
+			await self.write_output_yaml(repositories_found, opts.output_path)
+
+	async def write_output_yaml(self, repositories_found, output_path):
+		#TODO:cleanup: Refactor this function to extract the logic into methods of the git.Repo class.
+
+		# We only consider a config to be common is more than half has the same value.
+		config_yaml_common_config_threshold = len(repositories_found) // 2
+		# The gitdir and worktree paths are relative to user's home folder.
+		config_yaml_relative_to = pathlib.Path.home()
+		def make_path_relative(path):
+			try:
+				return path.relative_to(config_yaml_relative_to)
+			except ValueError:
+				return path
+		# This is the resulting object that will be serialized into the output YAML document.
+		config_yaml = {
+			# Initially this will hold {"git-config-key": {"git-config-value": count}}, but later
+			# Will be changed to be {"git-config-key": "git-config-value"} where the value is the
+			# one with with the highest count if it is above the threshold.
+			"config": (config_yaml_common_config := collections.defaultdict(
+				functools.partial(collections.defaultdict, int)
+			)),
+			"repositories": (config_yaml_repositories := [])
+		}
+		for repo in sorted(repositories_found, key=lambda r: r.gitdir):
+			config_yaml_repositories.append(
+				#TODO:yaml: Change this to collections.OrderedDict() and make PyYAML to render it as a regular dict.
+				repo_entry := {}
+			)
+
+			repo_entry["gitdir"] = make_path_relative(repo.gitdir).as_posix()
+			#TODO:yaml: The gitdir path above and worktree paths below should be added as pathlib.Path objects and rendered by the YAML library in quotes and unbroken.
+			if repo.worktree and repo.is_worktree_custom():
+				#TODO:bug: Handle bare repos explicitly
+				repo_entry["worktrees"] = {
+					#TODO:worktrees: Think through the schema of the worktrees sub-object.
+					make_path_relative(repo.worktree).as_posix(): None
+				}
+				#TODO:worktrees: List all the added worktrees along with their custom configurations.
+
+
+			repo_entry["remotes"] = {}
+			repo_entry["branches"] = {}
+			repo_entry["config"] = {}
+
+			async for c in git.list_config(repo.gitdir):
+				key, sep, value = c.partition("\n")
+				assert sep == "\n", c
+
+				# Special Cases
+				if key == "core.worktree":
+					assert (repo.gitdir / pathlib.Path(value)).samefile(repo.worktree)
+					continue
+
+				if key.startswith("remote."):
+					section, remote_name, name = git.split_config_key(key)
+					assert section == "remote", (key, section, remote_name, name)
+					if remote_name:
+						if name == "url":
+							repo_entry["remotes"][remote_name] = value
+							continue
+
+				if key.startswith("branch."):
+					section, branch_name, name = git.split_config_key(key)
+					assert section == "branch", (key, section, branch_name, name)
+					if branch_name:
+						repo_entry["branches"].setdefault(branch_name, [None, None])
+						if name == "remote":
+							repo_entry["branches"][branch_name][0] = value
+							continue
+
+						if name == "merge":
+							repo_entry["branches"][branch_name][1] = value
+							continue
+
+				repo_entry["config"][key] = single_value_or_tuple(repo_entry["config"].get(key), value)
+
+			remote_fetch_keys = [
+				k for k in repo_entry["config"].keys()
+					if k.startswith("remote.") and k.endswith(".fetch")
+			]
+			for key in remote_fetch_keys:
+				section, remote_name, name = git.split_config_key(key)
+				assert section == "remote" and name == "fetch", (key, section, remote_name, name)
+				value = repo_entry["config"][key]
+
+				# Instead of skipping this, decompose the string to components and compare.
+				if value == f"+refs/heads/*:refs/remotes/{remote_name}/*":
+					del repo_entry["config"][key]
+
+					# In case the "remote.<name>.fetch" is either alone or came first
+					repo_entry["remotes"].setdefault(remote_name, None)
+
+			#TODO:yaml: Instead of relying on dict preserving key order, the yaml renderer should sort these keys.
+			repo_entry["remotes"] = dict(sorted(repo_entry["remotes"].items(), reverse=True))
+			if not len(repo_entry["remotes"]):
+				del repo_entry["remotes"]
+
+			# Updating the common config values as a separate loop to accommodate multi-value keys.
+			for key, value in repo_entry["config"].items():
+				config_yaml_common_config[key][value] += 1
+				#TODO:yaml: The yaml renderer should be able to take tuples as lists.
+				repo_entry["config"][key] = str_or_list(value)
+
+			# Normalize the branches
+			branches_flat = []
+			for key, value in repo_entry["branches"].items():
+				remote, branch = value
+				branches_flat.append(f"{key} <- {remote}:{branch}")
+			if len(branches_flat):
+				repo_entry["branches"] = branches_flat
+			else:
+				del repo_entry["branches"]
+
+		# Convert the initial object with counts to have a flat key-value mapping but only if
+		# the same value for the key appears in more than the threshold count of repos.
+		config_yaml_config = {}
+		for k, v in config_yaml["config"].items():
+			# Get the value which has the biggest count
+			v, count = sorted(v.items(), key=lambda x: x[1], reverse=True)[0]
+			if count < config_yaml_common_config_threshold:
+				# Only continue if count of the most often occurring value is at or above the threshold
+				continue
+			config_yaml_config[k] = str_or_list(v)
+		config_yaml["config"] = config_yaml_config
+
+		for repo_entry in config_yaml["repositories"]:
+			for common_config_key, common_config_value in config_yaml["config"].items():
+				if common_config_key not in repo_entry["config"]:
+					repo_entry["config"][common_config_key] = None
+				elif repo_entry["config"][common_config_key] == common_config_value:
+					del repo_entry["config"][common_config_key]
+			if not len(repo_entry["config"]):
+				del repo_entry["config"]
+
+		with open(output_path, "w", encoding="UTF-8") as fo:
+			yaml.dump(config_yaml, default_flow_style=False, sort_keys=False, stream=fo)
+
+	async def report_new_repo(self, repo):
+		gitdir = repo.gitdir
+		try:
+			gitdir = gitdir.relative_to(pathlib.Path.home())
+		except ValueError:
+			pass
+		sys.stdout.write(gitdir.as_posix())
+		if repo.is_worktree_custom():
+			sys.stdout.write("\t-> ")
+			sys.stdout.write(repo.worktree.as_posix() if repo.worktree else "(bare)")
+		sys.stdout.write("\n")
+		sys.stdout.flush()
+
+
+def single_value_or_tuple(previous_value, new_value):
+	if previous_value is None:
+		return new_value
+	elif isinstance(previous_value, str):
+		return (previous_value, new_value)
+	else:
+		return (*previous_value, new_value)
+
+
+def str_or_list(value):
+	return value if isinstance(value, str) else list(value)