git-recap 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

git_recap/providers/__init__.py

@@ -1,9 +1,11 @@
 from git_recap.providers.azure_fetcher import AzureFetcher
 from git_recap.providers.github_fetcher import GitHubFetcher
 from git_recap.providers.gitlab_fetcher import GitLabFetcher
+from git_recap.providers.url_fetcher import URLFetcher
 
 __all__ = [
     "AzureFetcher",
     "GitHubFetcher",
-    "GitLabFetcher"
-]
+    "GitLabFetcher",
+    "URLFetcher"
+]

git_recap/providers/azure_fetcher.py

@@ -13,10 +13,23 @@ class AzureFetcher(BaseFetcher):
         self.connection = Connection(base_url=self.organization_url, creds=credentials)
         self.core_client = self.connection.clients.get_core_client()
         self.git_client = self.connection.clients.get_git_client()
-        # If no authors provided, default to an empty list.
+        self.repos = self.get_repos()
+        # Azure DevOps doesn't provide an affiliation filter;
+        # we'll iterate over all repos in each project.
         if authors is None:
             self.authors = []
-    
+
+    def get_repos(self):
+        projects = self.core_client.get_projects().value
+        # Get all repositories in each project
+        repos = [self.git_client.get_repositories(project.id) for project in projects]
+        return repos
+
+    @property
+    def repos_names(self)->List[str]:
+        "to be implemented later"
+        ...
+
     def _filter_by_date(self, date_obj: datetime) -> bool:
         if self.start_date and date_obj < self.start_date:
             return False
@@ -32,39 +45,35 @@ class AzureFetcher(BaseFetcher):
     def fetch_commits(self) -> List[Dict[str, Any]]:
         entries = []
         processed_commits = set()
-        projects = self.core_client.get_projects().value
-        for project in projects:
-            repos = self.git_client.get_repositories(project.id)
-            for repo in repos:
-                if self.repo_filter and repo.name not in self.repo_filter:
+        for repo in self.repos:
+            if self.repo_filter and repo.name not in self.repo_filter:
+                continue
+            for author in self.authors:
+                try:
+                    commits = self.git_client.get_commits(
+                        project=repo.id,
+                        repository_id=repo.id,
+                        search_criteria={"author": author}
+                    )
+                except Exception:
                     continue
-                # Iterate over provided authors
-                for author in self.authors:
-                    try:
-                        commits = self.git_client.get_commits(
-                            project=project.id,
-                            repository_id=repo.id,
-                            search_criteria={"author": author}
-                        )
-                    except Exception:
-                        continue
-                    for commit in commits:
-                        # Azure DevOps returns a commit with an 'author' property
-                        commit_date = commit.author.date  # type: datetime
-                        if self._filter_by_date(commit_date):
-                            sha = commit.commit_id
-                            if sha not in processed_commits:
-                                entry = {
-                                    "type": "commit",
-                                    "repo": repo.name,
-                                    "message": commit.comment.strip(),
-                                    "timestamp": commit_date,
-                                    "sha": sha,
-                                }
-                                entries.append(entry)
-                                processed_commits.add(sha)
-                        if self._stop_fetching(commit_date):
-                            break
+                for commit in commits:
+                    # Azure DevOps returns a commit with an 'author' property.
+                    commit_date = commit.author.date  # assumed datetime
+                    if self._filter_by_date(commit_date):
+                        sha = commit.commit_id
+                        if sha not in processed_commits:
+                            entry = {
+                                "type": "commit",
+                                "repo": repo.name,
+                                "message": commit.comment.strip(),
+                                "timestamp": commit_date,
+                                "sha": sha,
+                            }
+                            entries.append(entry)
+                            processed_commits.add(sha)
+                    if self._stop_fetching(commit_date):
+                        break
         return entries
 
     def fetch_pull_requests(self) -> List[Dict[str, Any]]:
@@ -130,9 +139,8 @@ class AzureFetcher(BaseFetcher):
 
     def fetch_issues(self) -> List[Dict[str, Any]]:
         entries = []
-        # Azure DevOps issues are typically tracked as Work Items.
         wit_client = self.connection.clients.get_work_item_tracking_client()
-        # Query work items for each author; this is a simplified WIQL query.
+        # Query work items for each author using a simplified WIQL query.
         for author in self.authors:
             wiql = f"SELECT [System.Id], [System.Title], [System.CreatedDate] FROM WorkItems WHERE [System.AssignedTo] CONTAINS '{author}'"
             try:
@@ -141,7 +149,6 @@ class AzureFetcher(BaseFetcher):
                 continue
             for item_ref in query_result:
                 work_item = wit_client.get_work_item(item_ref.id)
-                # The created date is a string; convert it to a datetime.
                 created_date = datetime.fromisoformat(work_item.fields["System.CreatedDate"])
                 if self._filter_by_date(created_date):
                     entry = {

git_recap/providers/base_fetcher.py

@@ -25,6 +25,11 @@ class BaseFetcher(ABC):
         self.repo_filter = repo_filter or []
         self.limit = -1
         self.authors = [] if authors is None else authors
+
+    @property
+    @abstractmethod
+    def repos_names(self)->List[str]:
+        pass
         
     @abstractmethod
     def fetch_commits(self) -> List[str]:
@@ -46,7 +51,10 @@ class BaseFetcher(ABC):
         """
         commit_entries = self.fetch_commits()
         pr_entries = self.fetch_pull_requests()
-        issue_entries = self.fetch_issues()
+        try:
+            issue_entries = self.fetch_issues()
+        except Exception as e:
+            issue_entries = []
 
         all_entries = pr_entries + commit_entries + issue_entries
 

git_recap/providers/github_fetcher.py

@@ -7,9 +7,14 @@ class GitHubFetcher(BaseFetcher):
     def __init__(self, pat: str, start_date=None, end_date=None, repo_filter=None, authors=None):
         super().__init__(pat, start_date, end_date, repo_filter, authors)
         self.github = Github(self.pat)
-        self.user = self.github.get_user()
+        self.user = self.github.get_user()        
+        self.repos = self.user.get_repos(affiliation="owner,collaborator,organization_member")
         self.authors.append(self.user.login)
 
+    @property
+    def repos_names(self)->List[str]:
+        return [repo.name for repo in self.repos]
+
     def _stop_fetching(self, date_obj: datetime) -> bool:
         if self.start_date and date_obj < self.start_date:
             return True
@@ -25,8 +30,7 @@ class GitHubFetcher(BaseFetcher):
     def fetch_commits(self) -> List[Dict[str, Any]]:
         entries = []
         processed_commits = set()
-        repos = self.user.get_repos()
-        for repo in repos:
+        for repo in self.repos:
             if self.repo_filter and repo.name not in self.repo_filter:
                 continue
             for author in self.authors:
@@ -53,13 +57,13 @@ class GitHubFetcher(BaseFetcher):
         entries = []
         # Maintain a local set to skip duplicate commits already captured in a PR.
         processed_pr_commits = set()
-        repos = self.user.get_repos()
-        for repo in repos:
+        # Retrieve repos where you're owner, a collaborator, or an organization member.
+        for repo in self.repos:
             if self.repo_filter and repo.name not in self.repo_filter:
                 continue
             pulls = repo.get_pulls(state='all')
             for i, pr in enumerate(pulls, start=1):
-                if pr.user.login not in  self.authors:
+                if pr.user.login not in self.authors:
                     continue
                 pr_date = pr.updated_at  # alternatively, use pr.created_at
                 if not self._filter_by_date(pr_date):
@@ -97,6 +101,7 @@ class GitHubFetcher(BaseFetcher):
                     break
         return entries
 
+
     def fetch_issues(self) -> List[Dict[str, Any]]:
         entries = []
         issues = self.user.get_issues()

git_recap/providers/gitlab_fetcher.py

@@ -8,12 +8,19 @@ class GitLabFetcher(BaseFetcher):
         super().__init__(pat, start_date, end_date, repo_filter, authors)
         self.gl = gitlab.Gitlab(url, private_token=self.pat)
         self.gl.auth()
+        # Instead of only owned projects, retrieve projects where you're a member.
+        self.projects = self.gl.projects.list(membership=True, all=True)
         # Default to the authenticated user's username if no authors are provided.
         if authors is None:
             self.authors = [self.gl.user.username]
         else:
             self.authors = authors
 
+    @property
+    def repos_names(self)->List[str]:
+        "to be implemented later"
+        return [project.name for project in self.projects]
+
     def _filter_by_date(self, date_str: str) -> bool:
         date_obj = datetime.fromisoformat(date_str)
         if self.start_date and date_obj < self.start_date:
@@ -31,8 +38,7 @@ class GitLabFetcher(BaseFetcher):
     def fetch_commits(self) -> List[Dict[str, Any]]:
         entries = []
         processed_commits = set()
-        projects = self.gl.projects.list(owned=True, all=True)
-        for project in projects:
+        for project in self.projects:
             if self.repo_filter and project.name not in self.repo_filter:
                 continue
             for author in self.authors:
@@ -59,11 +65,10 @@ class GitLabFetcher(BaseFetcher):
     def fetch_pull_requests(self) -> List[Dict[str, Any]]:
         entries = []
         processed_pr_commits = set()
-        projects = self.gl.projects.list(owned=True, all=True)
-        for project in projects:
+        for project in self.projects:
             if self.repo_filter and project.name not in self.repo_filter:
                 continue
-            # Fetch merge requests (the GitLab equivalent of pull requests)
+            # Fetch merge requests (GitLab's pull requests)
             merge_requests = project.mergerequests.list(state='all', all=True)
             for mr in merge_requests:
                 if mr.author['username'] not in self.authors:
@@ -105,8 +110,7 @@ class GitLabFetcher(BaseFetcher):
 
     def fetch_issues(self) -> List[Dict[str, Any]]:
         entries = []
-        projects = self.gl.projects.list(owned=True, all=True)
-        for project in projects:
+        for project in self.projects:
             if self.repo_filter and project.name not in self.repo_filter:
                 continue
             issues = project.issues.list(assignee_id=self.gl.user.id)

git_recap/providers/url_fetcher.py

@@ -0,0 +1,232 @@
+import os
+import re
+import shutil
+import subprocess
+from pathlib import Path
+import tempfile
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+from git_recap.providers.base_fetcher import BaseFetcher
+
+
+class URLFetcher(BaseFetcher):
+    """Fetcher implementation for generic Git repository URLs."""
+    
+    GIT_URL_PATTERN = re.compile(
+        r'^(?:http|https|git|ssh)://'  # Protocol
+        r'(?:\S+@)?'  # Optional username
+        r'([^/]+)'  # Domain
+        r'(?:[:/])([^/]+/[^/]+?)(?:\.git)?$'  # Repo path
+    )
+    
+    def __init__(
+        self,
+        url: str,
+        start_date: Optional[datetime] = None,
+        end_date: Optional[datetime] = None,
+        repo_filter: Optional[List[str]] = None,
+        authors: Optional[List[str]] = None
+    ):
+        super().__init__(
+            pat="",  # No PAT needed for URL fetcher
+            start_date=start_date,
+            end_date=end_date,
+            repo_filter=repo_filter,
+            authors=authors
+        )
+        self.url = self._normalize_url(url)
+        self.temp_dir = None
+        self._validate_url()
+        self._clone_repo()
+
+    def _normalize_url(self, url: str) -> str:
+        """Normalize the Git URL to ensure consistent format."""
+        url = url.strip()
+        if not url.endswith('.git'):
+            url += '.git'
+        if not any(url.startswith(proto) for proto in ('http://', 'https://', 'git://', 'ssh://')):
+            url = f'https://{url}'
+        return url
+
+    def _validate_url(self) -> None:
+        """Validate the Git repository URL using git ls-remote."""
+        if not self.GIT_URL_PATTERN.match(self.url):
+            raise ValueError(f"Invalid Git repository URL format: {self.url}")
+            
+        try:
+            result = subprocess.run(
+                ["git", "ls-remote", self.url],
+                capture_output=True,
+                text=True,
+                check=True,
+                timeout=10  # Add timeout to prevent hanging
+            )
+            if not result.stdout.strip():
+                raise ValueError(f"URL {self.url} points to an empty repository")
+        except subprocess.TimeoutExpired:
+            raise ValueError(f"Timeout while validating URL {self.url}")
+        except subprocess.CalledProcessError as e:
+            raise ValueError(f"Invalid Git repository URL: {self.url}. Error: {e.stderr}") from e
+
+    def _clone_repo(self) -> None:
+        """Clone the repository to a temporary directory with all branches."""
+        self.temp_dir = tempfile.mkdtemp(prefix="gitrecap_")
+        try:
+            # First clone with --no-checkout to save bandwidth
+            subprocess.run(
+                ["git", "clone", "--no-checkout", self.url, self.temp_dir],
+                check=True,
+                capture_output=True,
+                text=True,
+                timeout=300
+            )
+            
+            # Fetch all branches
+            subprocess.run(
+                ["git", "-C", self.temp_dir, "fetch", "--all"],
+                check=True,
+                capture_output=True,
+                text=True,
+                timeout=300
+            )
+            
+            # Verify the cloned repository has at least one commit
+            verify_result = subprocess.run(
+                ["git", "-C", self.temp_dir, "rev-list", "--count", "--all"],
+                capture_output=True,
+                text=True,
+                check=True
+            )
+            if int(verify_result.stdout.strip()) == 0:
+                raise ValueError("Cloned repository has no commits")
+                
+        except subprocess.TimeoutExpired:
+            raise RuntimeError("Repository cloning timed out")
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"Failed to clone repository: {e.stderr}") from e
+        except Exception as e:
+            self.clear()
+            raise RuntimeError(f"Unexpected error during cloning: {str(e)}") from e
+
+    @property
+    def repos_names(self) -> List[str]:
+        """Return list of repository names (single item for URL fetcher)."""
+        if not self.temp_dir:
+            return []
+        
+        match = self.GIT_URL_PATTERN.match(self.url)
+        if not match:
+            return []
+            
+        repo_name = match.group(2).split('/')[-1]
+        if repo_name.endswith(".git"):
+            repo_name = repo_name[:-4]
+            
+        return [repo_name]
+
+    def _get_all_branches(self) -> List[str]:
+        """Get list of all remote branches in the repository."""
+        if not self.temp_dir:
+            return []
+            
+        try:
+            result = subprocess.run(
+                ["git", "-C", self.temp_dir, "branch", "-r", "--format=%(refname:short)"],
+                capture_output=True,
+                text=True,
+                check=True
+            )
+            branches = [b.strip() for b in result.stdout.splitlines() if b.strip()]
+            # Filter out HEAD reference if present
+            return [b for b in branches if not b.endswith('/HEAD')]
+        except subprocess.CalledProcessError:
+            return []
+
+    def _run_git_log(self, extra_args: List[str] = None) -> List[Dict[str, Any]]:
+        """Run git log command with common arguments and parse output."""
+        if not self.temp_dir:
+            return []
+
+        args = [
+            "git",
+            "-C", self.temp_dir,
+            "log",
+            "--pretty=format:%H|%an|%ad|%s",
+            "--date=iso",
+            "--all"  # Include all branches and tags
+        ]
+
+        if self.start_date:
+            args.extend(["--since", self.start_date.isoformat()])
+        if self.end_date:
+            args.extend(["--until", self.end_date.isoformat()])
+        if self.authors:
+            authors_filter = "|".join(self.authors)
+            args.extend(["--author", authors_filter])
+        if extra_args:
+            args.extend(extra_args)
+
+        try:
+            result = subprocess.run(
+                args,
+                capture_output=True,
+                text=True,
+                check=True,
+                timeout=120  # Increased timeout for large repositories
+            )
+            return self._parse_git_log(result.stdout)
+        except subprocess.TimeoutExpired:
+            return []
+        except subprocess.CalledProcessError:
+            return []
+
+    def _parse_git_log(self, log_output: str) -> List[Dict[str, Any]]:
+        """Parse git log output into structured data."""
+        entries = []
+        for line in log_output.splitlines():
+            if not line.strip():
+                continue
+                
+            try:
+                sha, author, date_str, message = line.split("|", 3)
+                timestamp = datetime.fromisoformat(date_str)
+                
+                if self.start_date and timestamp < self.start_date:
+                    continue
+                if self.end_date and timestamp > self.end_date:
+                    continue
+
+                entries.append({
+                    "type": "commit",
+                    "repo": self.repos_names[0],
+                    "message": message,
+                    "sha": sha,
+                    "author": author,
+                    "timestamp": timestamp
+                })
+            except ValueError:
+                continue  # Skip malformed log entries
+                
+        return entries
+
+    def fetch_commits(self) -> List[Dict[str, Any]]:
+        """Fetch commits from all branches in the cloned repository."""
+        return self._run_git_log()
+
+    def fetch_pull_requests(self) -> List[Dict[str, Any]]:
+        """Fetch pull requests (not implemented for generic Git URLs)."""
+        return []
+
+    def fetch_issues(self) -> List[Dict[str, Any]]:
+        """Fetch issues (not implemented for generic Git URLs)."""
+        return []
+
+    def clear(self) -> None:
+        """Clean up temporary directory."""
+        if self.temp_dir and os.path.exists(self.temp_dir):
+            try:
+                shutil.rmtree(self.temp_dir, ignore_errors=True)
+            except Exception:
+                pass  # Ensure we don't raise during cleanup
+            finally:
+                self.temp_dir = None

{git_recap-0.1.0.dist-info → git_recap-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: git-recap
-Version: 0.1.0
+Version: 0.1.2
 Summary: A modular Python tool that aggregates and formats user-authored messages from repositories.
 Author: Bruno V.
 Author-email: bruno.vitorino@tecnico.ulisboa.pt

git_recap-0.1.2.dist-info/RECORD

@@ -0,0 +1,14 @@
+git_recap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+git_recap/fetcher.py,sha256=oRlenzd9OsiBkCtpUgSZGlUoUnVpdkolUZhFOF1USBs,2677
+git_recap/utils.py,sha256=mAGZ6bNklBmE-qpA6RG17_Bka8WRXzjLRAAGS9p3QXk,3646
+git_recap/providers/__init__.py,sha256=njrsu58KB60-wN78P0egm1lkQLIRq_PHLKM4WBt39Os,330
+git_recap/providers/azure_fetcher.py,sha256=PfPEeWTtx1Fqa1jClFPKqXhY5Yvs8MyEkSJkLAl4QvQ,7241
+git_recap/providers/base_fetcher.py,sha256=SS35FP8B9_RwnU6CnMbcV9g5_YRWp7v-S7pwbt402Ng,3067
+git_recap/providers/github_fetcher.py,sha256=rDb00RVy2NMpwv8s8Ry1-UjS6wf1Sh2Ndl0xKOZXMxU,4998
+git_recap/providers/gitlab_fetcher.py,sha256=ch1V9O-MPv5_nyg9yLs6EC9e-yqsFXBXMm6xVevWzuQ,5396
+git_recap/providers/url_fetcher.py,sha256=_4CGHfXt701GUHBjsWiu-2ZMTtNerwp60ZxTQh-Mwtc,8256
+git_recap-0.1.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+git_recap-0.1.2.dist-info/METADATA,sha256=USOwnxiUUkGbMt82rPVA2Dw0oAc-YqTl7ow-CvBKyaQ,4721
+git_recap-0.1.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+git_recap-0.1.2.dist-info/top_level.txt,sha256=1JUKd3WPB8c3LcD1deIW-1UTmYzA0zJqwugAz72YZ_o,10
+git_recap-0.1.2.dist-info/RECORD,,

git_recap-0.1.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-git_recap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-git_recap/fetcher.py,sha256=oRlenzd9OsiBkCtpUgSZGlUoUnVpdkolUZhFOF1USBs,2677
-git_recap/utils.py,sha256=mAGZ6bNklBmE-qpA6RG17_Bka8WRXzjLRAAGS9p3QXk,3646
-git_recap/providers/__init__.py,sha256=amk4xSzKOkjW8RaorBSJoiDnjiyFHoz8jFrgFM6Wx_o,256
-git_recap/providers/azure_fetcher.py,sha256=KMqXdIKYGp_EC2BHKhqBIfdXJ7nENx6OSSdqtySa618,7270
-git_recap/providers/base_fetcher.py,sha256=d8QuadaVaGxksbImrnAV10k21i21OSxVopOQYDbuIcA,2902
-git_recap/providers/github_fetcher.py,sha256=yq5MF4VoZww5-yKxyKb270X1EZsRqtDc9P6XKBVs6s0,4770
-git_recap/providers/gitlab_fetcher.py,sha256=2ivuBdVJVgT7qGqTakkMYkkOHd2veyh54in6ttr59FM,5284
-git_recap-0.1.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-git_recap-0.1.0.dist-info/METADATA,sha256=a1A8NFSnAhjd6haIP5SpzzNbIdKjUSpS92OLJNrS6b8,4721
-git_recap-0.1.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-git_recap-0.1.0.dist-info/top_level.txt,sha256=1JUKd3WPB8c3LcD1deIW-1UTmYzA0zJqwugAz72YZ_o,10
-git_recap-0.1.0.dist-info/RECORD,,