repominer-GDeLuisi 0.1.0__py3-none-any.whl

repominer_gdeluisi-0.1.0.dist-info/METADATA

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: repominer-GDeLuisi
+Version: 0.1.0
+Summary: Tool for mining git managed repositories
+Author: Gerardo De Luisi
+License-File: LICENSE
+Keywords: git,miner,projects
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# repository_miner
+A python library used for mining git repositories
+git >= 2.40 is needed

repominer_gdeluisi-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+src/repository_miner/__init__.py,sha256=sYtS1nL_0XlvcmJrIxE7TQwfcO1f2HDcs1rvFWt-aww,115
+src/repository_miner/data_typing.py,sha256=9mZcFk0s-7CrrGcvKxUqNiORkNk8x_sn0Vjc8PxwRqw,2964
+src/repository_miner/exceptions.py,sha256=xFKI_ra-6FW_Ux3vF-xVM2XRFowAX0FRXzFRLTuTZUg,411
+src/repository_miner/git.py,sha256=UzbDWLbRGxcwMT8tyPSn8U0UOGoHgJvDWV0-BcBnmR8,1293
+src/repository_miner/helper.py,sha256=dLVoA-xW1kQoT7VPYy4kyj1FDmkOdElZJPs_S3Ev51c,5829
+src/repository_miner/miner.py,sha256=GTGJHL-3KF9VamFnlPy0RQvaCyiijxsDlzb-fJZ9I4Y,5035
+src/repository_miner/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/repository_miner/utility.py,sha256=y6rgmmitk4UuZI8AthuNa716UfJSjOL9kZaJ-d7Zc8I,1275
+repominer_gdeluisi-0.1.0.dist-info/METADATA,sha256=zLcpyckfl5WolpDm0e3ccyfj2ELAgy-Pk1NTl4FQjcU,646
+repominer_gdeluisi-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+repominer_gdeluisi-0.1.0.dist-info/licenses/LICENSE,sha256=rPNAmE1lTJ0R9WVW3rDw85fHnHGXR_AbHj_aSI7njO8,1064
+repominer_gdeluisi-0.1.0.dist-info/RECORD,,

repominer_gdeluisi-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

repominer_gdeluisi-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 GeggeDL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

src/repository_miner/__init__.py

@@ -0,0 +1,6 @@
+from .utility import *
+from .data_typing import *
+from .git import Git
+from .helper import *
+from .miner import *
+

src/repository_miner/data_typing.py

@@ -0,0 +1,106 @@
+from  dataclasses import dataclass,field
+from datetime import datetime
+from time import strptime
+from typing import Literal,get_args,Iterable,Optional,Union,Generator,Callable
+import json
+from pathlib import Path
+from repository_miner.utility import Call
+
+@dataclass
+class Author():
+    email:str
+    name:str
+    commits_authored:list[str]=field(default_factory=lambda: [])
+    def __hash__(self):
+        return hash(repr(self.name)+repr(self.email))
+    def __eq__(self, value):
+        if not isinstance(value,Author):
+            raise TypeError(f"Expected value of type <Author>, received {type(value)}")
+        return self.name==value.name and self.email==value.email
+    def __str__(self):
+        return f"Name: {self.name} , Email: {self.email}"
+    def __repr__(self):
+        return  f"Name: {self.name} , Email: {self.email} , Commits: {self.commits_authored}"
+    
+@dataclass
+class CommitInfo():
+    commit_hash:str
+    abbr_hash:str
+    tree:str
+    refs:str
+    subject:str
+    author_name:str
+    author_email:str
+    date:datetime
+    def __hash__(self):
+        return hash(self.commit_hash)
+    def get_tree(self)->'Tree':
+        raise NotImplementedError()
+
+@dataclass
+class Head():
+    name:str
+    hash:str
+    def __hash__(self):
+        return hash(self.hash)
+    def traverse_commits(self)->Generator[CommitInfo,None,None]:
+        raise NotImplementedError()
+    
+class HeadImpl(Head):
+    def __init__(self,name:str,hash:str,retrieve_func:Call):
+        super().__init__(name,hash)
+        self.retrieve_func=retrieve_func
+    def traverse_commits(self):
+        return self.retrieve_func()
+
+
+@dataclass
+class Blob():
+    hash:str
+    name:str
+    path:str
+    size:int
+    def __hash__(self):
+        return hash(self.hash)
+    def get_source(self)->list[str]:
+        raise NotImplementedError()
+
+@dataclass
+class Tree():
+    hash:str
+    path:str
+    def traverse(self)->Generator[Union['Tree',Blob],None,None]:
+        raise NotImplementedError()
+    def __hash__(self):
+        return hash(self.hash)
+    
+class TreeImpl(Tree):
+    def __init__(self,hash:str,path:str,iter_function:Call):
+        super().__init__(hash,path)
+        self.iter_func=iter_function
+    def traverse(self)->Generator[Union[Tree,Blob],None,None]:
+        return self.iter_func()
+    
+class CommitInfoImpl(CommitInfo):
+    def __init__(self
+    ,commit_hash:str
+    ,abbr_hash:str
+    ,tree:str
+    ,refs:str
+    ,subject:str
+    ,author_name:str
+    ,author_email:str
+    ,date:datetime
+    ,tree_func:Call):
+        super().__init__(commit_hash,abbr_hash,tree,refs,subject,author_name,author_email,date)
+        self.tree_func=tree_func
+    
+    def get_tree(self)->Tree:
+        return self.tree_func()
+    
+class BlobImpl(Blob):
+    def __init__(self,hash:str,name:str,path:str,size:int,source_func:Call):
+        super().__init__(hash,name,path,size)
+        self.source_func=source_func
+    def get_source(self):
+        return self.source_func()

src/repository_miner/exceptions.py

@@ -0,0 +1,15 @@
+class GitNotFoundException(Exception):
+    def __init__(self, *args):
+        super().__init__(*args)
+
+class GitCmdError(Exception):
+    def __init__(self, *args):
+        super().__init__(*args)
+    
+class NotGitRepositoryError(Exception):
+    def __init__(self, *args):
+        super().__init__(*args)
+        
+class ParsingException(Exception):
+    def __init__(self, *args):
+        super().__init__(*args)

src/repository_miner/git.py

@@ -0,0 +1,31 @@
+from .utility import execute_command
+from .helper import cmd_builder,log_builder,rev_list_builder,get_head_commit,is_dir_a_repo,is_git_available
+from .exceptions import *
+from functools import partial
+from typing import Iterable,Optional
+from datetime import datetime
+from subprocess import CalledProcessError
+class Git():
+    def __init__(self,path:str):
+        if not is_git_available():
+            raise GitNotFoundException("Git not found")
+        if not is_dir_a_repo(path):
+            raise NotGitRepositoryError(f"Directory {path} is not a git repository")
+        self.path=path
+
+    def _execute_command(self,command:str,*args)->str:
+        cmd=""
+        try:
+            if len(args)==1 and not isinstance(args[0],str) and isinstance(args[0],Iterable):
+                cmd=cmd_builder(command,self.path,*args[0])
+            else:
+                cmd=cmd_builder(command,self.path,*args)
+            return execute_command(cmd)
+        except CalledProcessError as e:
+            raise GitCmdError(f"Command {cmd} raised an error {e.stderr}")
+    
+    def __getattr__(self, name:str):
+        if name in self.__dict__ or name in self.__class__.__dict__:
+            return getattr(self,name)
+        name=name.replace("_","-")
+        return partial(self._execute_command,name)

src/repository_miner/helper.py

@@ -0,0 +1,156 @@
+from pathlib import Path
+from shutil import which
+import sys
+import os
+import subprocess
+from typing import Optional
+from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
+from math import floor,ceil
+from functools import partial
+from typing import Iterable
+import json
+from datetime import datetime
+# max_worker = min(32,os.cpu_count())
+
+def date_builder(since:Optional[datetime]=None,to:Optional[datetime]=None)->list[str]:
+    args=[]
+    if to and since and to<since:
+        raise ValueError("'to' cannot come before 'since'")
+    if since:
+        d_str=since.strftime(r"%Y-%m-%d")
+        args.append(f"--since='{d_str}'")
+    if to:
+        d_str=to.strftime(r"%Y-%m-%d")
+        args.append(f"--until='{d_str}'")
+    return args
+    
+def cmd_builder(command:str,repo:str,*args)->str:
+    """Base git command generator
+
+    Args:
+        command (str): command to use
+        repo (str): git directory to execute the command on
+
+    Returns:
+        str: The complete command as a string
+    """
+            
+    arg_string=f"git -C {repo} {command}"
+    arg_string=arg_string + " "+ " ".join(args)
+    return arg_string
+
+def range_builder(from_commmit:str,to_commit:Optional[str]=None)->str:
+    if not from_commmit:
+        raise ValueError("'from_commit' parameter must always be valorized")
+    if to_commit:
+        return f"{to_commit}..{from_commmit}"
+    else:
+        return from_commmit
+
+def log_builder(from_commit:str,to_commit:Optional[str]=None,pretty:Optional[str]=None,merges:bool=False,max_count:Optional[int]=None,skip:Optional[int]=None,author:Optional[str]=None,follow:Optional[str]=None,since:Optional[datetime]=None,to:Optional[datetime]=None,args=[])->str:
+    """Builds the complete command string for a log command
+
+    Args:
+        repo (str): Git repository to execute the command on
+        commit (str): The commit from which start the logging operation
+        pretty (Optional[str], optional): The format used by --pretty. Defaults to None.
+        merges (bool, optional): Specifies whether load merge commits. Defaults to False.
+        max_count (Optional[int], optional): Paramenter for --max-count flag. Defaults to None.
+        skip (Optional[int], optional): Parameter  for --skip flag. Defaults to None.
+        author (Optional[str], optional): Filter only commits coming authored by the passed author. Defaults to None.
+        follow (Optional[str], optional): Filter only commits which changed the passed file. Defaults to None.
+
+    Returns:
+        str: Returns the git command string
+    """
+    arg_list=[range_builder(from_commit,to_commit)]
+    if max_count!=None:
+        if max_count<=0:
+            raise ValueError("max_count cannot be negative or 0")
+        arg_list.append(f"--max-count={max_count}")
+    if skip!=None:
+        if skip<0:
+            raise ValueError("skip cannot be negative")
+        arg_list.append(f"--skip={skip}")
+    if merges:
+        arg_list.append("--no-merges")
+    if pretty!=None:
+        arg_list.append(f'--pretty="format:{pretty}"')
+    if author:
+        arg_list.append(f'--author="{author}"')
+    arg_list.extend(date_builder(since,to))
+    arg_list.extend(args)
+    if follow:
+        arg_list.append(f'--follow -- "{follow}"')
+    return " ".join(arg_list)
+
+def rev_list_builder(from_commit:str,to_commit:Optional[str]=None,pretty:Optional[str]=None,merges:bool=False,max_count:Optional[int]=None,skip:Optional[int]=None,author:Optional[str]=None,since:Optional[datetime]=None,to:Optional[datetime]=None,args=[])->str:
+    """Builds the complete command string for a log command
+
+    Args:
+        repo (str): Git repository to execute the command on
+        commit (str): The commit from which start the logging operation
+        pretty (Optional[str], optional): The format used by --pretty. Defaults to None.
+        merges (bool, optional): Specifies whether load merge commits. Defaults to False.
+        max_count (Optional[int], optional): Paramenter for --max-count flag. Defaults to None.
+        skip (Optional[int], optional): Parameter  for --skip flag. Defaults to None.
+        author (Optional[str], optional): Filter only commits coming authored by the passed author. Defaults to None.
+
+    Returns:
+        str: Returns the git command string
+    """
+    arg_list=[range_builder(from_commit,to_commit)]
+    if max_count!=None:
+        if max_count<=0:
+            raise ValueError("max_count cannot be negative or 0")
+        arg_list.append(f"--max-count={max_count}")
+    if skip!=None:
+        if skip<0:
+            raise ValueError("skip cannot be negative")
+        arg_list.append(f"--skip={skip}")
+    if merges:
+        arg_list.append("--no-merges")
+    if pretty!=None:
+        arg_list.append(f'--pretty="format:{pretty}"')
+    if author!=None:
+        arg_list.append(f'--author="{author}"')
+    arg_list.extend(date_builder(since,to))
+    arg_list.extend(args)
+    return " ".join(arg_list)
+
+def is_git_available()->bool:
+    """Checks whether git is on PATH
+
+    Returns:
+        bool: If git is on PATH
+    """
+    return which("git")!=None
+
+def is_dir_a_repo(path:str)->bool:
+    """Checks whether the path points to a git directory
+
+    Args:
+        path (str): path to repo dir
+
+    Returns:
+        bool: Returns wheter the directory is a repo
+    """
+    cmd = f"git -C {Path(path).resolve().as_posix()} rev-parse HEAD"
+    try:
+        subprocess.check_call(cmd,shell=True)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+def get_head_commit(path:str)->str:
+    """Return head commit
+
+    Args:
+        path (str): path to git directory
+
+    Returns:
+        str: Returns HEAD's commit sha
+    """
+    cmd = f"git -C {Path(path).resolve().as_posix()} rev-parse HEAD"
+    return subprocess.check_output(cmd,shell=True).decode()[:-1]
+

src/repository_miner/miner.py

@@ -0,0 +1,98 @@
+from .git import Git
+from .utility import Call
+from .helper import cmd_builder,log_builder,rev_list_builder,get_head_commit,is_dir_a_repo,is_git_available
+from .exceptions import *
+from .data_typing import *
+from functools import partial
+from typing import Iterable,Optional,Generator
+from datetime import datetime
+import re
+class RepoMiner():
+    def __init__(self,path:str):
+        self.git=Git(path)
+        self.path=path
+        
+    def retrieve_commits(self,from_commit:Optional[str]=None,to_commit:Optional[str]=None,merges:bool=False,max_count:Optional[int]=None,skip:Optional[int]=None,author:Optional[str]=None,follow:Optional[str]=None,since:Optional[datetime]=None,to:Optional[datetime]=None,extra_args:Optional[Iterable[str]]=[])->Generator[CommitInfo,None,None]:
+        if not from_commit:
+            from_commit=get_head_commit(self.path)
+        pretty=r"%H///%T///%s///%an///%ae///%as///%D"
+        logs=self.git.log(log_builder(from_commit,to_commit,pretty,merges,max_count,skip,author,follow,since,to,extra_args))
+        for log in logs.splitlines(False):
+            try:
+                c_hash,tree,sub,a_name,a_email,c_date,ref=log.split(r"///")
+                yield CommitInfoImpl(c_hash,c_hash[:7],tree,ref,sub,a_name,a_email,datetime.strptime(c_date,r"%Y-%m-%d"),Call(self.tree,tree))
+            except ValueError as e:
+                raise ParsingException(f"Log {log} was not parsed")
+    
+    def n_commits(self,from_commit:Optional[str]=None,to_commit:Optional[str]=None,merges:bool=True,skip:Optional[int]=None,author:Optional[str]=None,since:Optional[datetime]=None,to:Optional[datetime]=None)->int:
+        if not from_commit:
+            from_commit=get_head_commit(self.path)
+        return int(self.git.rev_list(rev_list_builder(from_commit=from_commit,to_commit=to_commit,merges=merges,max_count=None,skip=skip,author=author,since=since,to=to,args=["--count"])))
+                
+    def tree(self,treeish:str)->Tree:
+        try:
+            t=self.git.cat_file(["-t",treeish])
+            if t == "blob" and t == "tag":
+                raise GitCmdError()
+            return TreeImpl(treeish,"",Call(self.iterate_tree,treeish,True))
+        except GitCmdError as e:
+            raise ValueError(f"Cannot retrieve a tree from {treeish}")
+    
+    def iterate_tree(self,treeish:str,recursive:bool=False)->Generator[Union[Tree,Blob],None,None]:
+        p_format="--format=\"%(objectname)///%(objecttype)///%(objectsize)///%(path)\""
+        args=[p_format]
+        if recursive:
+            args.append("-r")
+            args.append("-t")
+        args.append(treeish)
+        try:
+            res=self.git.ls_tree(args)
+            for line in res.splitlines(False):
+                h,t,size,path=line.split('///')
+                if t == "tree":
+                    yield TreeImpl(h,path,Call(self.iterate_tree,treeish=treeish,recursive=True))
+                elif t == "blob":
+                    size=int(size)
+                    yield BlobImpl(h,path.rsplit("/",1)[-1],path,size,Call(self.get_source,h))
+        except GitCmdError as e:
+            raise ValueError(f"Cannot retrieve a tree from {treeish}")
+        except ValueError as e:
+            raise ParsingException(f"Unable to parse tree line {line}")
+    
+    def get_commit(self,commit_sha:str)->CommitInfo:
+        pretty=r"%H///%T///%s///%an///%ae///%as///%D"
+        log=self.git.log(log_builder(commit_sha,None,pretty,max_count=1))
+        c_hash,tree,sub,a_name,a_email,c_date,ref=log.split(r"///")
+        return CommitInfoImpl(c_hash,c_hash[:7],tree,ref,sub,a_name,a_email,datetime.strptime(c_date,r"%Y-%m-%d"),Call(self.tree,tree))
+    
+    def local_branches(self)->Generator[Head,None,None]:
+        branches=self.git.branch("-l").splitlines()
+        for branch in branches:
+            name=branch.strip("*").strip()
+            yield HeadImpl(name,self.git.rev_parse(name),Call(self.retrieve_commits,from_commit=name,merges=True))
+
+    def authors(self)->set[Author]:
+        pattern=re.compile(r'([A-Za-zÀ-ÖØ-öø-ÿé\s]+) <([a-z0-9A-ZÀ-ÖØ-öø-ÿé!#$%@.&*+\/=?^_{|}~-]+)> \(\d+\)')
+        authors=set()
+        res=self.git.shortlog(["-e","--all","--pretty='format:%H'"])
+        res=res.split("\n\n")[:-1]
+        for a_block in res:
+            tmp=a_block.split("\n")
+            author=tmp.pop(0).strip()
+            match=re.match(pattern=pattern,string=author)
+            if not match:
+                continue
+            name,email=match.groups()
+            author = Author(name,email,[])
+            for line in tmp:
+                author.commits_authored.append(line.strip())
+            authors.add(author)
+        return authors
+    
+    def get_source(self, id:str)->list[str]:
+        try:
+            if self.git.cat_file("-t",id) != "blob":
+                raise TypeError(f"Hexsha {id} in not a blob")
+        except GitCmdError:
+            raise FileNotFoundError("Couldn't retrieve the object")
+        return re.split(string=self.git.cat_file("-p",id),pattern=r"\r\n|\r|\n")

src/repository_miner/utility.py

@@ -0,0 +1,40 @@
+import subprocess
+from math import floor,ceil
+from typing import Iterable,Callable,Any
+
+class Call():
+    def __init__(self,func:Callable[...,Any],*args,**kwargs):
+        self.func = func
+        self.args=args
+        self.kwargs=kwargs
+    def __call__(self, *args, **kwds):
+        return self.func(*self.args,**self.kwargs)
+    
+def execute_command(command:str)->str:
+    return subprocess.check_output(command,shell=True,text=True,encoding="utf-8").strip()
+
+def create_batches(it:Iterable,n:int)->Iterable[Iterable]:
+    """create batches of n items for batch using the items in the iterable
+
+    Args:
+        it (Iterable): iterable from which batches are created
+        n (int): number of items for each batch
+
+    Raises:
+        ValueError: If iterable is empty or None and if the number of items for batch is not correct
+
+    Returns:
+        Iterable[Iterable]: Iterable containing the batches
+    """
+    if not n:
+        raise ValueError("n must be at least 1")
+    if not it:
+        raise ValueError("Iterable cannot be None or empty")
+    batches=[]
+    tmp=list(it)
+    n_items=len(tmp)
+    if n_items==0:
+        raise ValueError("Iterable must not be empty")
+    for i in range(0,n_items,n):
+        batches.append(tmp[i:i+n])
+    return tuple(batches)