grimoirelab-chronicler 0.0.1rc2__tar.gz

grimoirelab_chronicler-0.0.1rc2/AUTHORS

@@ -0,0 +1,3 @@
+Santiago Dueñas <sduenas@bitergia.com>
+Jose Javier Merchante <jjmerchante@bitergia.com>
+

grimoirelab_chronicler-0.0.1rc2/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.1
+Name: grimoirelab-chronicler
+Version: 0.0.1rc2
+Summary: Generator of GrimoireLab events using Perceval data.
+Home-page: https://chaoss.github.io/grimoirelab/
+License: GPL-3.0+
+Keywords: software analytics,events,grimoirelab
+Author: GrimoireLab Developers
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Software Development
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: cloudevents (>=1.10.1,<2.0.0)
+Requires-Dist: coverage (>=7.2.3,<8.0.0)
+Project-URL: Repository, https://github.com/chaoss/grimoirelab-chronicler
+Description-Content-Type: text/markdown
+
+# Chronicler
+
+Tool and library to convert Perceval data into events.
+
+## Usage
+
+```text
+$ chronicler --help
+Usage: chronicler [OPTIONS] DATASOURCE
+
+Generates GrimoireLab events from the items obtained by Perceval.
+
+The chronicler is a command line tool and a library that converts items
+generated by Perceval into events. The tool reads these events from the
+standard input or from a given file.
+
+To run it, you will have to give the type of items the chronicler is
+receiving with DATASOURCE argument (e.g. git, github, gitlab).
+
+Output is produced using JSON format. Use the option <json_line>
+to generate JSON object per line.
+
+Options:
+  --input FILENAME   File with perceval items
+  --output FILENAME  File where events will be written
+  --json-line        Produce a JSON line for each output item
+  --version          Show the version and exit.
+  --help             Show this message and exit.
+```
+
+You can connect the **Chronicler** with **Perceval** to generate events.
+
+```sh
+perceval git --json-line https://example.com | chronicler git
+```
+
+## Installation
+
+There are several methods to install this tool.
+
+### pip
+
+```sh
+pip install grimoirelab-chronicler
+```
+
+### poetry (source code)
+
+We use [poetry](https://python-poetry.org/) for dependency management and
+packaging. You can install **chronicler** and its dependencies on an
+isolated environment provided by **poetry**.
+
+```sh
+poetry install
+poetry shell
+```
+
+## Contributing
+
+Chronicler is part of the GrimoireLab project. Please read its
+[Contributing Guidelines](https://github.com/chaoss/grimoirelab/blob/main/CONTRIBUTING.md)
+for more information.
+

grimoirelab_chronicler-0.0.1rc2/README.md

@@ -0,0 +1,62 @@
+# Chronicler
+
+Tool and library to convert Perceval data into events.
+
+## Usage
+
+```text
+$ chronicler --help
+Usage: chronicler [OPTIONS] DATASOURCE
+
+Generates GrimoireLab events from the items obtained by Perceval.
+
+The chronicler is a command line tool and a library that converts items
+generated by Perceval into events. The tool reads these events from the
+standard input or from a given file.
+
+To run it, you will have to give the type of items the chronicler is
+receiving with DATASOURCE argument (e.g. git, github, gitlab).
+
+Output is produced using JSON format. Use the option <json_line>
+to generate JSON object per line.
+
+Options:
+  --input FILENAME   File with perceval items
+  --output FILENAME  File where events will be written
+  --json-line        Produce a JSON line for each output item
+  --version          Show the version and exit.
+  --help             Show this message and exit.
+```
+
+You can connect the **Chronicler** with **Perceval** to generate events.
+
+```sh
+perceval git --json-line https://example.com | chronicler git
+```
+
+## Installation
+
+There are several methods to install this tool.
+
+### pip
+
+```sh
+pip install grimoirelab-chronicler
+```
+
+### poetry (source code)
+
+We use [poetry](https://python-poetry.org/) for dependency management and
+packaging. You can install **chronicler** and its dependencies on an
+isolated environment provided by **poetry**.
+
+```sh
+poetry install
+poetry shell
+```
+
+## Contributing
+
+Chronicler is part of the GrimoireLab project. Please read its
+[Contributing Guidelines](https://github.com/chaoss/grimoirelab/blob/main/CONTRIBUTING.md)
+for more information.

grimoirelab_chronicler-0.0.1rc2/chronicler/_version.py

@@ -0,0 +1,2 @@
+# File auto-generated by semverup on 2025-01-16 09:15:00.503408
+__version__ = "0.0.1-rc.2"

grimoirelab_chronicler-0.0.1rc2/chronicler/chronicler.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Developers
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import sys
+
+import click
+import cloudevents.conversion
+
+from ._version import __version__
+from .eventizer import eventize
+
+
+@click.command
+@click.option(
+    "--input",
+    help="File with perceval items",
+    type=click.File("r"),
+    default=sys.stdin
+)
+@click.option(
+    "--output",
+    help="File where events will be written",
+    type=click.File("w"),
+    default=sys.stdout
+)
+@click.option(
+    "--json-line",
+    help="Produce a JSON line for each output item",
+    is_flag=True,
+    show_default=True,
+    default=False
+)
+@click.argument('datasource')
+@click.version_option(__version__, message="%(prog)s %(version)s")
+def chronicler(datasource, input, output, json_line):
+    """Generates GrimoireLab events from the items fetched by Perceval.
+
+    The chronicler is a command line tool and a library that converts
+    items generated by Perceval into events. The tool reads
+    these events from the standard input or from a given file.
+
+    To run it, you will have to give the type of items the chronicler
+    is receiving with DATASOURCE argument (e.g. git, github, gitlab).
+
+    Output is produced using JSON format. Use the option <json_line>
+    to generate JSON object per line.
+    """
+    def _read_input(input):
+        for line in input:
+            yield json.loads(line)
+
+    for item in eventize(datasource, _read_input(input)):
+        if json_line:
+            obj = json.dumps(cloudevents.conversion.to_dict(item),
+                             separators=(',', ':'), sort_keys=True)
+        else:
+            obj = json.dumps(cloudevents.conversion.to_dict(item),
+                             indent=4, sort_keys=True)
+        output.write(obj)
+        output.write('\n')

grimoirelab_chronicler-0.0.1rc2/chronicler/eventizer.py

@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Developers
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import importlib
+import pkgutil
+import os
+
+from collections.abc import Iterator, Generator
+from typing import Any
+
+from cloudevents.http import CloudEvent
+
+
+class Eventizer:
+    """Abstract class to eventize data.
+
+    Events will be generated calling to the `eventize` method.
+    This method will take items produced by `perceval` and
+    will convert them in GrimoireLab events.
+
+    The class can only process items of one type. In other words,
+    `git` items can't be mixed with `github` items. However,
+    the same eventizer can process different categories of
+    the same type. A `github` eventizer would process `issues`,
+    `pull requests`, etc.
+
+    To create your own eventizer, sub-class this class by implementing
+    the method `eventize_item`. This method should have the
+    logic to given a perceval item, produce the events associated to
+    that item.
+    """
+    def eventize(self, raw_items: Iterator[dict[str, Any]]) -> Generator[CloudEvent]:
+        """Generate GrimoireLab events.
+
+        Produce events from the given list of perceval items.
+        The items must be of the same type.
+        """
+        for raw_item in raw_items:
+            yield from self.eventize_item(raw_item)
+
+    def eventize_item(self, raw_item: dict[str, Any]) -> list[CloudEvent]:
+        """Eventize a item."""
+        raise NotImplementedError
+
+
+def eventize(name: str, raw_items: Iterator[dict[str, Any]]) -> Generator[CloudEvent]:
+    """Eventize data of a given type.
+
+    Handy function to produce events from a set of perceval items
+    of given type.
+    """
+    top_package_name = os.environ.get('CHRONICLER_EVENTIZERS',
+                                      'chronicler.events')
+
+    eventizers = _find_eventizers(top_package_name)
+
+    try:
+        eventizer = eventizers[name]()
+    except KeyError:
+        raise ValueError(f"Unknown eventizer '{name}'")
+
+    yield from eventizer.eventize(raw_items)
+
+
+def _find_eventizers(top_package_name: str) -> dict[str, Eventizer]:
+    """Find available eventizers.
+
+    Look for the `Eventizer` classes under `top_package`
+    and its sub-packages. When `top_package` defines a namespace,
+    classes under that same namespace will be found too.
+
+    :param top_package: package storing eventizer classes
+
+    :returns: a dict with `Eventizer`
+    """
+    top_package = importlib.import_module(top_package_name)
+
+    candidates = pkgutil.walk_packages(top_package.__path__,
+                                       prefix=top_package.__name__ + '.')
+
+    modules = [name for _, name, is_pkg in candidates if not is_pkg]
+
+    return _import_eventizers(modules)
+
+
+def _import_eventizers(modules):
+    for module in modules:
+        importlib.import_module(module)
+
+    klasses = _find_classes(Eventizer, modules)
+
+    eventizers = {name: kls for name, kls in klasses}
+
+    return eventizers
+
+
+def _find_classes(parent, modules):
+    parents = parent.__subclasses__()
+
+    while parents:
+        kls = parents.pop()
+
+        m = kls.__module__
+
+        if m not in modules:
+            continue
+
+        name = m.split('.')[-1]
+        parents.extend(kls.__subclasses__())
+
+        yield name, kls
+
+
+def uuid(*args):
+    """Generate a UUID based on the given parameters.
+
+    The UUID will be the SHA1 of the concatenation of the values
+    from the list. The separator between these values is ':'.
+    Each value must be a non-empty string, otherwise, the function
+    will raise an exception.
+
+    :param *args: list of arguments used to generate the UUID
+
+    :returns: a universal unique identifier
+
+    :raises ValueError: when anyone of the values is not a string,
+        is empty or `None`.
+    """
+    import hashlib
+
+    def check_value(v):
+        if not isinstance(v, str):
+            raise ValueError("%s value is not a string instance" % str(v))
+        elif not v:
+            raise ValueError("value cannot be None or empty")
+        else:
+            return v
+
+    s = ':'.join(map(check_value, args))
+
+    sha1 = hashlib.sha1(s.encode('utf-8', errors='surrogateescape'))
+    uuid_sha1 = sha1.hexdigest()
+
+    return uuid_sha1

grimoirelab_chronicler-0.0.1rc2/chronicler/events/core/__init__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Developers
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from ..._version import __version__
+
+__version__ = __version__

grimoirelab_chronicler-0.0.1rc2/chronicler/events/core/git.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Developers
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from typing import Any
+
+from cloudevents.http import CloudEvent
+
+from ...eventizer import Eventizer, uuid
+
+
+GIT_EVENT_COMMIT = "org.grimoirelab.events.git.commit"
+GIT_EVENT_MERGE_COMMIT = "org.grimoirelab.events.git.merge"
+GIT_EVENT_ACTION_ADDED = "org.grimoirelab.events.git.file.added"
+GIT_EVENT_ACTION_MODIFIED = "org.grimoirelab.events.git.file.modified"
+GIT_EVENT_ACTION_DELETED = "org.grimoirelab.events.git.file.deleted"
+GIT_EVENT_ACTION_REPLACED = "org.grimoirelab.events.git.file.replaced"
+GIT_EVENT_ACTION_COPIED = "org.grimoirelab.events.git.file.copied"
+GIT_EVENT_ACTION_TYPE_CHANGED = "org.grimoirelab.events.git.file.typechanged"
+
+
+class GitEventizer(Eventizer):
+    """Eventize git commits"""
+
+    def eventize_item(self, raw_item: dict[str, Any]) -> list[dict[str, Any]]:
+        events = []
+
+        item_uuid = raw_item.get('uuid', None)
+
+        if not item_uuid:
+            raise ValueError("'uuid' attribute not found on item.")
+        if raw_item['backend_name'].lower() != 'git':
+            raise ValueError(f"Item {item_uuid} is not a 'git' item.")
+        if raw_item['category'] != 'commit':
+            raise ValueError(f"Invalid category '{raw_item['category']}' for '{item_uuid}' item.")
+
+        if 'Merge' in raw_item['data']:
+            event_type = GIT_EVENT_MERGE_COMMIT
+        else:
+            event_type = GIT_EVENT_COMMIT
+
+        attributes = {
+            "id": item_uuid,
+            "type": event_type,
+            "source": raw_item['origin'],
+            "time": raw_item['updated_on'],
+        }
+
+        event = CloudEvent(attributes, raw_item['data'])
+        events.append(event)
+
+        action_events = self._eventize_commit_actions(event,
+                                                      raw_item['data']['files'])
+
+        events.extend(action_events)
+
+        return events
+
+    def _eventize_commit_actions(self, parent_event: CloudEvent, raw_files_data):
+
+        events = []
+
+        for file_data in raw_files_data:
+            actions = file_data.get('action', None)
+
+            if not actions and parent_event['type'] == GIT_EVENT_COMMIT:
+                raise ValueError(f"No action for commit event {parent_event['id']}")
+            elif not actions:
+                continue
+
+            if parent_event['type'] == GIT_EVENT_COMMIT:
+                action_event = self._process_action(parent_event['source'],
+                                                    parent_event['time'],
+                                                    parent_event['id'], actions, file_data)
+                events.append(action_event)
+            else:
+                prev_merge_action = None
+                for action in actions:
+                    if action == prev_merge_action:
+                        continue
+
+                    action_event = self._process_action(parent_event['source'],
+                                                        parent_event['time'],
+                                                        parent_event['id'], action, file_data)
+                    events.append(action_event)
+                    prev_merge_action = action
+        return events
+
+    def _process_action(self, source, time, event_uuid, action, file_data):
+        if action == 'A':
+            event_type = GIT_EVENT_ACTION_ADDED
+        elif action == 'M':
+            event_type = GIT_EVENT_ACTION_MODIFIED
+        elif action == 'D':
+            event_type = GIT_EVENT_ACTION_DELETED
+        elif action.startswith('R'):
+            event_type = GIT_EVENT_ACTION_REPLACED
+        elif action.startswith('C'):
+            event_type = GIT_EVENT_ACTION_COPIED
+        elif action.startswith('T'):
+            event_type = GIT_EVENT_ACTION_TYPE_CHANGED
+        else:
+            raise ValueError(f"No valid action: {action}")
+
+        event_id = uuid(event_uuid, file_data['file'], action)
+
+        data = {
+            "filename": file_data['file'],
+            "modes": file_data['modes'],
+            "indexes": file_data['indexes'],
+            "similarity": action[1:] if action in ('R', 'C') else None,
+            "new_filename": file_data.get('newfile', None),
+            "added_lines": file_data.get('added', None),
+            "deleted_lines": file_data.get('removed', None)
+        }
+
+        attributes = {
+            "id": event_id,
+            "linked_event": event_uuid,
+            "type": event_type,
+            "source": source,
+            "time": time,
+        }
+
+        event = CloudEvent(attributes, data)
+
+        return event

grimoirelab_chronicler-0.0.1rc2/pyproject.toml

@@ -0,0 +1,56 @@
+[tool.poetry]
+name = "grimoirelab-chronicler"
+version = "0.0.1-rc.2"
+description = "Generator of GrimoireLab events using Perceval data."
+authors = [
+    "GrimoireLab Developers"
+]
+license = "GPL-3.0+"
+
+readme = "README.md"
+
+homepage = "https://chaoss.github.io/grimoirelab/"
+repository = "https://github.com/chaoss/grimoirelab-chronicler"
+
+keywords = [
+    "software analytics",
+    "events",
+    "grimoirelab"
+]
+
+classifiers = [
+   "Development Status :: 3 - Alpha",
+   "Intended Audience :: Developers",
+   "Intended Audience :: Information Technology",
+   "Intended Audience :: Science/Research",
+   "Topic :: Software Development",
+   "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+   "Programming Language :: Python :: 3"
+]
+
+packages = [
+    { include = "chronicler"},
+]
+
+include = [
+    { path = "AUTHORS", format = "sdist" },
+    { path = "NEWS", format = "sdist" },
+    { path = "README.md", format = "sdist" },
+]
+
+[tool.poetry.scripts]
+chronicler = "chronicler.chronicler:chronicler"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+
+click = "^8.1.7"
+cloudevents = "^1.10.1"
+coverage = "^7.2.3"
+
+[tool.poetry.group.dev.dependencies]
+flake8 = "^7.1.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"