grimoirelab-core 0.0.1rc2__py3-none-any.whl

grimoirelab/core/_version.py

@@ -0,0 +1,2 @@
+# File auto-generated by semverup on 2025-01-16 11:50:05.274707
+__version__ = "0.0.1-rc.2"

grimoirelab/core/app/asgi.py

@@ -0,0 +1,9 @@
+"""ASGI config for grimoirelab project"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'grimoirelab.core.config.settings')
+
+application = get_asgi_application()

grimoirelab/core/app/urls.py

@@ -0,0 +1,12 @@
+"""GrimoireLab URL Configuration"""
+
+
+from django.urls import path, include, re_path
+from django.views.generic import TemplateView
+
+from grimoirelab.core.scheduler.urls import urlpatterns as sched_urlpatterns
+
+urlpatterns = [
+    path("scheduler/", include(sched_urlpatterns)),
+    re_path(r'^(?!static|scheduler).*$', TemplateView.as_view(template_name="index.html"))
+]

grimoirelab/core/app/wsgi.py

@@ -0,0 +1,10 @@
+"""WSGI config for GrimoireLab."""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+
+application = get_wsgi_application()

grimoirelab/core/config/settings.py

@@ -0,0 +1,307 @@
+#
+# GrimoireLab core basic settings file.
+#
+# This file defines the required settings to run GrimoireLab.
+# Due to GrimoireLab is a Django based app, these settings are
+# based on the configuration file generated by Django by
+# default.
+#
+# Please check the next links for details about the configuration
+# in a production environment:
+#
+# https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
+# https://docs.djangoproject.com/en/4.2/ref/settings/
+#
+
+import os
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+SILENCED_SYSTEM_CHECKS = [
+    'django_mysql.E016'
+]
+
+#
+# General app parameters
+#
+
+#
+# You must never enable debug in production.
+#
+# https://docs.djangoproject.com/en/4.2/ref/settings/#std:setting-DEBUG
+#
+
+DEBUG = os.environ.get('GRIMOIRELAB_DEBUG', 'False').lower() in ('true', '1')
+
+#
+# ALLOWED_HOST protects the site against CSRF attacks.
+# If DEBUG is set to False, you will need to configure this parameter,
+# with the host you are using to serve GrimoireLab.
+#
+# https://docs.djangoproject.com/en/4.2/ref/settings/#allowed-hosts
+#
+
+if 'GRIMOIRELAB_ALLOWED_HOST' in os.environ:
+    ALLOWED_HOSTS = os.environ['GRIMOIRELAB_ALLOWED_HOST'].split(',')
+else:
+    ALLOWED_HOSTS = [
+        '127.0.0.1',
+        'localhost',
+    ]
+
+#
+# Cross-Origin Resource Sharing (CORS)
+#
+# You'll HAVE TO configure the origins that are authorized to make
+# cross-site HTTP requests. Check the following link to understand
+# the possibilities and parameters you can use.
+#
+# https://github.com/adamchainz/django-cors-headers#configuration
+#
+
+if 'GRIMOIRELAB_CORS_ALLOWED_ORIGINS' in os.environ:
+    CORS_ALLOWED_ORIGINS = os.environ['GRIMOIRELAB_CORS_ALLOWED_ORIGINS'].split(',')
+elif 'GRIMOIRELAB_CORS_ALLOWED_ORIGIN_REGEXES' in os.environ:
+    CORS_ALLOWED_ORIGIN_REGEXES = os.environ['GRIMOIRELAB_CORS_ALLOWED_ORIGIN_REGEXES'].split(',')
+else:
+    CORS_ALLOWED_ORIGINS = [
+        'http://localhost:5173',
+    ]
+
+CORS_ALLOW_CREDENTIALS = True
+
+
+#
+# The secret key must be a large random value and it must be kept secret.
+#
+# https://docs.djangoproject.com/en/4.2/ref/settings/#secret-key
+#
+
+SECRET_KEY = os.environ.get('GRIMOIRELAB_SECRET_KEY', 'fake-key')
+
+#
+# Application definition - DO NOT MODIFY
+#
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django_rq',
+    'corsheaders',
+    'rest_framework',
+    'grimoirelab.core.scheduler',
+    'grimoirelab.core.scheduler.tasks',
+]
+
+MIDDLEWARE = [
+    'corsheaders.middleware.CorsMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'grimoirelab.core.app.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [BASE_DIR / 'templates'],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'grimoirelab.core.app.wsgi.application'
+
+
+#
+# Grimoirelab core database
+#
+# You MUST set the database parameters in order to run
+# GrimoireLab.
+#
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'HOST': os.environ.get('GRIMOIRELAB_DB_HOST', '127.0.0.1'),
+        'PORT': os.environ.get('GRIMOIRELAB_DB_PORT', 3306),
+        'USER': os.environ.get('GRIMOIRELAB_DB_USER', 'root'),
+        'PASSWORD': os.environ.get('GRIMOIRELAB_DB_PASSWORD', ''),
+        'NAME': os.environ.get('GRIMOIRELAB_DB_DATABASE', 'grimoirelab_test'),
+        'OPTIONS': {'charset': 'utf8mb4'},
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+#
+# Internationalization
+#
+# https://docs.djangoproject.com/en/4.2/topics/i18n/
+#
+#
+
+LANGUAGE_CODE = 'en-us'
+USE_I18N = True
+
+#
+# Time Zone
+#
+
+USE_TZ = True
+TIME_ZONE = 'UTC'
+
+#
+# GrimoireLab Logging
+#
+# https://docs.djangoproject.com/en/4.2/topics/logging/#configuring-logging
+#
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'simple': {
+            'format': '[{asctime}] {message}',
+            'style': '{',
+        },
+        'verbose': {
+            'format': '[{asctime} - {levelname} - {name}:{lineno}] {message}',
+            'style': '{',
+        },
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose',
+        },
+    },
+    'root': {
+        'handlers': ['console'],
+        'level': 'INFO',
+    },
+}
+
+
+#
+# Static files (CSS, JavaScript, Images)
+#
+# https://docs.djangoproject.com/en/4.2/howto/static-files/
+#
+
+STATIC_URL = '/static/'
+STATICFILES_DIRS = [
+    BASE_DIR / 'templates/static'
+]
+
+# UI static files will be copied to the next path when
+# 'collectstatic' is run.
+# If you are serving these files in a dedicated server, you will
+# need to copy them to their final destination.
+
+STATIC_ROOT = os.path.join(BASE_DIR, 'static')
+
+# MEDIA_URL is only needed when DEBUG is set to True.
+# Modify this URL if you want to run the server in developer mode.
+
+MEDIA_URL = 'http://media.localhost/'
+
+#
+# Default primary key field type
+#
+# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
+#
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+# Django REST Framework settings
+
+REST_FRAMEWORK = {
+    'DEFAULT_RENDERER_CLASSES': (
+        'rest_framework.renderers.JSONRenderer',
+    ),
+    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
+    'PAGE_SIZE': 100
+}
+
+#
+# GrimoireLab uses RQ to run background and async jobs.
+# You'll HAVE TO set the next parameters in order to run
+# them in the background.
+#
+# You can define the names of the RQ queues using the following
+# environment variables.
+#
+# Take into account RQ uses Redis database. You have more
+# info about these parameters on the following link:
+#
+# https://github.com/rq/django-rq
+#
+
+Q_PERCEVAL_JOBS = os.environ.get('GRIMOIRELAB_Q_PERCEVAL_JOBS', 'default')
+Q_STORAGE_ITEMS = os.environ.get('GRIMOIRELAB_Q_STORAGE_ITEMS', 'items')
+Q_EVENTS = os.environ.get('GRIMOIRELAB_Q_EVENTS', 'events')
+
+_RQ_DATABASE = {
+    'HOST': os.environ.get('GRIMOIRELAB_REDIS_HOST', '127.0.0.1'),
+    'PORT': os.environ.get('GRIMOIRELAB_REDIS_PORT', 6379),
+    'PASSWORD': os.environ.get('GRIMOIRELAB_REDIS_PASSWORD', ''),
+    'DB': os.environ.get('GRIMOIRELAB_REDIS_DB', 0),
+}
+
+RQ_QUEUES = {
+    Q_PERCEVAL_JOBS: _RQ_DATABASE,
+    Q_STORAGE_ITEMS: _RQ_DATABASE,
+    Q_EVENTS: _RQ_DATABASE,
+}
+
+RQ = {
+    'JOB_CLASS': 'grimoirelab.core.scheduler.jobs.GrimoireLabJob',
+    'WORKER_CLASS': 'grimoirelab.core.scheduler.worker.GrimoireLabWorker',
+}
+
+#
+# Task default configuration
+#
+
+GRIMOIRELAB_JOB_INTERVAL = int(os.environ.get('GRIMOIRELAB_JOB_INTERVAL', 60 * 60 * 2))
+GRIMOIRELAB_JOB_MAX_RETRIES = int(os.environ.get('GRIMOIRELAB_JOB_MAX_RETRIES', 5))
+GRIMOIRELAB_JOB_RESULT_TTL = int(os.environ.get('GRIMOIRELAB_JOB_RESULT_TTL', 300))
+GRIMOIRELAB_JOB_TIMEOUT = int(os.environ.get('GRIMOIRELAB_JOB_TIMEOUT', -1))
+
+GIT_STORAGE_PATH = os.environ.get('GRIMOIRELAB_GIT_PATH', '~/.perceval')

grimoirelab/core/models.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import annotations
+
+import typing
+
+from django.db.models import (
+    DateTimeField,
+    Model
+)
+from grimoirelab_toolkit.datetime import datetime_utcnow
+
+
+if typing.TYPE_CHECKING:
+    import datetime
+
+
+# Innodb and utf8mb4 can only index 191 characters
+# For more information regarding this topic see:
+# https://dev.mysql.com/doc/refman/5.5/en/charset-unicode-conversion.html
+MAX_SIZE_CHAR_INDEX = 191
+MAX_SIZE_CHAR_FIELD = 128
+
+
+class CreationDateTimeField(DateTimeField):
+    """Field automatically set to the current date when an object is created."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        kwargs.setdefault('editable', False)
+        kwargs.setdefault('default', datetime_utcnow)
+        super().__init__(*args, **kwargs)
+
+
+class LastModificationDateTimeField(DateTimeField):
+    """Field automatically set to the current date on each save() call."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        kwargs.setdefault('editable', False)
+        kwargs.setdefault('default', datetime_utcnow)
+        super().__init__(*args, **kwargs)
+
+    def pre_save(self, model_instance: Model, add: bool) -> datetime.datetime:
+        value = datetime_utcnow()
+        setattr(model_instance, self.attname, value)
+        return value
+
+
+class BaseModel(Model):
+    """Base abstract model for all models.
+
+    This model provides the common fields for all GrimoireLab Core
+    models.
+
+    The fields `created_at` and `last_modified` are automatically
+    set to the current date when the object is created or updated.
+    """
+    created_at = CreationDateTimeField()
+    last_modified = LastModificationDateTimeField()
+
+    class Meta:
+        abstract = True

grimoirelab/core/runner/cmd.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import annotations
+
+import typing
+import os
+
+import click
+import django.core.wsgi
+
+from .commands.admin import admin
+from .commands.run import run
+
+if typing.TYPE_CHECKING:
+    from click import Context
+
+
+@click.group()
+@click.option('--config', 'cfg',
+              envvar='GRIMOIRELAB_CONFIG',
+              default='grimoirelab.core.config.settings',
+              show_default=True,
+              help="Configuration module in Python path syntax")
+@click.pass_context
+def grimoirelab(ctx: Context, cfg: str):
+    """Toolset for software development analytics.
+
+    GrimoireLab is a set of tools and a platform to retrieve, analyze,
+    and provide insights about data coming from software development
+    repositories. With this command, you'll be able to configure and
+    run its different services.
+
+    It requires to pass a configuration file module using
+    the Python path syntax (e.g. grimoirelab.core.config.settings).
+    Take into account the configuration should be accessible by your
+    PYTHON_PATH. You can also use the environment variable
+    GRIMOIRELAB_CONFIG to define the config location.
+    """
+    env = os.environ
+
+    if cfg:
+        env['DJANGO_SETTINGS_MODULE'] = cfg
+        ctx.ensure_object(dict)
+        ctx.obj["cfg"] = cfg
+    else:
+        raise click.ClickException(
+            "Configuration file not given. "
+            "Set it with '--config' option "
+            "or 'GRIMOIRELAB_CONFIG' env variable."
+        )
+
+    _ = django.core.wsgi.get_wsgi_application()
+
+
+grimoirelab.add_command(admin)
+grimoirelab.add_command(run)

grimoirelab/core/runner/commands/admin.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import annotations
+
+import typing
+
+import click
+import django.core
+import django_rq
+
+if typing.TYPE_CHECKING:
+    from click import Context
+
+
+@click.group()
+@click.pass_context
+def admin(ctx: Context):
+    """GrimoireLab administration tool.
+
+    This swiss army knife tool allows to run administrative tasks to
+    configure, initialize, update, or check the service.
+    """
+    pass
+
+
+@admin.command()
+def setup():
+    """Run initialization tasks to configure GrimoireLab.
+
+    This command will create the database, tables and apply the
+    defined migrations and fixtures.
+    """
+    _setup()
+
+
+def _setup():
+    """Setup GrimoireLab"""
+
+    _create_database()
+    _setup_database()
+    _install_static_files()
+
+    click.secho("\nGrimoirelab configuration completed", fg='bright_cyan')
+
+
+def _create_database(database: str = 'default', db_name: str | None = None):
+    """Create an empty database"""
+
+    import MySQLdb
+    from django.conf import settings
+
+    db_params = settings.DATABASES[database]
+    db_name = db_name if db_name else db_params['NAME']
+
+    click.secho("## GrimoireLab database creation\n", fg='bright_cyan')
+
+    try:
+        cursor = MySQLdb.connect(
+            user=db_params['USER'],
+            password=db_params['PASSWORD'],
+            host=db_params['HOST'],
+            port=int(db_params['PORT'])
+        ).cursor()
+        cursor.execute(
+            f"CREATE DATABASE IF NOT EXISTS {db_name} "
+            "CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci;"
+        )
+    except MySQLdb.DatabaseError as exc:
+        msg = f"Error creating database '{db_name}' for '{database}': {exc}."
+        raise click.ClickException(msg)
+
+    click.echo(f"GrimoireLab database '{db_name}' for '{database}' created.\n")
+
+
+def _setup_database(database: str = 'default'):
+    """Apply migrations and fixtures to the database"""
+
+    click.secho(f"## GrimoireLab database setup for {database}\n",
+                fg='bright_cyan')
+
+    django.core.management.call_command('migrate', database=database)
+
+    click.echo()
+
+
+def _install_static_files():
+    """Collect static files and install them."""
+
+    click.secho('## GrimoireLab static files installation\n',
+                fg='bright_cyan')
+
+    django.core.management.call_command('collectstatic',
+                                        ignore=['admin', 'rest_framework'],
+                                        clear=True,
+                                        interactive=False)
+
+    click.echo()
+
+
+@admin.group()
+@click.pass_context
+def queues(ctx: Context):
+    """Manage the GrimoireLab job queues."""
+
+    pass
+
+
+@queues.command(name='list')
+def list_jobs():
+    """List the jobs in the queues.
+
+    Print the jobs classified by their status.
+    """
+    queue = django_rq.get_queue()
+
+    jobs = {
+        'Started': [jid for jid in queue.started_job_registry.get_job_ids()],
+        'Scheduled': [jid for jid in queue.scheduled_job_registry.get_job_ids()],
+        'Failed': [jid for jid in queue.failed_job_registry.get_job_ids()],
+        'Deferred': [jid for jid in queue.deferred_job_registry.get_job_ids()],
+        'Canceled': [jid for jid in queue.canceled_job_registry.get_job_ids()],
+        'Finished': [jid for jid in queue.finished_job_registry.get_job_ids()],
+    }
+    for key, value in jobs.items():
+        click.echo(key)
+        click.echo(value)
+        click.echo()
+
+
+@queues.command(name='purge')
+def remove_jobs():
+    """Remove the jobs from the queues."""
+
+    queue = django_rq.get_queue()
+
+    registries = {
+        'Started': queue.failed_job_registry,
+        'Scheduled': queue.finished_job_registry,
+        'Failed': queue.scheduled_job_registry,
+        'Deferred': queue.started_job_registry,
+        'Canceled': queue.canceled_job_registry,
+        'Finished': queue.deferred_job_registry
+    }
+    for key, registry in registries.items():
+        for jid in registry.get_job_ids():
+            registry.remove(jid)
+        click.echo(f"{key} registry removed.")