sortinghat-eclipse-foundation 0.1.2rc1__tar.gz → 0.2.0__tar.gz

{sortinghat_eclipse_foundation-0.1.2rc1 → sortinghat_eclipse_foundation-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sortinghat-eclipse-foundation
-Version: 0.1.2rc1
+Version: 0.2.0
 Summary: SortingHat backend to import identities from the Eclipse Foundation
 License-File: AUTHORS
 License-File: LICENSE.txt
@@ -22,7 +22,7 @@ SortingHat backend to import identities from Eclipse Foundation
 
 ## Requirements
 
- - Python >= 3.9
+ - Python >= 3.10
 
 You will also need some other libraries for running the tool, you can find the
 whole list of dependencies in [pyproject.toml](pyproject.toml) file.

{sortinghat_eclipse_foundation-0.1.2rc1 → sortinghat_eclipse_foundation-0.2.0}/README.md

@@ -4,7 +4,7 @@ SortingHat backend to import identities from Eclipse Foundation
 
 ## Requirements
 
- - Python >= 3.9
+ - Python >= 3.10
 
 You will also need some other libraries for running the tool, you can find the
 whole list of dependencies in [pyproject.toml](pyproject.toml) file.

{sortinghat_eclipse_foundation-0.1.2rc1 → sortinghat_eclipse_foundation-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sortinghat-eclipse-foundation"
-version = "0.1.2-rc.1"
+version = "0.2.0"
 description = "SortingHat backend to import identities from the Eclipse Foundation"
 authors = [
     {name = "Bitergia Developers"}

sortinghat_eclipse_foundation-0.2.0/sortinghat/core/importer/backends/_version.py

@@ -0,0 +1,2 @@
+# File auto-generated by semverup on 2025-11-11 14:58:49.929536
+__version__ = "0.2.0"

{sortinghat_eclipse_foundation-0.1.2rc1 → sortinghat_eclipse_foundation-0.2.0}/sortinghat/core/importer/backends/eclipse.py

@@ -16,10 +16,13 @@
 #
 
 import logging
+import time
 
 import dateutil.relativedelta
 import requests
 
+from concurrent.futures import ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED
+
 from django.conf import settings
 from django.db.models import (Q, Subquery)
 
@@ -46,6 +49,11 @@ from sortinghat.core import models as sh_models
 ECLIPSE_SOURCE = "eclipsefdn"
 GITHUB_SOURCE = "github"
 
+# Parallel processing
+MAX_WORKERS = 8
+MAX_QUEUE_SIZE = 100
+
+REQUEST_TIMEOUT = 30
 
 logger = logging.getLogger(__name__)
 
@@ -73,7 +81,7 @@ class EclipseFoundationAccountsImporter(IdentitiesImporter):
     """
     NAME = "EclipseFoundation"
 
-    def __init__(self, ctx, url, from_date=None):
+    def __init__(self, ctx, url, from_date=None, from_page=1):
         super().__init__(ctx, url)
 
         min_date = datetime_utcnow() - dateutil.relativedelta.relativedelta(years=1)
@@ -85,6 +93,11 @@ class EclipseFoundationAccountsImporter(IdentitiesImporter):
         else:
             self.from_date = from_date
 
+        if not from_page:
+            self.from_page = 1
+        else:
+            self.from_page = int(from_page)
+
         if self.from_date < min_date:
             msg = (
                 "Invalid 'from_date' value. It can only import identities updated since a year ago."
@@ -104,68 +117,99 @@ class EclipseFoundationAccountsImporter(IdentitiesImporter):
 
         epoch = int(self.from_date.timestamp())
 
-        # Fetch accounts pages
-        for account in client.fetch_accounts(epoch=epoch):
-            ef_profile = client.fetch_account_profile(account['name'])
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            pending = set()
+
+            for account in client.fetch_accounts(epoch=epoch, from_page=self.from_page):
+                future = executor.submit(self.process_account, client, account)
+                pending.add(future)
+
+                # Wait to complete when reaching max queue size
+                if len(pending) >= MAX_QUEUE_SIZE:
+                    done, pending = wait(pending, return_when=FIRST_COMPLETED, timeout=120)
+                    if not done:
+                        raise TimeoutError("Timeout waiting for Eclipse account processing")
+                    for future in done:
+                        try:
+                            individual = future.result()
+                            if individual:
+                                yield individual
+                        except Exception as exc:
+                            logger.error(f"Error processing Eclipse account {account}; error={exc}")
+
+        # Process remaining futures
+        for future in as_completed(pending, timeout=600):
+            try:
+                individual = future.result()
+                if individual:
+                    yield individual
+            except Exception as exc:
+                logger.error(f"Error processing Eclipse account; error={exc}")
 
-            if not ef_profile:
-                continue
+    @staticmethod
+    def process_account(client, account):
+        """Process a single Eclipse account to create an Individual."""
 
-            individual = Individual(uuid=ef_profile['uid'])
+        ef_profile = client.fetch_account_profile(account['name'])
 
-            name = ef_profile['first_name'] + ' ' + ef_profile['last_name']
-            email = ef_profile['mail']
+        if not ef_profile:
+            return None
 
-            prf = Profile()
-            prf.name = name
-            prf.email = email
+        individual = Individual(uuid=ef_profile['uid'])
 
-            individual.profile = prf
+        name = ef_profile['first_name'] + ' ' + ef_profile['last_name']
+        email = ef_profile['mail']
 
-            eclipse_id = Identity(
-                source=ECLIPSE_SOURCE,
+        prf = Profile()
+        prf.name = name
+        prf.email = email
+
+        individual.profile = prf
+
+        eclipse_id = Identity(
+            source=ECLIPSE_SOURCE,
+            name=name,
+            email=email,
+            username=ef_profile['name'],
+        )
+        individual.identities.append(eclipse_id)
+
+        if ef_profile['github_handle']:
+            idt = Identity(
+                source=GITHUB_SOURCE,
                 name=name,
+                username=ef_profile['github_handle'],
                 email=email,
-                username=ef_profile['name'],
             )
-            individual.identities.append(eclipse_id)
-
-            if ef_profile['github_handle']:
-                idt = Identity(
-                    source=GITHUB_SOURCE,
-                    name=name,
-                    username=ef_profile['github_handle'],
-                    email=email,
-                )
-                individual.identities.append(idt)
+            individual.identities.append(idt)
 
-            # Fetch enrollments for the identity. If no enrollment is set
-            # use the organization field from the profile, if set.
-            employment_history = client.fetch_employment_history(account['name'])
+        # Fetch enrollments for the identity. If no enrollment is set
+        # use the organization field from the profile, if set.
+        employment_history = client.fetch_employment_history(account['name'])
 
-            if employment_history:
-                for employment in employment_history:
-                    org = Organization(name=employment['organization_name'])
-                    start, end = None, None
+        if employment_history:
+            for employment in employment_history:
+                org = Organization(name=employment['organization_name'])
+                start, end = None, None
 
-                    if employment['start']:
-                        start = str_to_datetime(employment['start'])
-                    if employment['end']:
-                        end = str_to_datetime(employment['end'])
+                if employment['start']:
+                    start = str_to_datetime(employment['start'])
+                if employment['end']:
+                    end = str_to_datetime(employment['end'])
 
-                    enr = Enrollment(org, start=start, end=end)
-                    individual.enrollments.append(enr)
+                enr = Enrollment(org, start=start, end=end)
+                individual.enrollments.append(enr)
 
-            if not individual.enrollments:
-                company = ef_profile.get('org', None)
-                if company:
-                    org = Organization(name=company)
-                    enr = Enrollment(org)
-                    individual.enrollments.append(enr)
+        if not individual.enrollments:
+            company = ef_profile.get('org', None)
+            if company:
+                org = Organization(name=company)
+                enr = Enrollment(org)
+                individual.enrollments.append(enr)
 
-            logger.info(f"Eclipse account processed; account={account['name']}; changed={account['changed']}")
+        logger.info(f"Eclipse account processed; account={account['name']}; changed={account['changed']}")
 
-            yield individual
+        return individual
 
     def post_process_individual(self, individual, uuid):
         """Post processing for Eclipse identities.
@@ -235,10 +279,10 @@ class EclipseFoundationAPIClient:
 
         self.token = None
 
-    def fetch_accounts(self, epoch):
+    def fetch_accounts(self, epoch, from_page=1):
         """Fetch accounts updated from a given UNIX time."""
 
-        page = 1
+        page = from_page
         total_accounts = 0
 
         logger.info(f"Fetching accounts from API; url={self.ECLIPSE_ACCOUNTS_URL}, epoch={epoch}")
@@ -249,10 +293,14 @@ class EclipseFoundationAPIClient:
                 'since': epoch,
                 'page': page,
                 'pagesize': 100,
+                'sortby': 'uid',
             }
 
             logger.debug(f"Fetching accounts from API; url={url}, params={params}")
             data = self._fetch(url, params=params)
+            if not data:
+                logger.error(f"No data returned from API; url={url}, params={params}")
+                continue
 
             for account in data['result']:
                 yield account
@@ -316,7 +364,7 @@ class EclipseFoundationAPIClient:
 
         while retries < max_retries:
             try:
-                response = requests.get(url, params=params, auth=self.token)
+                response = requests.get(url, params=params, auth=self.token, timeout=REQUEST_TIMEOUT)
             except ExpiredAccessToken:
                 # Refresh token and try again
                 self.login(self.user_id, self.password)
@@ -332,13 +380,14 @@ class EclipseFoundationAPIClient:
             elif 500 <= response.status_code < 600:
                 # Errors could have been related to server overloading
                 retries += 1
+                time.sleep(2 ** retries)
             else:
                 response.raise_for_status()
 
-        response = requests.get(url, params=params, auth=self.token)
+        response = requests.get(url, params=params, auth=self.token, timeout=REQUEST_TIMEOUT)
         response.raise_for_status()
 
-        return response
+        return response.json()
 
     def _authenticate(self, client_id, client_secret, scope):
         """Authenticate using OAuth2.

sortinghat_eclipse_foundation-0.1.2rc1/sortinghat/core/importer/backends/_version.py

@@ -1,2 +0,0 @@
-# File auto-generated by semverup on 2025-10-31 13:33:01.052904
-__version__ = "0.1.2-rc.1"