jf-ingest 0.0.151.dev0__tar.gz → 0.0.152.dev0__tar.gz

{jf_ingest-0.0.151.dev0 → jf_ingest-0.0.152.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jf-ingest
-Version: 0.0.151.dev0
+Version: 0.0.152.dev0
 Summary: library used for ingesting jira data
 Author-email: jellyfish-oss <oss@jellyfish.co>
 License: MIT

{jf_ingest-0.0.151.dev0 → jf_ingest-0.0.152.dev0}/jf_ingest/adaptive_throttler.py

@@ -219,7 +219,8 @@ class AdaptiveThrottler:
 
             with self._lock:
                 self._response_time_histogram.measure(
-                    value=response_time, attributes=self.metrics_attrs
+                    value=response_time,
+                    attributes=self.metrics_attrs,
                 )
                 self._total_requests_counter.add(amount=1, attributes=self.metrics_attrs)
 
@@ -248,10 +249,14 @@ class AdaptiveThrottler:
                     )
 
                     self._request_rate_gauge.measure(
-                        value=self._request_rate, attributes=self.metrics_attrs
+                        value=self._request_rate,
+                        attributes=self.metrics_attrs,
+                        record_immediate=True,
                     )
-                    self._backoff_counter.add(amount=1, attributes=self.metrics_attrs)
-                    self._backoff_time_histogram.measure(value=backoff_time)
+                    self._backoff_counter.add(
+                        amount=1, attributes=self.metrics_attrs, record_immediate=True
+                    )
+                    self._backoff_time_histogram.measure(value=backoff_time, record_immediate=True)
 
                     time.sleep(backoff_time)
                 elif self._request_rate < self._initial_request_rate:
@@ -272,9 +277,13 @@ class AdaptiveThrottler:
                         self._response_within_threshold_counter = 0
 
                         self._request_rate_gauge.measure(
-                            value=self._request_rate, attributes=self.metrics_attrs
+                            value=self._request_rate,
+                            attributes=self.metrics_attrs,
+                            record_immediate=True,
+                        )
+                        self._reverse_backoff_counter.add(
+                            amount=1, attributes=self.metrics_attrs, record_immediate=True
                         )
-                        self._reverse_backoff_counter.add(amount=1, attributes=self.metrics_attrs)
 
                         send_to_agent_log_file(
                             f"Request rate increased to {self._request_rate:.4f} rps",

{jf_ingest-0.0.151.dev0 → jf_ingest-0.0.152.dev0}/jf_ingest/jf_git/clients/github.py

@@ -34,6 +34,8 @@ GIT_DEFAULT_API_ENDPOINT = 'https://api.github.com'
 
 GQL_RATE_LIMIT_QUERY_BLOCK = "rateLimit {remaining, resetAt}"
 
+GITHUB_STATUSES_TO_RETRY_ON = tuple(list(DEFAULT_HTTP_CODES_TO_RETRY_ON) + [403])
+
 
 def parse_date(dt):
     if dt is None:
@@ -237,10 +239,10 @@ class GithubClient:
                 if e.response.status_code == 401:
                     self._update_token()
                     continue
-                # We can get transient 403 level errors that have to do with rate limiting,
+                # We can get transient errors that have to do with rate limiting,
                 # but aren't directly related to the above GqlRateLimitedExceptionInner logic.
                 # Do a simple retry loop here
-                elif e.response.status_code == 403:
+                elif e.response.status_code in GITHUB_STATUSES_TO_RETRY_ON:
                     pass
                 else:
                     raise
@@ -452,7 +454,7 @@ class GithubClient:
         # HACK: A 403 appears to happen after we have been
         # rate-limited when hitting certain URLs. Add 403s
         # to HTTP Codes to retry
-        statuses_to_retry = list(DEFAULT_HTTP_CODES_TO_RETRY_ON) + [403]
+        statuses_to_retry = GITHUB_STATUSES_TO_RETRY_ON
         result = retry_for_status(self.session.get, url, statuses_to_retry=statuses_to_retry)
         result.raise_for_status()
         return result.json()
@@ -471,7 +473,7 @@ class GithubClient:
             # HACK: A 403 appears to happen after we have been
             # rate-limited when hitting certain URLs. Add 403s
             # to HTTP Codes to retry
-            statuses_to_retry = list(DEFAULT_HTTP_CODES_TO_RETRY_ON) + [403]
+            statuses_to_retry = GITHUB_STATUSES_TO_RETRY_ON
             result: requests.Response = retry_for_status(
                 self.session.get, url, statuses_to_retry=statuses_to_retry
             )
@@ -495,7 +497,7 @@ class GithubClient:
         # HACK: A 403 appears to happen after we have been
         # rate-limited when hitting certain URLs. Add 403s
         # to HTTP Codes to retry
-        statuses_to_retry = list(DEFAULT_HTTP_CODES_TO_RETRY_ON) + [403]
+        statuses_to_retry = GITHUB_STATUSES_TO_RETRY_ON
         while True:
             url = f'{self.rest_api_url}/repos/{org_login}/{repo_name}/labels?per_page={page_size}&page={page_number}'
             result = retry_for_status(self.session.get, url, statuses_to_retry=statuses_to_retry)

{jf_ingest-0.0.151.dev0 → jf_ingest-0.0.152.dev0}/jf_ingest/telemetry/metrics.py

@@ -42,19 +42,28 @@ class JellyMetricBase:
         if not _METRIC_EXPORTER:
             logger.info("No exporter configured. Using ConsoleMetricExporter for NoOp metrics.")
             self._reader = PeriodicExportingMetricReader(ConsoleMetricExporter())
-            self._provider: Union[MeterProvider, NoOpMeterProvider] = NoOpMeterProvider()
+            self._provider = NoOpMeterProvider()
         else:
             self._reader = PeriodicExportingMetricReader(_METRIC_EXPORTER)
             self._provider = MeterProvider(metric_readers=[self._reader])
 
-        atexit.register(self.shutdown)
+        atexit.register(self.shutdown_reader)
 
     def _get_meter(self):
         return self._provider.get_meter(__name__)
 
-    def shutdown(self):
-        logger.info("Shutting down metrics reader and exporting all pending metrics.")
-        self._reader.shutdown()
+    def shutdown_reader(self):
+        """Ensures the reader is flushed and shutdown at exit."""
+        current_log_level = logger.level
+
+        # The OTel library will log a warning message if .shutdown is called on a shutdown reader, but there isn't any
+        # way to check if the reader is already shutdown. This is a workaround to suppress the warning message.
+        try:
+            logger.setLevel(logging.ERROR)
+            self._reader.force_flush()
+            self._reader.shutdown()
+        finally:
+            logger.setLevel(current_log_level)
 
 
 class JellyGauge(JellyMetricBase):
@@ -98,7 +107,6 @@ class JellyGauge(JellyMetricBase):
         Set record_immediate to be true if you want to immediately flush the reader after this metric is collected -- should only
         be used in short-lived process situations.
         """
-
         logger.debug(
             f"Received measure call for {self._name}, reporting measurement {value}, {attributes}"
         )
@@ -108,6 +116,7 @@ class JellyGauge(JellyMetricBase):
             self._internal_gauge = self._meter.create_observable_gauge(
                 name=self._name, callbacks=[self._callback]
             )
+
         if record_immediate:
             self._reader.collect()
 
@@ -115,7 +124,6 @@ class JellyGauge(JellyMetricBase):
         """
         Callback function for async gauge. Clears the observations as they're reported.
         """
-
         logger.debug(
             f"Received callback function call for {self._name}, reporting observations {self._observations}"
         )

{jf_ingest-0.0.151.dev0 → jf_ingest-0.0.152.dev0}/jf_ingest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: jf-ingest
-Version: 0.0.151.dev0
+Version: 0.0.152.dev0
 Summary: library used for ingesting jira data
 Author-email: jellyfish-oss <oss@jellyfish.co>
 License: MIT

{jf_ingest-0.0.151.dev0 → jf_ingest-0.0.152.dev0}/tests/jf_git/github/test_github_gql_client.py

@@ -21,6 +21,13 @@ TEST_BASE_URL = 'https://gav.com/gql'
 GQL_DATETIME_STR = "2024-12-29T12:00:58Z"
 
 
+@pytest.fixture()
+def client() -> GithubClient:
+    # Mock out a Github GQL Client
+    github_auth_config = GithubAuthConfig(company_slug='a company', base_url=TEST_BASE_URL, token='test')
+    return GithubClient(github_auth_config)
+
+
 def test_gql_page_results_block():
     """This is a sanity check that the GQL_PAGE_INFO_BLOCK and GQL_RATE_LIMIT_QUERY_BLOCK
     are valid. This are pretty vague smoke tests... it's unlikely anybody would mess with
@@ -53,12 +60,9 @@ def test_git_auth_config_base_url():
     assert client.rest_api_url == github_api_url
     assert client.gql_base_url == f'{github_api_url}/graphql'
 
-def test_get_rate_limit(requests_mock: requests_mock.Mocker):
+def test_get_rate_limit(requests_mock: requests_mock.Mocker, client: GithubClient):
     # Mock classes/data
     session = requests.Session()
-    # Mock out a Github GQL Client
-    github_auth_config = GithubAuthConfig(company_slug='a company', base_url=TEST_BASE_URL, session=session, token='test')
-    client = GithubClient(github_auth_config)
     client.session = session
     
     def _test_get_rate_limit_inner(remaining_rate_limit: int, reset_at_rate_limit_str: str):
@@ -89,7 +93,7 @@ def test_get_rate_limit(requests_mock: requests_mock.Mocker):
         _test_get_rate_limit_inner('cat', GQL_DATETIME_STR)
 
 
-def test_get_raw_gql_result_simple(requests_mock: requests_mock.Mocker):
+def test_get_raw_gql_result_simple(requests_mock: requests_mock.Mocker, client: GithubClient):
     # Test Data
     test_query_body = "test_query_body"
     # Mock classes/data
@@ -97,9 +101,6 @@ def test_get_raw_gql_result_simple(requests_mock: requests_mock.Mocker):
     session_mock_caller_data = {'query': test_query_body}
     session_mock_return_data = {'data': {'test': {'remaining': 'test', 'resetAt': 'test'}}}
     requests_mock.post(url=f'{TEST_BASE_URL}/graphql', json=session_mock_return_data)
-    # Mock out a Github GQL Client
-    github_auth_config = GithubAuthConfig(company_slug='a company', base_url=TEST_BASE_URL, session=session, token='test')
-    client = GithubClient(github_auth_config)
     client.session = session
 
     # call function
@@ -109,13 +110,10 @@ def test_get_raw_gql_result_simple(requests_mock: requests_mock.Mocker):
     assert requests_mock.last_request.json() == session_mock_caller_data
 
 
-def test_get_raw_gql_result_force_error_raise(mocker, requests_mock: requests_mock.Mocker):
+def test_get_raw_gql_result_force_error_raise(mocker, requests_mock: requests_mock.Mocker, client: GithubClient):
     mocker.patch('time.sleep', return_value=None)
     # Mock classes/data
     session = requests.Session()
-    # Mock out a Github GQL Client
-    github_auth_config = GithubAuthConfig(company_slug='A Company', base_url=TEST_BASE_URL, session=session, token='test')
-    client = GithubClient(github_auth_config)
     client.session = session
 
     requests_mock.post(
@@ -153,7 +151,7 @@ def test_get_raw_gql_result_force_error_raise(mocker, requests_mock: requests_mo
         client.get_raw_result(query_body='', max_attempts=2)
 
 
-def test_get_raw_gql_result_successful_retry(mocker, requests_mock: requests_mock.Mocker):
+def test_get_raw_gql_result_successful_retry(mocker, requests_mock: requests_mock.Mocker, client: GithubClient):
     mocker.patch('time.sleep', return_value=None)
     session = requests.Session()
     successful_data = {'success': True}
@@ -169,15 +167,57 @@ def test_get_raw_gql_result_successful_retry(mocker, requests_mock: requests_moc
         ],
     )
 
-    # Mock out a Github GQL Client
-    github_auth_config = GithubAuthConfig(company_slug='A Company', base_url=TEST_BASE_URL, session=session, token='test')
-    client = GithubClient(github_auth_config)
     client.session = session
 
     data = client.get_raw_result(query_body='', max_attempts=10)
     assert data == successful_data
 
 
+def test_github_client_error_raise_403(mocker, requests_mock: requests_mock.Mocker, client: GithubClient):
+    mocker.patch('time.sleep', return_value=None)
+    session = requests.Session()
+    successful_data = {'success': True}
+    requests_mock.post(
+        url=f'{TEST_BASE_URL}/graphql',
+        response_list=[
+            {'status_code': 403},
+        ],
+    )
+
+    # Mock out a Github GQL Client
+    client.session = session
+
+    retry_count = 10
+    with pytest.raises(requests.HTTPError):
+        client.get_raw_result(query_body='', max_attempts=retry_count)
+        assert requests_mock.call_count == retry_count
+
+def test_github_client_error_resolves_after_retry(mocker, requests_mock: requests_mock.Mocker, client: GithubClient):
+    mocker.patch('time.sleep', return_value=None)
+    session = requests.Session()
+    successful_data = {'mantra': 'never give UP!'}
+    response_list = [
+        # A Beautiful Error Medley
+        {'status_code': 403},
+        {'status_code': 429},
+        {'status_code': 500},
+        {'status_code': 502},
+        {'status_code': 503},
+        {'status_code': 504},
+        {'json': successful_data}
+    ]
+    retry_count = len(response_list)
+    requests_mock.post(
+        url=f'{TEST_BASE_URL}/graphql',
+        response_list=response_list,
+    )
+
+    # Mock out a Github GQL Client
+    client.session = session
+
+    assert client.get_raw_result(query_body='', max_attempts=retry_count) == successful_data
+    assert requests_mock.call_count == retry_count
+
 def test_page_results_gql(mocker, requests_mock: requests_mock.Mocker):
     session = requests.Session()
     # Mock out a Github GQL Client