ingestr 0.14.6__tar.gz → 0.14.8__tar.gz

{ingestr-0.14.6 → ingestr-0.14.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.6
+Version: 0.14.8
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.6 → ingestr-0.14.8}/docs/supported-sources/jira.md

@@ -53,6 +53,21 @@ Jira source allows ingesting the following tables:
 
 Use these as the `--source-table` parameter in the `ingestr ingest` command.
 
+## Filtering archived projects
+
+`projects`, `project_versions` and `project_components` can be suffixed with `:skip_archived` to filter out archived projects.
+
+For instance:
+```bash
+ingestr ingest \
+  --source-uri 'jira://company.atlassian.net?email=user@company.com&api_token=ATATT3xFfGF0...' \
+  --source-table 'project_versions:skip_archived' \
+  --dest-uri duckdb:///jira.duckdb \
+  --dest-table 'dest.live_project_versions'
+```
+
+Will only load versions of non-archived projects.
+
 ## Incremental Loading
 
 The `issues` table supports incremental loading based on the `updated` field. This means subsequent runs will only fetch issues that have been modified since the last run, making the data ingestion more efficient for large Jira instances.

ingestr-0.14.8/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.8"

{ingestr-0.14.6 → ingestr-0.14.8}/ingestr/src/errors.py

@@ -1,3 +1,6 @@
+import requests
+
+
 class MissingValueError(Exception):
     def __init__(self, value, source):
         super().__init__(f"{value} is required to connect to {source}")
@@ -16,3 +19,8 @@ class InvalidBlobTableError(Exception):
             f"Invalid source table for {source} "
             "Ensure that the table is in the format {bucket-name}/{file glob}"
         )
+
+
+class HTTPError(Exception):
+    def __init__(self, source: requests.HTTPError):
+        super().__init__(f"HTTP {source.response.status_code}: {source.response.text}")

{ingestr-0.14.6 → ingestr-0.14.8}/ingestr/src/freshdesk/freshdesk_client.py

@@ -8,6 +8,8 @@ import pendulum
 from dlt.common.typing import TDataItem
 from dlt.sources.helpers import requests
 
+from ingestr.src.errors import HTTPError
+
 
 class FreshdeskClient:
     """
@@ -62,7 +64,7 @@ class FreshdeskClient:
                 else:
                     # If the error is not a rate limit (429), raise the exception to be
                     # handled elsewhere or stop execution
-                    raise
+                    raise HTTPError(e) from e
 
     def paginated_response(
         self,

{ingestr-0.14.6 → ingestr-0.14.8}/ingestr/src/sources.py

@@ -1849,6 +1849,14 @@ class JiraSource:
         if not api_token:
             raise ValueError("api_token is required for connecting to Jira")
 
+        flags = {
+            "skip_archived": False,
+        }
+        if ":" in table:
+            table, rest = table.split(":", 1)  # type: ignore
+            for k in rest.split(":"):
+                flags[k] = True
+
         if table not in self.resources:
             raise ValueError(
                 f"Resource '{table}' is not supported for Jira source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
@@ -1863,6 +1871,8 @@ class JiraSource:
         dlt.secrets["sources.jira_source.api_token"] = api_token[0]
 
         src = jira_source()
+        if flags["skip_archived"]:
+            src.projects.add_filter(lambda p: not p.get("archived", False))
         return src.with_resources(table)
 
 

ingestr-0.14.6/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.14.6"