unstructured-ingest 1.0.46.dev0__py3-none-any.whl → 1.0.48__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.46-dev0"  # pragma: no cover
+__version__ = "1.0.48"  # pragma: no cover

unstructured_ingest/processes/connectors/delta_table.py

@@ -134,8 +134,15 @@ class DeltaTableUploader(Uploader):
 
                 response = s3_client.get_bucket_location(Bucket=bucket_name)
 
-                if self.connection_config.aws_region != response.get("LocationConstraint"):
-                    raise ValueError("Wrong AWS Region was provided.")
+                bucket_region = _normalize_location_constraint(response.get("LocationConstraint"))
+
+                if self.connection_config.aws_region != bucket_region:
+                    raise ValueError(
+                        "Wrong AWS region provided: bucket "
+                        f"'{bucket_name}' resides in '{bucket_region}', "
+                        "but configuration specifies "
+                        f"'{self.connection_config.aws_region}'."
+                    )
 
             except Exception as e:
                 logger.error(f"failed to validate connection: {e}", exc_info=True)
@@ -270,6 +277,29 @@ class DeltaTableUploader(Uploader):
         self.upload_dataframe(df=df, file_data=file_data)
 
 
+def _normalize_location_constraint(location: Optional[str]) -> str:
+    """Return canonical AWS region name for a LocationConstraint value.
+
+    The S3 GetBucketLocation operation returns `null` (`None`) for buckets in
+    the legacy `us-east-1` region and `EU` for very old buckets that were
+    created in the historical `EU` region (now `eu-west-1`). For every other
+    region the API already returns the correct AWS region string. This helper
+    normalises the legacy values so callers can reliably compare regions.
+
+    Args:
+        location: The LocationConstraint value returned by the S3 GetBucketLocation operation.
+
+    Returns:
+        The canonical AWS region name for the given location constraint.
+    """
+
+    if location is None:
+        return "us-east-1"
+    if location == "EU":
+        return "eu-west-1"
+    return location
+
+
 delta_table_destination_entry = DestinationRegistryEntry(
     connection_config=DeltaTableConnectionConfig,
     uploader=DeltaTableUploader,

unstructured_ingest/processes/connectors/jira.py

@@ -41,10 +41,17 @@ DEFAULT_R_SEP = "\n"
 class JiraIssueMetadata(BaseModel):
     id: str
     key: str
+    fields: Optional[dict] = None  # Add fields to capture attachment data
 
     def get_project_id(self) -> str:
         return self.key.split("-")[0]
 
+    def get_attachments(self) -> List[dict]:
+        """Extract attachment information from fields"""
+        if self.fields and "attachment" in self.fields:
+            return self.fields["attachment"]
+        return []
+
 
 class FieldGetter(dict):
     def __getitem__(self, key):
@@ -196,15 +203,17 @@ class JiraIndexer(Indexer):
                     yield JiraIssueMetadata.model_validate(issue)
 
     def _get_issues_within_projects(self) -> Generator[JiraIssueMetadata, None, None]:
-        fields = ["key", "id", "status"]
+        fields = ["key", "id", "status", "attachment"]  # Add attachment field
         jql = "project in ({})".format(", ".join(self.index_config.projects))
         jql = self._update_jql(jql)
         logger.debug(f"running jql: {jql}")
         return self.run_jql(jql=jql, fields=fields)
 
-    def _get_issues_within_single_board(self, board_id: str) -> List[JiraIssueMetadata]:
+    def _get_issues_within_single_board(
+        self, board_id: str
+    ) -> Generator[JiraIssueMetadata, None, None]:
         with self.connection_config.get_client() as client:
-            fields = ["key", "id"]
+            fields = ["key", "id", "attachment"]  # Add attachment field
             if self.index_config.status_filters:
                 jql = "status in ({}) ORDER BY id".format(
                     ", ".join([f'"{s}"' for s in self.index_config.status_filters])
@@ -233,23 +242,38 @@ class JiraIndexer(Indexer):
         return jql
 
     def _get_issues_by_keys(self) -> Generator[JiraIssueMetadata, None, None]:
-        fields = ["key", "id"]
+        fields = ["key", "id", "attachment"]  # Add attachment field
         jql = "key in ({})".format(", ".join(self.index_config.issues))
         jql = self._update_jql(jql)
         logger.debug(f"running jql: {jql}")
         return self.run_jql(jql=jql, fields=fields)
 
     def _create_file_data_from_issue(self, issue: JiraIssueMetadata) -> FileData:
-        # Build metadata
+        # Construct relative path and filename first
+        filename = f"{issue.key}.txt"
+        relative_path = str(Path(issue.get_project_id()) / filename)
+
+        # Build metadata with attachments included in record_locator
+        record_locator = {"id": issue.id, "key": issue.key, "full_path": relative_path}
+
+        # Add attachments to record_locator if they exist
+        attachments = issue.get_attachments()
+        if attachments:
+            record_locator["attachments"] = [
+                {
+                    "id": att["id"],
+                    "filename": att["filename"],
+                    "created": att.get("created"),
+                    "mimeType": att.get("mimeType"),
+                }
+                for att in attachments
+            ]
+
         metadata = FileDataSourceMetadata(
             date_processed=str(time()),
-            record_locator=issue.model_dump(),
+            record_locator=record_locator,
         )
 
-        # Construct relative path and filename
-        filename = f"{issue.id}.txt"
-        relative_path = str(Path(issue.get_project_id()) / filename)
-
         source_identifiers = SourceIdentifiers(
             filename=filename,
             fullpath=relative_path,
@@ -400,21 +424,37 @@ class JiraDownloader(Downloader):
         self, attachment_dict: dict, parent_filedata: FileData
     ) -> FileData:
         new_filedata = parent_filedata.model_copy(deep=True)
-        if new_filedata.metadata.record_locator is None:
-            new_filedata.metadata.record_locator = {}
-        new_filedata.metadata.record_locator["parent_issue"] = (
-            parent_filedata.metadata.record_locator["id"]
-        )
+
+        # Create attachment record_locator with parent context
+        attachment_record_locator = {
+            "id": attachment_dict["id"],
+            "filename": attachment_dict["filename"],
+            "created": attachment_dict.get("created"),
+            "mimeType": attachment_dict.get("mimeType"),
+            "parent": {
+                "id": parent_filedata.metadata.record_locator["id"],
+                "key": parent_filedata.metadata.record_locator["key"],
+                "full_path": parent_filedata.source_identifiers.fullpath,
+            },
+        }
+
         # Append an identifier for attachment to not conflict with issue ids
         new_filedata.identifier = "{}a".format(attachment_dict["id"])
-        filename = attachment_dict["filename"]
-        new_filedata.metadata.filesize_bytes = attachment_dict.pop("size", None)
-        new_filedata.metadata.date_created = attachment_dict.pop("created", None)
-        new_filedata.metadata.url = attachment_dict.pop("self", None)
-        new_filedata.metadata.record_locator = attachment_dict
+        filename = f"{attachment_dict['filename']}.{attachment_dict['id']}"
+        new_filedata.metadata.filesize_bytes = attachment_dict.get("size")
+        new_filedata.metadata.date_created = attachment_dict.get("created")
+        new_filedata.metadata.url = attachment_dict.get("self")
+        new_filedata.metadata.record_locator = attachment_record_locator
+        full_path = (
+            Path(parent_filedata.source_identifiers.fullpath).with_suffix("") / Path(filename)
+        ).as_posix()
+        new_filedata.metadata.record_locator["full_path"] = full_path
         new_filedata.source_identifiers = SourceIdentifiers(
             filename=filename,
-            fullpath=(Path(str(attachment_dict["id"])) / Path(filename)).as_posix(),
+            # add issue_parent to the fullpath and rel_path
+            # to ensure that the attachment is saved in the same folder as the parent issue
+            fullpath=full_path,
+            rel_path=full_path,
         )
         return new_filedata
 

{unstructured_ingest-1.0.46.dev0.dist-info → unstructured_ingest-1.0.48.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.46.dev0
+Version: 1.0.48
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.46.dev0.dist-info → unstructured_ingest-1.0.48.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=IKm0h9VzCLTBY4HULhS6HxTXk7DRRvPTI9GEd28jGJM,48
+unstructured_ingest/__version__.py,sha256=oxIYGy4gQ782VEa3kKpqmMWHoZ9FUqF1PhsN2QyWE88,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -68,12 +68,12 @@ unstructured_ingest/processes/connectors/azure_ai_search.py,sha256=szhSRXzUHk0DE
 unstructured_ingest/processes/connectors/chroma.py,sha256=q5_Fu4xb6_W_NyrPxVa3-jVwZLqVdlBNlR4dFvbd7l0,7235
 unstructured_ingest/processes/connectors/confluence.py,sha256=aA2B_FPdAjlVAJtmMldYu6lld2sR-6JL5tWh7yItiwg,22828
 unstructured_ingest/processes/connectors/couchbase.py,sha256=KCHoYDNya9B05NIB5D78zXoizFyfpJRepcYBe1nLSOs,12298
-unstructured_ingest/processes/connectors/delta_table.py,sha256=JrpiX9V-YD1VhExKi6KFwlYatCheSs3t-xB3Td1BVFk,11487
+unstructured_ingest/processes/connectors/delta_table.py,sha256=Y0P-4knPBc7Q8QwlvlDe6ksIKppNY4dBZhC1vwGARi0,12661
 unstructured_ingest/processes/connectors/discord.py,sha256=CD-SBECMdr3pnmqbPvBMyPU2cBroXUhyW6F7L3laP6A,5348
 unstructured_ingest/processes/connectors/github.py,sha256=smHCz6jOH1p_hW2S25bYunBBj_pYjz8HTw6wkzaJz_A,7765
 unstructured_ingest/processes/connectors/gitlab.py,sha256=Fdq6_lk-By1JDmLGVjoKJkaHESiKTZsbvoHhMsljlE0,10114
 unstructured_ingest/processes/connectors/google_drive.py,sha256=jQb4_rKL_tJg7s7m-H8nrvc0GKwxiubtg8KL3-ZIGPM,35304
-unstructured_ingest/processes/connectors/jira.py,sha256=BuZwExmdcI-R_MGPUwm8TnFh2jEjjwkyA1T51Bgqh-U,18558
+unstructured_ingest/processes/connectors/jira.py,sha256=DCP1pSagcjmQP0he1N726WNLsiHXlTUuhFwAtxt4zs8,20278
 unstructured_ingest/processes/connectors/kdbai.py,sha256=XhxYpKSAoFPBsDQWwNuLX03DCxOVr7yquj9VYM55Rtc,5174
 unstructured_ingest/processes/connectors/local.py,sha256=CesMduUiSPqdJpqIyW28icGvGAo4hfa-4fzbYajmMSo,7450
 unstructured_ingest/processes/connectors/milvus.py,sha256=L-PM5osheNyNsLGYZmiF3rRmeulp7Ejk92JCoaQ_F9Y,12075
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.46.dev0.dist-info/METADATA,sha256=nT9dK2ltS-aMKLa_nCewjwQSUYlaDVnvNUNoWDwjHPw,8847
-unstructured_ingest-1.0.46.dev0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.46.dev0.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.46.dev0.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.46.dev0.dist-info/RECORD,,
+unstructured_ingest-1.0.48.dist-info/METADATA,sha256=mFnSAhVaigxOwDyNbTK1XCVXRwY40tsATd3zYSEQE9k,8842
+unstructured_ingest-1.0.48.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.48.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.48.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.48.dist-info/RECORD,,