unstructured-ingest 1.0.40__tar.gz → 1.0.41__tar.gz

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.40
+Version: 1.0.41
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.0.41/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.41"  # pragma: no cover

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/airtable.py

@@ -184,6 +184,7 @@ class AirtableIndexer(Indexer):
                     filename=str(Path(fullpath).name),
                     fullpath=fullpath,
                 ),
+                display_name=fullpath,
             )
 
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/astradb.py

@@ -195,8 +195,10 @@ class AstraDBIndexer(Indexer):
         all_ids = self._get_doc_ids()
         ids = list(all_ids)
         id_batches = batch_generator(ids, self.index_config.batch_size)
-
         for batch in id_batches:
+            batch_items = [BatchItem(identifier=b) for b in batch]
+            display_name = (f"{self.index_config.collection_name}-{self.index_config.keyspace}"
+                            f"-[{batch_items[0].identifier}..{batch_items[-1].identifier}]")
             fd = AstraDBBatchFileData(
                 connector_type=CONNECTOR_TYPE,
                 metadata=FileDataSourceMetadata(
@@ -206,7 +208,8 @@ class AstraDBIndexer(Indexer):
                     collection_name=self.index_config.collection_name,
                     keyspace=self.index_config.keyspace,
                 ),
-                batch_items=[BatchItem(identifier=b) for b in batch],
+                batch_items=batch_items,
+                display_name=display_name,
             )
             yield fd
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/confluence.py

@@ -233,6 +233,7 @@ class ConfluenceIndexer(Indexer):
                     metadata=metadata,
                     additional_metadata=additional_metadata,
                     source_identifiers=source_identifiers,
+                    display_name=source_identifiers.fullpath,
                 )
                 yield file_data
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/databricks/volumes.py

@@ -133,14 +133,15 @@ class DatabricksVolumesIndexer(Indexer, ABC):
                 if rel_path.startswith("/"):
                     rel_path = rel_path[1:]
                 filename = Path(file_info.path).name
+                source_identifiers = SourceIdentifiers(
+                    filename=filename,
+                    rel_path=rel_path,
+                    fullpath=file_info.path,
+                )
                 yield FileData(
                     identifier=str(uuid5(NAMESPACE_DNS, file_info.path)),
                     connector_type=self.connector_type,
-                    source_identifiers=SourceIdentifiers(
-                        filename=filename,
-                        rel_path=rel_path,
-                        fullpath=file_info.path,
-                    ),
+                    source_identifiers=source_identifiers,
                     additional_metadata={
                         "catalog": self.index_config.catalog,
                         "path": file_info.path,
@@ -148,6 +149,7 @@ class DatabricksVolumesIndexer(Indexer, ABC):
                     metadata=FileDataSourceMetadata(
                         url=file_info.path, date_modified=str(file_info.modification_time)
                     ),
+                    display_name=source_identifiers.fullpath,
                 )
         except Exception as e:
             raise self.connection_config.wrap_error(e=e)

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/discord.py

@@ -91,6 +91,7 @@ class DiscordIndexer(Indexer):
             connector_type=CONNECTOR_TYPE,
             source_identifiers=source_identifiers,
             metadata=metadata,
+            display_name=source_identifiers.fullpath,
         )
 
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/elasticsearch/elasticsearch.py

@@ -199,17 +199,24 @@ class ElasticsearchIndexer(Indexer):
         all_ids = self._get_doc_ids()
         ids = list(all_ids)
         for batch in batch_generator(ids, self.index_config.batch_size):
+            batch_items = [BatchItem(identifier=b) for b in batch]
+            url = f"{self.connection_config.hosts[0]}/{self.index_config.index_name}"
+            display_name = (
+                f"url={url}, batch_size={len(batch_items)} "
+                f"ids={batch_items[0].identifier}..{batch_items[-1].identifier}"
+            )  # noqa: E501
             # Make sure the hash is always a positive number to create identified
             yield ElasticsearchBatchFileData(
                 connector_type=CONNECTOR_TYPE,
                 metadata=FileDataSourceMetadata(
-                    url=f"{self.connection_config.hosts[0]}/{self.index_config.index_name}",
+                    url=url,
                     date_processed=str(time()),
                 ),
                 additional_metadata=ElastisearchAdditionalMetadata(
                     index_name=self.index_config.index_name,
                 ),
-                batch_items=[BatchItem(identifier=b) for b in batch],
+                batch_items=batch_items,
+                display_name=display_name,
             )
 
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/gitlab.py

@@ -190,21 +190,22 @@ class GitLabIndexer(Indexer):
                     "file_path": file["path"],
                     "ref": ref,
                 }
-
+                source_identifiers = SourceIdentifiers(
+                    fullpath=file["path"],
+                    filename=Path(file["path"]).name,
+                    rel_path=relative_path,
+                )
                 yield FileData(
                     identifier=file["id"],
                     connector_type=CONNECTOR_TYPE,
-                    source_identifiers=SourceIdentifiers(
-                        fullpath=file["path"],
-                        filename=Path(file["path"]).name,
-                        rel_path=relative_path,
-                    ),
+                    source_identifiers=source_identifiers,
                     metadata=FileDataSourceMetadata(
                         url=file["id"],
                         record_locator=record_locator,
                         permissions_data=[{"mode": file["mode"]}],
                     ),
                     additional_metadata={},
+                    display_name=source_identifiers.fullpath,
                 )
 
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/jira.py

@@ -262,6 +262,7 @@ class JiraIndexer(Indexer):
             metadata=metadata,
             additional_metadata=issue.model_dump(),
             source_identifiers=source_identifiers,
+            display_name=source_identifiers.fullpath,
         )
         return file_data
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/local.py

@@ -119,21 +119,21 @@ class LocalIndexer(Indexer):
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         for file_path in self.list_files():
+            source_identifiers = SourceIdentifiers(
+                fullpath=str(file_path.resolve()),
+                filename=file_path.name,
+                rel_path=(
+                    str(file_path.resolve()).replace(str(self.index_config.path.resolve()), "")[1:]
+                    if not self.index_config.path.is_file()
+                    else self.index_config.path.name
+                ),
+            )
             file_data = FileData(
                 identifier=str(file_path.resolve()),
                 connector_type=CONNECTOR_TYPE,
-                source_identifiers=SourceIdentifiers(
-                    fullpath=str(file_path.resolve()),
-                    filename=file_path.name,
-                    rel_path=(
-                        str(file_path.resolve()).replace(str(self.index_config.path.resolve()), "")[
-                            1:
-                        ]
-                        if not self.index_config.path.is_file()
-                        else self.index_config.path.name
-                    ),
-                ),
+                source_identifiers=source_identifiers,
                 metadata=self.get_file_metadata(path=file_path),
+                display_name=source_identifiers.fullpath,
             )
             yield file_data
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/mongodb.py

@@ -149,6 +149,10 @@ class MongoDBIndexer(Indexer):
 
         for id_batch in batch_generator(ids, batch_size=batch_size):
             # Make sure the hash is always a positive number to create identifier
+            display_name = (
+                f"{self.index_config.database}.{self.index_config.collection}, "
+                f"batch {id_batch[0]}-{id_batch[-1]}"
+            )
             metadata = FileDataSourceMetadata(
                 date_processed=str(time()),
                 record_locator={
@@ -164,6 +168,7 @@ class MongoDBIndexer(Indexer):
                 additional_metadata=MongoDBAdditionalMetadata(
                     collection=self.index_config.collection, database=self.index_config.database
                 ),
+                display_name=display_name,
             )
             yield file_data
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/notion/connector.py

@@ -174,6 +174,7 @@ class NotionIndexer(Indexer):
                 source_identifiers=source_identifiers,
                 metadata=metadata,
                 additional_metadata=additional_metadata,
+                display_name=source_identifiers.fullpath,
             )
         except Exception as e:
             logger.error(f"Error retrieving page {page_id}: {e}")
@@ -210,6 +211,7 @@ class NotionIndexer(Indexer):
                 source_identifiers=source_identifiers,
                 metadata=metadata,
                 additional_metadata=additional_metadata,
+                display_name=source_identifiers.fullpath,
             )
         except Exception as e:
             logger.error(f"Error retrieving database {database_id}: {e}")

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/onedrive.py

@@ -223,6 +223,7 @@ class OnedriveIndexer(Indexer):
                 },
             ),
             additional_metadata=self.get_properties_sync(drive_item=drive_item),
+            display_name=server_path,
         )
 
     async def drive_item_to_file_data(self, drive_item: "DriveItem") -> FileData:

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/outlook.py

@@ -149,11 +149,11 @@ class OutlookIndexer(Indexer):
 
     def _message_to_file_data(self, message: "Message") -> FileData:
         fullpath = self._generate_fullpath(message)
-
+        source_identifiers = SourceIdentifiers(filename=fullpath.name, fullpath=str(fullpath))
         return FileData(
             identifier=message.id,
             connector_type=CONNECTOR_TYPE,
-            source_identifiers=SourceIdentifiers(filename=fullpath.name, fullpath=str(fullpath)),
+            source_identifiers=source_identifiers,
             metadata=FileDataSourceMetadata(
                 url=message.resource_url,
                 version=message.change_key,
@@ -178,6 +178,7 @@ class OutlookIndexer(Indexer):
                 "has_attachments": message.has_attachments,
                 "importance": message.importance,
             },
+            display_name=source_identifiers.fullpath,
         )
 
     def _generate_fullpath(self, message: "Message") -> Path:

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/salesforce.py

@@ -182,14 +182,15 @@ class SalesforceIndexer(Indexer):
                     record_with_extension = record["Id"] + self.get_file_extension(
                         record["attributes"]["type"]
                     )
+                    source_identifiers = SourceIdentifiers(
+                        filename=record_with_extension,
+                        fullpath=f"{record['attributes']['type']}/{record_with_extension}",
+                    )
                     files_list.append(
                         FileData(
                             connector_type=CONNECTOR_TYPE,
                             identifier=record["Id"],
-                            source_identifiers=SourceIdentifiers(
-                                filename=record_with_extension,
-                                fullpath=f"{record['attributes']['type']}/{record_with_extension}",
-                            ),
+                            source_identifiers=source_identifiers,
                             metadata=FileDataSourceMetadata(
                                 url=record["attributes"]["url"],
                                 version=str(parser.parse(record["SystemModstamp"]).timestamp()),
@@ -200,6 +201,7 @@ class SalesforceIndexer(Indexer):
                                 record_locator={"id": record["Id"]},
                             ),
                             additional_metadata={"record_type": record["attributes"]["type"]},
+                            display_name=source_identifiers.fullpath,
                         )
                     )
             except SalesforceMalformedRequest as e:

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/slack.py

@@ -122,12 +122,13 @@ class SlackIndexer(Indexer):
         identifier = hashlib.sha256(identifier_base.encode("utf-8")).hexdigest()
         filename = identifier[:16]
 
+        source_identifiers = SourceIdentifiers(
+            filename=f"{filename}.xml", fullpath=f"{filename}.xml"
+        )
         return FileData(
             identifier=identifier,
             connector_type=CONNECTOR_TYPE,
-            source_identifiers=SourceIdentifiers(
-                filename=f"{filename}.xml", fullpath=f"{filename}.xml"
-            ),
+            source_identifiers=source_identifiers,
             metadata=FileDataSourceMetadata(
                 date_created=ts_oldest,
                 date_modified=ts_newest,
@@ -138,6 +139,7 @@ class SlackIndexer(Indexer):
                     "latest": ts_newest,
                 },
             ),
+            display_name=source_identifiers.fullpath,
         )
 
     @SourceConnectionError.wrap

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/sql/sql.py

@@ -130,7 +130,11 @@ class SQLIndexer(Indexer, ABC):
                 (len(ids) + self.index_config.batch_size - 1) // self.index_config.batch_size
             )
         ]
+
         for batch in id_batches:
+            batch_items = [BatchItem(identifier=str(b)) for b in batch]
+            display_name = (f"{self.index_config.table_name}-{self.index_config.id_column}"
+                            f"-[{batch_items[0].identifier}..{batch_items[-1].identifier}]")
             # Make sure the hash is always a positive number to create identified
             yield SqlBatchFileData(
                 connector_type=self.connector_type,
@@ -140,7 +144,8 @@ class SQLIndexer(Indexer, ABC):
                 additional_metadata=SqlAdditionalMetadata(
                     table_name=self.index_config.table_name, id_column=self.index_config.id_column
                 ),
-                batch_items=[BatchItem(identifier=str(b)) for b in batch],
+                batch_items=batch_items,
+                display_name=display_name
             )
 
 

{unstructured_ingest-1.0.40 → unstructured_ingest-1.0.41}/unstructured_ingest/processes/connectors/zendesk/zendesk.py

@@ -86,12 +86,13 @@ class ZendeskIndexer(Indexer):
     async def get_tickets(self) -> AsyncGenerator[ZendeskFileData, None]:
         async with self.connection_config.get_client() as client:
             async for ticket in client.get_tickets():
+                source_identifiers = SourceIdentifiers(
+                    filename=f"{ticket.id}.txt", fullpath=f"tickets/{ticket.id}.txt"
+                )
                 yield ZendeskFileData(
                     identifier=str(ticket.id),
                     connector_type=self.connector_type,
-                    source_identifiers=SourceIdentifiers(
-                        filename=f"{ticket.id}.txt", fullpath=f"tickets/{ticket.id}.txt"
-                    ),
+                    source_identifiers=source_identifiers,
                     additional_metadata=ZendeskAdditionalMetadata(
                         item_type="ticket", content=ticket
                     ),
@@ -101,17 +102,19 @@ class ZendeskIndexer(Indexer):
                         date_modified=ticket.updated_at.isoformat() if ticket.updated_at else None,
                         date_processed=str(time()),
                     ),
+                    display_name=source_identifiers.fullpath,
                 )
 
     async def get_articles(self) -> AsyncGenerator[ZendeskFileData, None]:
         async with self.connection_config.get_client() as client:
             async for article in client.get_articles():
+                source_identifiers = SourceIdentifiers(
+                    filename=f"{article.id}.html", fullpath=f"articles/{article.id}.html"
+                )
                 yield ZendeskFileData(
                     identifier=str(article.id),
                     connector_type=self.connector_type,
-                    source_identifiers=SourceIdentifiers(
-                        filename=f"{article.id}.html", fullpath=f"articles/{article.id}.html"
-                    ),
+                    source_identifiers=source_identifiers,
                     additional_metadata=ZendeskAdditionalMetadata(
                         item_type="article", content=article
                     ),
@@ -123,6 +126,7 @@ class ZendeskIndexer(Indexer):
                         ),
                         date_processed=str(time()),
                     ),
+                    display_name=source_identifiers.fullpath,
                 )
 
     async def run_async(self, **kwargs: Any) -> AsyncGenerator[ZendeskFileData, None]:

unstructured_ingest-1.0.40/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.40"  # pragma: no cover