glean-indexing-sdk 0.0.3__tar.gz → 0.2.0__tar.gz

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/.cz.toml

@@ -1,5 +1,5 @@
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.0.3"
+version = "0.2.0"
 tag_format = "v$version"
 version_files = ["pyproject.toml:version", "src/glean/indexing/__init__.py:__version__"] 

glean_indexing_sdk-0.2.0/CHANGELOG.md

@@ -0,0 +1,10 @@
+## v0.1.0 (2025-07-23)
+
+### Feat
+
+- Adds property definition builder
+
+### Fix
+
+- Fixing format of tags for release
+- Adds addition model for re-export

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: glean-indexing-sdk
-Version: 0.0.3
+Version: 0.2.0
 Summary: SDK for building custom Glean indexing integrations
 Project-URL: Source Code, https://github.com/glean-io/glean-indexing-sdk
 Author-email: Steve Calvert <steve.calvert@glean.com>
@@ -232,6 +232,18 @@ connector.configure_datasource()
 connector.index_data(mode=IndexingMode.FULL)
 ```
 
+**When to use forced restarts:**
+- When you need to abort and restart a failed or interrupted upload
+- When you want to ensure a clean upload state by discarding partial uploads  
+- When recovering from upload errors or inconsistent states
+
+**How it works:**
+- Generates a new `upload_id` to ensure clean separation from previous uploads
+- Sets `forceRestartUpload=True` on the **first batch only**
+- Continues with normal batch processing for subsequent batches
+
+This feature is available on all connector types: `BaseDatasourceConnector`, `BaseStreamingDatasourceConnector`, and `BasePeopleConnector`.
+
 ### Complete Example
 
 ```python snippet=non_streaming/complete.py

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/README.md

@@ -202,6 +202,18 @@ connector.configure_datasource()
 connector.index_data(mode=IndexingMode.FULL)
 ```
 
+**When to use forced restarts:**
+- When you need to abort and restart a failed or interrupted upload
+- When you want to ensure a clean upload state by discarding partial uploads  
+- When recovering from upload errors or inconsistent states
+
+**How it works:**
+- Generates a new `upload_id` to ensure clean separation from previous uploads
+- Sets `forceRestartUpload=True` on the **first batch only**
+- Continues with normal batch processing for subsequent batches
+
+This feature is available on all connector types: `BaseDatasourceConnector`, `BaseStreamingDatasourceConnector`, and `BasePeopleConnector`.
+
 ### Complete Example
 
 ```python snippet=non_streaming/complete.py

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "glean-indexing-sdk"
-version = "0.0.3"
+version = "0.2.0"
 description = "SDK for building custom Glean indexing integrations"
 authors = [{ name = "Steve Calvert", email = "steve.calvert@glean.com" }]
 readme = "README.md"

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/src/glean/indexing/__init__.py

@@ -53,4 +53,4 @@ __all__ = [
 try:
     __version__ = version("glean-indexing-sdk")
 except PackageNotFoundError:
-    __version__ = "0.0.3"
+    __version__ = "0.2.0"

glean_indexing_sdk-0.2.0/src/glean/indexing/common/property_definition_builder.py

@@ -0,0 +1,115 @@
+from typing import List, Optional
+
+from glean.api_client.models.propertydefinition import PropertyDefinition, PropertyType, UIOptions
+
+
+class PropertyDefinitionBuilder:
+    """
+    Builder class for creating PropertyDefinition objects with a fluent interface.
+
+    This class provides a convenient way to build multiple PropertyDefinition objects
+    with proper validation and type safety.
+
+    Example:
+        builder = PropertyDefinitionBuilder()
+        properties = (builder
+            .add_property("title", "Title", property_type=PropertyType.TEXT)
+            .add_property("author", "Author", display_label_plural="Authors")
+            .build())
+    """
+
+    def __init__(self) -> None:
+        self.properties: List[PropertyDefinition] = []
+
+    def add_property(
+        self,
+        name: str,
+        display_label: str,
+        display_label_plural: Optional[str] = None,
+        property_type: PropertyType = PropertyType.TEXT,
+        ui_options: UIOptions = UIOptions.SEARCH_RESULT,
+        hide_ui_facet: bool = False,
+        ui_facet_order: Optional[int] = None,
+        group: Optional[str] = None,
+    ) -> "PropertyDefinitionBuilder":
+        """
+        Add a property definition to the builder.
+
+        Args:
+            name: The property name (must not be empty)
+            display_label: The display label for the property
+            display_label_plural: Optional plural form of the display label
+            property_type: The type of property (defaults to TEXT)
+            ui_options: UI options for the property (defaults to SEARCH_RESULT)
+            hide_ui_facet: Whether to hide the UI facet
+            ui_facet_order: Optional order for UI facet display
+            group: Optional group name for the property
+
+        Returns:
+            Self for method chaining
+
+        Raises:
+            ValueError: If name or display_label is empty
+        """
+        if not name or not name.strip():
+            raise ValueError("Property name cannot be empty")
+        if not display_label or not display_label.strip():
+            raise ValueError("Display label cannot be empty")
+
+        base_params = {
+            "name": name.strip(),
+            "display_label": display_label.strip(),
+            "property_type": property_type.value,
+            "ui_options": ui_options.value,
+            "hide_ui_facet": hide_ui_facet,
+        }
+
+        optional_params = {
+            k: v
+            for k, v in {
+                "display_label_plural": display_label_plural.strip()
+                if display_label_plural
+                else None,
+                "ui_facet_order": ui_facet_order,
+                "group": group.strip() if group else None,
+            }.items()
+            if v is not None
+        }
+
+        params = {**base_params, **optional_params}
+
+        try:
+            prop = PropertyDefinition(**params)
+            self.properties.append(prop)
+        except Exception as e:
+            raise ValueError(f"Failed to create PropertyDefinition: {e}") from e
+
+        return self
+
+    def clear(self) -> "PropertyDefinitionBuilder":
+        """
+        Clear all properties from the builder.
+
+        Returns:
+            Self for method chaining
+        """
+        self.properties.clear()
+        return self
+
+    def count(self) -> int:
+        """
+        Get the number of properties currently in the builder.
+
+        Returns:
+            Number of properties
+        """
+        return len(self.properties)
+
+    def build(self) -> List[PropertyDefinition]:
+        """
+        Build and return the list of PropertyDefinition objects.
+
+        Returns:
+            List of PropertyDefinition objects
+        """
+        return self.properties.copy()

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/src/glean/indexing/connectors/base_connector.py

@@ -55,6 +55,13 @@ class BaseConnector(ABC, Generic[TSourceData, TIndexableEntityDefinition]):
         pass
 
     @abstractmethod
-    def index_data(self, mode: IndexingMode = IndexingMode.FULL) -> None:
-        """Index data from the connector to Glean."""
+    def index_data(
+        self, mode: IndexingMode = IndexingMode.FULL, force_restart: bool = False
+    ) -> None:
+        """Index data from the connector to Glean.
+
+        Args:
+            mode: The indexing mode to use (FULL or INCREMENTAL).
+            force_restart: If True, forces a restart of the upload, discarding any previous upload progress.
+        """
         pass

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/src/glean/indexing/connectors/base_datasource_connector.py

@@ -114,12 +114,16 @@ class BaseDatasourceConnector(BaseConnector[TSourceData, DocumentDefinition], AB
             client.indexing.datasources.add(**config.dict(exclude_unset=True))
             logger.info(f"Successfully configured datasource: {config.name}")
 
-    def index_data(self, mode: IndexingMode = IndexingMode.FULL) -> None:
+    def index_data(
+        self, mode: IndexingMode = IndexingMode.FULL, force_restart: bool = False
+    ) -> None:
         """
         Index data from the datasource to Glean with identity crawl followed by content crawl.
 
         Args:
             mode: The indexing mode to use (FULL or INCREMENTAL).
+            force_restart: If True, forces a restart of the upload, discarding any previous upload progress.
+                          This sets forceRestartUpload=True on the first batch and generates a new upload ID.
         """
         self._observability.start_execution()
 
@@ -169,7 +173,7 @@ class BaseDatasourceConnector(BaseConnector[TSourceData, DocumentDefinition], AB
             self._observability.start_timer("data_upload")
             if documents:
                 logger.info(f"Indexing {len(documents)} documents")
-                self._batch_index_documents(documents)
+                self._batch_index_documents(documents, force_restart=force_restart)
             self._observability.end_timer("data_upload")
 
             logger.info(f"Successfully indexed {len(documents)} documents to Glean")
@@ -272,8 +276,15 @@ class BaseDatasourceConnector(BaseConnector[TSourceData, DocumentDefinition], AB
                 self._observability.increment_counter("batch_upload_errors")
                 raise
 
-    def _batch_index_documents(self, documents: Sequence[DocumentDefinition]) -> None:
-        """Index documents in batches with proper page signaling."""
+    def _batch_index_documents(
+        self, documents: Sequence[DocumentDefinition], force_restart: bool = False
+    ) -> None:
+        """Index documents in batches with proper page signaling.
+
+        Args:
+            documents: The documents to index
+            force_restart: If True, forces a restart by generating a new upload ID and setting forceRestartUpload=True on the first batch
+        """
         if not documents:
             return
 
@@ -285,14 +296,21 @@ class BaseDatasourceConnector(BaseConnector[TSourceData, DocumentDefinition], AB
         upload_id = str(uuid.uuid4())
         for i, batch in enumerate(batches):
             try:
+                is_first_page = i == 0
+                bulk_index_kwargs = {
+                    "datasource": self.name,
+                    "documents": list(batch),
+                    "upload_id": upload_id,
+                    "is_first_page": is_first_page,
+                    "is_last_page": (i == total_batches - 1),
+                }
+
+                if force_restart and is_first_page:
+                    bulk_index_kwargs["forceRestartUpload"] = True
+                    logger.info("Force restarting upload - discarding any previous upload progress")
+
                 with api_client() as client:
-                    client.indexing.documents.bulk_index(
-                        datasource=self.name,
-                        documents=list(batch),
-                        upload_id=upload_id,
-                        is_first_page=(i == 0),
-                        is_last_page=(i == total_batches - 1),
-                    )
+                    client.indexing.documents.bulk_index(**bulk_index_kwargs)
 
                 logger.info(f"Document batch {i + 1}/{total_batches} uploaded successfully")
                 self._observability.increment_counter("batches_uploaded")

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/src/glean/indexing/connectors/base_people_connector.py

@@ -58,11 +58,15 @@ class BasePeopleConnector(BaseConnector[TSourceData, EmployeeInfoDefinition], AB
         """The observability instance for this connector."""
         return self._observability
 
-    def index_data(self, mode: IndexingMode = IndexingMode.FULL) -> None:
+    def index_data(
+        self, mode: IndexingMode = IndexingMode.FULL, force_restart: bool = False
+    ) -> None:
         """Index people data to Glean.
 
         Args:
             mode: The indexing mode to use (FULL or INCREMENTAL).
+            force_restart: If True, forces a restart of the upload, discarding any previous upload progress.
+                          This sets forceRestartUpload=True on the first batch and generates a new upload ID.
         """
         self._observability.start_execution()
 
@@ -89,7 +93,7 @@ class BasePeopleConnector(BaseConnector[TSourceData, EmployeeInfoDefinition], AB
             self._observability.record_metric("employees_transformed", len(employees))
 
             self._observability.start_timer("data_upload")
-            self._batch_index_employees(employees)
+            self._batch_index_employees(employees, force_restart=force_restart)
             self._observability.end_timer("data_upload")
 
             logger.info(f"Successfully indexed {len(employees)} employees to Glean")
@@ -113,8 +117,15 @@ class BasePeopleConnector(BaseConnector[TSourceData, EmployeeInfoDefinition], AB
         """
         return self.data_client.get_source_data(since=since)
 
-    def _batch_index_employees(self, employees: Sequence[EmployeeInfoDefinition]) -> None:
-        """Index employees to Glean in batches."""
+    def _batch_index_employees(
+        self, employees: Sequence[EmployeeInfoDefinition], force_restart: bool = False
+    ) -> None:
+        """Index employees to Glean in batches.
+
+        Args:
+            employees: The employees to index
+            force_restart: If True, forces a restart by generating a new upload ID and setting forceRestartUpload=True on the first batch
+        """
         if not employees:
             return
 
@@ -126,13 +137,20 @@ class BasePeopleConnector(BaseConnector[TSourceData, EmployeeInfoDefinition], AB
         upload_id = str(uuid.uuid4())
         for i, batch in enumerate(batches):
             try:
+                is_first_page = i == 0
+                bulk_index_kwargs = {
+                    "employees": list(batch),
+                    "upload_id": upload_id,
+                    "is_first_page": is_first_page,
+                    "is_last_page": (i == total_batches - 1),
+                }
+
+                if force_restart and is_first_page:
+                    bulk_index_kwargs["forceRestartUpload"] = True
+                    logger.info("Force restarting upload - discarding any previous upload progress")
+
                 with api_client() as client:
-                    client.indexing.people.bulk_index(
-                        employees=list(batch),
-                        upload_id=upload_id,
-                        is_first_page=(i == 0),
-                        is_last_page=(i == total_batches - 1),
-                    )
+                    client.indexing.people.bulk_index(**bulk_index_kwargs)
 
                 logger.info(f"Employee batch {i + 1}/{total_batches} uploaded successfully")
                 self._observability.increment_counter("batches_uploaded")

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/src/glean/indexing/connectors/base_streaming_datasource_connector.py

@@ -47,6 +47,7 @@ class BaseStreamingDatasourceConnector(BaseDatasourceConnector[TSourceData], ABC
         super().__init__(name, data_client)  # type: ignore[arg-type]
         self.batch_size = 1000
         self._upload_id: Optional[str] = None
+        self._force_restart: bool = False
 
     def generate_upload_id(self) -> str:
         """Generate a unique upload ID for batch tracking."""
@@ -67,12 +68,16 @@ class BaseStreamingDatasourceConnector(BaseDatasourceConnector[TSourceData], ABC
         logger.info(f"Fetching streaming data from source{' since ' + since if since else ''}")
         yield from self.data_client.get_source_data(since=since)
 
-    def index_data(self, mode: IndexingMode = IndexingMode.FULL) -> None:
+    def index_data(
+        self, mode: IndexingMode = IndexingMode.FULL, force_restart: bool = False
+    ) -> None:
         """
         Index data from the datasource to Glean using streaming.
 
         Args:
             mode: The indexing mode to use (FULL or INCREMENTAL).
+            force_restart: If True, forces a restart of the upload, discarding any previous upload progress.
+                          This sets forceRestartUpload=True on the first batch and generates a new upload ID.
         """
         logger.info(f"Starting {mode.name.lower()} streaming indexing for datasource '{self.name}'")
 
@@ -81,6 +86,7 @@ class BaseStreamingDatasourceConnector(BaseDatasourceConnector[TSourceData], ABC
             since = "2023-01-01T00:00:00Z"
 
         upload_id = self.generate_upload_id()
+        self._force_restart = force_restart
         data_iterator = self.get_data(since=since)
         is_first_batch = True
         batch: List[TSourceData] = []
@@ -150,14 +156,20 @@ class BaseStreamingDatasourceConnector(BaseDatasourceConnector[TSourceData], ABC
             transformed_batch = self.transform(batch)
             logger.info(f"Transformed batch {batch_number}: {len(transformed_batch)} documents")
 
+            bulk_index_kwargs = {
+                "datasource": self.name,
+                "documents": list(transformed_batch),
+                "upload_id": upload_id,
+                "is_first_page": is_first_batch,
+                "is_last_page": is_last_batch,
+            }
+
+            if self._force_restart and is_first_batch:
+                bulk_index_kwargs["forceRestartUpload"] = True
+                logger.info("Force restarting upload - discarding any previous upload progress")
+
             with api_client() as client:
-                client.indexing.documents.bulk_index(
-                    datasource=self.name,
-                    documents=list(transformed_batch),
-                    upload_id=upload_id,
-                    is_first_page=is_first_batch,
-                    is_last_page=is_last_batch,
-                )
+                client.indexing.documents.bulk_index(**bulk_index_kwargs)
 
             logger.info(f"Batch {batch_number} indexed successfully")
 

{glean_indexing_sdk-0.0.3 → glean_indexing_sdk-0.2.0}/taskfile.yml

@@ -178,11 +178,13 @@ tasks:
           {{.PYRIGHT}} {{.PYTHON_FILES}}
         fi
 
+  # Lint Readme task: Lint the README.md file
   lint:readme:
     desc: Lint the README.md file
     cmds:
       - npx -y markdown-code check
 
+  # Lint Readme fix task: Fix the README.md file
   lint:readme:fix:
     desc: Fix the README.md file
     cmds: