unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/{v2/pipeline → pipeline}/steps/upload.py

@@ -3,11 +3,11 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
-from unstructured_ingest.v2.interfaces import UploadContent
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.pipeline.interfaces import BatchPipelineStep
-from unstructured_ingest.v2.pipeline.otel import instrument
-from unstructured_ingest.v2.types.file_data import file_data_from_file
+from unstructured_ingest.data_types.file_data import file_data_from_file
+from unstructured_ingest.interfaces import UploadContent
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.interfaces import BatchPipelineStep
+from unstructured_ingest.pipeline.otel import instrument
 
 STEP_ID = "upload"
 

unstructured_ingest/{v2/processes → processes}/chunker.py

@@ -5,12 +5,12 @@ from typing import Any, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.logger import logger
+from unstructured_ingest.unstructured_api import call_api_async
 from unstructured_ingest.utils.chunking import assign_and_map_hash_ids
 from unstructured_ingest.utils.data_prep import get_json_data
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.unstructured_api import call_api_async
 
 CHUNK_MAX_CHARS_DEFAULT: int = 500
 CHUNK_MULTI_PAGE_DEFAULT: bool = True

unstructured_ingest/{v2/processes → processes}/connector_registry.py

@@ -2,7 +2,7 @@ from abc import ABC
 from dataclasses import dataclass
 from typing import Optional, Type, TypeVar
 
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     ConnectionConfig,
     Downloader,
     DownloaderConfig,

unstructured_ingest/{v2/processes → processes}/connectors/__init__.py

@@ -1,16 +1,16 @@
 from __future__ import annotations
 
-import unstructured_ingest.v2.processes.connectors.databricks  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.duckdb  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.elasticsearch  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.fsspec  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.ibm_watsonx  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.kafka  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.lancedb  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.qdrant  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.sql  # noqa: F401
-import unstructured_ingest.v2.processes.connectors.weaviate  # noqa: F401
-from unstructured_ingest.v2.processes.connector_registry import (
+import unstructured_ingest.processes.connectors.databricks  # noqa: F401
+import unstructured_ingest.processes.connectors.duckdb  # noqa: F401
+import unstructured_ingest.processes.connectors.elasticsearch  # noqa: F401
+import unstructured_ingest.processes.connectors.fsspec  # noqa: F401
+import unstructured_ingest.processes.connectors.ibm_watsonx  # noqa: F401
+import unstructured_ingest.processes.connectors.kafka  # noqa: F401
+import unstructured_ingest.processes.connectors.lancedb  # noqa: F401
+import unstructured_ingest.processes.connectors.qdrant  # noqa: F401
+import unstructured_ingest.processes.connectors.sql  # noqa: F401
+import unstructured_ingest.processes.connectors.weaviate  # noqa: F401
+from unstructured_ingest.processes.connector_registry import (
     add_destination_entry,
     add_source_entry,
 )
@@ -31,6 +31,8 @@ from .delta_table import CONNECTOR_TYPE as DELTA_TABLE_CONNECTOR_TYPE
 from .delta_table import delta_table_destination_entry
 from .discord import CONNECTOR_TYPE as DISCORD_CONNECTOR_TYPE
 from .discord import discord_source_entry
+from .github import CONNECTOR_TYPE as GITHUB_CONNECTOR_TYPE
+from .github import github_source_entry
 from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
 from .gitlab import gitlab_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
@@ -124,3 +126,4 @@ add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destina
 add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
 
 add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)
+add_source_entry(source_type=GITHUB_CONNECTOR_TYPE, entry=github_source_entry)

unstructured_ingest/{v2/processes → processes}/connectors/airtable.py

@@ -6,8 +6,8 @@ from uuid import NAMESPACE_DNS, uuid5
 import pandas
 from pydantic import BaseModel, Field, Secret, field_validator
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -16,10 +16,10 @@ from unstructured_ingest.v2.interfaces import (
     Indexer,
     IndexerConfig,
 )
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from pyairtable import Api

unstructured_ingest/{v2/processes → processes}/connectors/astradb.py

@@ -10,16 +10,19 @@ from pydantic import BaseModel, Field, Secret
 
 from unstructured_ingest import __name__ as integration_name
 from unstructured_ingest.__version__ import __version__ as integration_version
+from unstructured_ingest.data_types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
     SourceConnectionNetworkError,
 )
-from unstructured_ingest.utils.data_prep import batch_generator, get_data
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.string_and_date_utils import truncate_string_bytes
-from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -33,19 +36,16 @@ from unstructured_ingest.v2.interfaces import (
     UploadStagerConfig,
     download_responses,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.utils import format_and_truncate_orig_elements
-from unstructured_ingest.v2.types.file_data import (
-    BatchFileData,
-    BatchItem,
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
+from unstructured_ingest.processes.connectors.utils import format_and_truncate_orig_elements
+from unstructured_ingest.utils.constants import RECORD_ID_LABEL
+from unstructured_ingest.utils.data_prep import batch_generator, get_data
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.string_and_date_utils import truncate_string_bytes
 
 if TYPE_CHECKING:
     from astrapy import AsyncCollection as AstraDBAsyncCollection

unstructured_ingest/{v2/processes → processes}/connectors/azure_ai_search.py

@@ -5,11 +5,9 @@ from typing import TYPE_CHECKING, Any, Generator
 
 from pydantic import Field, Secret
 
+from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.error import DestinationConnectionError, WriteError
-from unstructured_ingest.utils.data_prep import batch_generator
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Uploader,
@@ -17,13 +15,14 @@ from unstructured_ingest.v2.interfaces import (
     UploadStager,
     UploadStagerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.utils import parse_datetime
-from unstructured_ingest.v2.types.file_data import FileData
-from unstructured_ingest.v2.utils import get_enhanced_element_id
+from unstructured_ingest.processes.connectors.utils import parse_datetime
+from unstructured_ingest.utils.constants import RECORD_ID_LABEL
+from unstructured_ingest.utils.data_prep import batch_generator, get_enhanced_element_id
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from azure.search.documents import SearchClient

unstructured_ingest/{v2/processes → processes}/connectors/chroma.py

@@ -6,10 +6,9 @@ from dateutil import parser
 from pydantic import Field, Secret
 from pydantic.functional_validators import BeforeValidator
 
+from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Uploader,
@@ -17,10 +16,14 @@ from unstructured_ingest.v2.interfaces import (
     UploadStager,
     UploadStagerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
-from unstructured_ingest.v2.types.file_data import FileData
-from unstructured_ingest.v2.utils import get_enhanced_element_id
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.utils.data_prep import (
+    batch_generator,
+    flatten_dict,
+    get_enhanced_element_id,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 from .utils import conform_string_to_dict
 

unstructured_ingest/{v2/processes → processes}/connectors/confluence.py

@@ -5,11 +5,13 @@ from typing import TYPE_CHECKING, Generator, List, Optional
 
 from pydantic import Field, Secret
 
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.html import HtmlMixin
-from unstructured_ingest.utils.string_and_date_utils import fix_unescaped_unicode
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -19,15 +21,13 @@ from unstructured_ingest.v2.interfaces import (
     IndexerConfig,
     download_responses,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.types.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.html import HtmlMixin
+from unstructured_ingest.utils.string_and_date_utils import fix_unescaped_unicode
 
 if TYPE_CHECKING:
     from atlassian import Confluence

unstructured_ingest/{v2/processes → processes}/connectors/couchbase.py

@@ -8,14 +8,19 @@ from typing import TYPE_CHECKING, Any, Generator, List
 
 from pydantic import BaseModel, Field, Secret
 
+from unstructured_ingest.data_types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
     SourceConnectionNetworkError,
 )
-from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -29,18 +34,13 @@ from unstructured_ingest.v2.interfaces import (
     UploadStagerConfig,
     download_responses,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.types.file_data import (
-    BatchFileData,
-    BatchItem,
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
+from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from couchbase.cluster import Cluster

unstructured_ingest/{v2/processes → processes}/connectors/databricks/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     add_destination_entry,
     add_source_entry,
 )

unstructured_ingest/{v2/processes → processes}/connectors/databricks/volumes.py

@@ -7,14 +7,18 @@ from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel, Field, Secret
 
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import (
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.errors_v2 import (
     ProviderError,
     RateLimitError,
     UserAuthError,
     UserError,
 )
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -25,12 +29,8 @@ from unstructured_ingest.v2.interfaces import (
     Uploader,
     UploaderConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.types.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from databricks.sdk import WorkspaceClient
@@ -68,7 +68,6 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
         "Databricks workspace endpoint or the "
         "Databricks accounts endpoint.",
     )
-    user_agent: str = "unstructuredio_oss"
 
     def wrap_error(self, e: Exception) -> Exception:
         from databricks.sdk.errors.base import DatabricksError
@@ -101,7 +100,9 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
         config = Config(
             host=self.host,
             **self.access_config.get_secret_value().model_dump(),
-        ).with_user_agent_extra("PyDatabricksSdk", self.user_agent)
+        ).with_user_agent_extra(
+            "PyDatabricksSdk", os.getenv("UNSTRUCTURED_USER_AGENT", "unstructuredio_oss")
+        )
 
         return WorkspaceClient(config=config)
 

unstructured_ingest/{v2/processes → processes}/connectors/databricks/volumes_aws.py

@@ -3,11 +3,11 @@ from typing import Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
+from unstructured_ingest.processes.connectors.databricks.volumes import (
     DatabricksVolumesAccessConfig,
     DatabricksVolumesConnectionConfig,
     DatabricksVolumesDownloader,
@@ -17,7 +17,7 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )

unstructured_ingest/{v2/processes → processes}/connectors/databricks/volumes_azure.py

@@ -3,11 +3,11 @@ from typing import Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
+from unstructured_ingest.processes.connectors.databricks.volumes import (
     DatabricksVolumesAccessConfig,
     DatabricksVolumesConnectionConfig,
     DatabricksVolumesDownloader,
@@ -17,7 +17,7 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )

unstructured_ingest/{v2/processes → processes}/connectors/databricks/volumes_gcp.py

@@ -3,11 +3,11 @@ from typing import Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
+from unstructured_ingest.processes.connectors.databricks.volumes import (
     DatabricksVolumesAccessConfig,
     DatabricksVolumesConnectionConfig,
     DatabricksVolumesDownloader,
@@ -17,7 +17,7 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )

unstructured_ingest/{v2/processes → processes}/connectors/databricks/volumes_native.py

@@ -3,11 +3,11 @@ from typing import Optional
 
 from pydantic import Field, Secret
 
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
+from unstructured_ingest.processes.connectors.databricks.volumes import (
     DatabricksVolumesAccessConfig,
     DatabricksVolumesConnectionConfig,
     DatabricksVolumesDownloader,
@@ -17,7 +17,7 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes import (
     DatabricksVolumesUploader,
     DatabricksVolumesUploaderConfig,
 )
-from unstructured_ingest.v2.processes.utils.blob_storage import (
+from unstructured_ingest.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )

unstructured_ingest/{v2/processes → processes}/connectors/databricks/volumes_table.py

@@ -7,25 +7,24 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field
 
-from unstructured_ingest.utils.data_prep import get_json_data, write_data
-from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces import (
     Uploader,
     UploaderConfig,
     UploadStager,
     UploadStagerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
 )
-from unstructured_ingest.v2.processes.connectors.databricks.volumes import DatabricksPathMixin
-from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
+from unstructured_ingest.processes.connectors.databricks.volumes import DatabricksPathMixin
+from unstructured_ingest.processes.connectors.sql.databricks_delta_tables import (
     DatabricksDeltaTablesConnectionConfig,
     DatabricksDeltaTablesUploadStagerConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileData
-from unstructured_ingest.v2.utils import get_enhanced_element_id
+from unstructured_ingest.utils.constants import RECORD_ID_LABEL
+from unstructured_ingest.utils.data_prep import get_enhanced_element_id, get_json_data, write_data
 
 CONNECTOR_TYPE = "databricks_volume_delta_tables"
 

unstructured_ingest/{v2/processes → processes}/connectors/delta_table.py

@@ -8,11 +8,9 @@ from urllib.parse import urlparse
 
 from pydantic import Field, Secret
 
+from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.utils.data_prep import get_data_df, get_json_data
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.table import convert_to_pandas_dataframe
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Uploader,
@@ -20,9 +18,11 @@ from unstructured_ingest.v2.interfaces import (
     UploadStager,
     UploadStagerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
-from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.utils.data_prep import get_data_df, get_json_data
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.table import convert_to_pandas_dataframe
 
 CONNECTOR_TYPE = "delta_table"
 

unstructured_ingest/{v2/processes → processes}/connectors/discord.py

@@ -4,9 +4,13 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field, Secret
 
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Downloader,
@@ -15,13 +19,9 @@ from unstructured_ingest.v2.interfaces import (
     Indexer,
     IndexerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
-from unstructured_ingest.v2.types.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from discord import Client as DiscordClient

unstructured_ingest/{v2/processes → processes}/connectors/duckdb/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from unstructured_ingest.v2.processes.connector_registry import (
+from unstructured_ingest.processes.connector_registry import (
     add_destination_entry,
 )
 

unstructured_ingest/{v2/processes → processes}/connectors/duckdb/base.py

@@ -2,11 +2,10 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
-from unstructured_ingest.utils.data_prep import get_data, write_data
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces import UploadStager
+from unstructured_ingest.utils.data_prep import get_data, get_enhanced_element_id, write_data
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import UploadStager
-from unstructured_ingest.v2.types.file_data import FileData
-from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 _COLUMNS = (
     "id",

unstructured_ingest/{v2/processes → processes}/connectors/duckdb/duckdb.py

@@ -5,20 +5,20 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field, Secret
 
+from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.utils.data_prep import get_data_df
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Uploader,
     UploaderConfig,
     UploadStagerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
-from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
-from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+from unstructured_ingest.utils.data_prep import get_data_df
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as DuckDBConnection

unstructured_ingest/{v2/processes → processes}/connectors/duckdb/motherduck.py

@@ -6,20 +6,20 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 from pydantic import Field, Secret
 
 from unstructured_ingest.__version__ import __version__ as unstructured_io_ingest_version
+from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.utils.data_prep import get_data_df
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
+from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
     Uploader,
     UploaderConfig,
     UploadStagerConfig,
 )
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
-from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
-from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+from unstructured_ingest.utils.data_prep import get_data_df
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as MotherDuckConnection