airbyte-cdk 6.7.0rc1__py3-none-any.whl → 6.7.0rc2__py3-none-any.whl

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -5,10 +5,10 @@
 import json
 import logging
 import pkgutil
-import re
 from copy import deepcopy
 from importlib import metadata
-from typing import Any, Dict, Iterator, List, Mapping, Optional, Tuple
+from typing import Any, Dict, Iterator, List, Mapping, Optional
+from packaging.version import Version, InvalidVersion
 
 import yaml
 from airbyte_cdk.models import (
@@ -245,45 +245,54 @@ class ManifestDeclarativeSource(DeclarativeSource):
                 "Validation against json schema defined in declarative_component_schema.yaml schema failed"
             ) from e
 
-        cdk_version = metadata.version("airbyte_cdk")
-        cdk_major, cdk_minor, cdk_patch = self._get_version_parts(cdk_version, "airbyte-cdk")
-        manifest_version = self._source_config.get("version")
-        if manifest_version is None:
+        cdk_version_str = metadata.version("airbyte_cdk")
+        cdk_version = self._parse_version(cdk_version_str, "airbyte-cdk")
+        manifest_version_str = self._source_config.get("version")
+        if manifest_version_str is None:
             raise RuntimeError(
                 "Manifest version is not defined in the manifest. This is unexpected since it should be a required field. Please contact support."
             )
-        manifest_major, manifest_minor, manifest_patch = self._get_version_parts(
-            manifest_version, "manifest"
-        )
+        manifest_version = self._parse_version(manifest_version_str, "manifest")
 
-        if cdk_version.startswith("0.0.0"):
+        if (cdk_version.major, cdk_version.minor, cdk_version.micro) == (0, 0, 0):
             # Skipping version compatibility check on unreleased dev branch
             pass
-        elif cdk_major < manifest_major or (
-            cdk_major == manifest_major and cdk_minor < manifest_minor
+        elif (cdk_version.major, cdk_version.minor) < (
+            manifest_version.major,
+            manifest_version.minor,
         ):
             raise ValidationError(
-                f"The manifest version {manifest_version} is greater than the airbyte-cdk package version ({cdk_version}). Your "
+                f"The manifest version {manifest_version!s} is greater than the airbyte-cdk package version ({cdk_version!s}). Your "
                 f"manifest may contain features that are not in the current CDK version."
             )
-        elif manifest_major == 0 and manifest_minor < 29:
+        elif (manifest_version.major, manifest_version.minor) < (0, 29):
             raise ValidationError(
                 f"The low-code framework was promoted to Beta in airbyte-cdk version 0.29.0 and contains many breaking changes to the "
-                f"language. The manifest version {manifest_version} is incompatible with the airbyte-cdk package version "
-                f"{cdk_version} which contains these breaking changes."
+                f"language. The manifest version {manifest_version!s} is incompatible with the airbyte-cdk package version "
+                f"{cdk_version!s} which contains these breaking changes."
             )
 
     @staticmethod
-    def _get_version_parts(version: str, version_type: str) -> Tuple[int, int, int]:
-        """
-        Takes a semantic version represented as a string and splits it into a tuple of its major, minor, and patch versions.
+    def _parse_version(
+        version: str,
+        version_type: str,
+    ) -> Version:
+        """Takes a semantic version represented as a string and splits it into a tuple.
+
+        The fourth part (prerelease) is not returned in the tuple.
+
+        Returns:
+            Version: the parsed version object
         """
-        version_parts = re.split(r"\.", version)
-        if len(version_parts) != 3 or not all([part.isdigit() for part in version_parts]):
+        try:
+            parsed_version = Version(version)
+        except InvalidVersion as ex:
             raise ValidationError(
-                f"The {version_type} version {version} specified is not a valid version format (ex. 1.2.3)"
-            )
-        return tuple(int(part) for part in version_parts)  # type: ignore # We already verified there were 3 parts and they are all digits
+                f"The {version_type} version '{version}' is not a valid version format."
+            ) from ex
+        else:
+            # No exception
+            return parsed_version
 
     def _stream_configs(self, manifest: Mapping[str, Any]) -> List[Dict[str, Any]]:
         # This has a warning flag for static, but after we finish part 4 we'll replace manifest with self._source_config

airbyte_cdk/sources/file_based/file_types/unstructured_parser.py

@@ -29,16 +29,25 @@ from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
 from airbyte_cdk.utils import is_cloud_environment
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from unstructured.file_utils.filetype import (
+    EXT_TO_FILETYPE,
     FILETYPE_TO_MIMETYPE,
     STR_TO_FILETYPE,
     FileType,
     detect_filetype,
 )
+import nltk
 
 unstructured_partition_pdf = None
 unstructured_partition_docx = None
 unstructured_partition_pptx = None
 
+try:
+    nltk.data.find("tokenizers/punkt.zip")
+    nltk.data.find("tokenizers/punkt_tab.zip")
+except LookupError:
+    nltk.download("punkt")
+    nltk.download("punkt_tab")
+
 
 def optional_decode(contents: Union[str, bytes]) -> str:
     if isinstance(contents, bytes):
@@ -108,9 +117,11 @@ class UnstructuredParser(FileTypeParser):
         format = _extract_format(config)
         with stream_reader.open_file(file, self.file_read_mode, None, logger) as file_handle:
             filetype = self._get_filetype(file_handle, file)
-
             if filetype not in self._supported_file_types() and not format.skip_unprocessable_files:
-                raise self._create_parse_error(file, self._get_file_type_error_message(filetype))
+                raise self._create_parse_error(
+                    file,
+                    self._get_file_type_error_message(filetype),
+                )
 
             return {
                 "content": {
@@ -159,6 +170,10 @@ class UnstructuredParser(FileTypeParser):
                     logger.warn(f"File {file.uri} cannot be parsed. Skipping it.")
                 else:
                     raise e
+            except Exception as e:
+                exception_str = str(e)
+                logger.error(f"File {file.uri} caused an error during parsing: {exception_str}.")
+                raise e
 
     def _read_file(
         self,
@@ -176,20 +191,32 @@ class UnstructuredParser(FileTypeParser):
             # check whether unstructured library is actually available for better error message and to ensure proper typing (can't be None after this point)
             raise Exception("unstructured library is not available")
 
-        filetype = self._get_filetype(file_handle, remote_file)
+        filetype: FileType | None = self._get_filetype(file_handle, remote_file)
 
-        if filetype == FileType.MD or filetype == FileType.TXT:
+        if filetype is None or filetype not in self._supported_file_types():
+            raise self._create_parse_error(
+                remote_file,
+                self._get_file_type_error_message(filetype),
+            )
+        if filetype in {FileType.MD, FileType.TXT}:
             file_content: bytes = file_handle.read()
             decoded_content: str = optional_decode(file_content)
             return decoded_content
-        if filetype not in self._supported_file_types():
-            raise self._create_parse_error(remote_file, self._get_file_type_error_message(filetype))
         if format.processing.mode == "local":
-            return self._read_file_locally(file_handle, filetype, format.strategy, remote_file)
+            return self._read_file_locally(
+                file_handle,
+                filetype,
+                format.strategy,
+                remote_file,
+            )
         elif format.processing.mode == "api":
             try:
                 result: str = self._read_file_remotely_with_retries(
-                    file_handle, format.processing, filetype, format.strategy, remote_file
+                    file_handle,
+                    format.processing,
+                    filetype,
+                    format.strategy,
+                    remote_file,
                 )
             except Exception as e:
                 # If a parser error happens during remotely processing the file, this means the file is corrupted. This case is handled by the parse_records method, so just rethrow.
@@ -336,7 +363,11 @@ class UnstructuredParser(FileTypeParser):
 
         return self._render_markdown([element.to_dict() for element in elements])
 
-    def _create_parse_error(self, remote_file: RemoteFile, message: str) -> RecordParseError:
+    def _create_parse_error(
+        self,
+        remote_file: RemoteFile,
+        message: str,
+    ) -> RecordParseError:
         return RecordParseError(
             FileBasedSourceError.ERROR_PARSING_RECORD, filename=remote_file.uri, message=message
         )
@@ -360,32 +391,51 @@ class UnstructuredParser(FileTypeParser):
         # detect_filetype is either using the file name or file content
         # if possible, try to leverage the file name to detect the file type
         # if the file name is not available, use the file content
-        file_type = detect_filetype(
-            filename=remote_file.uri,
-        )
-        if file_type is not None and not file_type == FileType.UNK:
+        file_type: FileType | None = None
+        try:
+            file_type = detect_filetype(
+                filename=remote_file.uri,
+            )
+        except Exception:
+            # Path doesn't exist locally. Try something else...
+            pass
+
+        if file_type and file_type != FileType.UNK:
             return file_type
 
         type_based_on_content = detect_filetype(file=file)
+        file.seek(0)  # detect_filetype is reading to read the file content, so we need to reset
 
-        # detect_filetype is reading to read the file content
-        file.seek(0)
+        if type_based_on_content and type_based_on_content != FileType.UNK:
+            return type_based_on_content
 
-        return type_based_on_content
+        extension = "." + remote_file.uri.split(".")[-1].lower()
+        if extension in EXT_TO_FILETYPE:
+            return EXT_TO_FILETYPE[extension]
+
+        return None
 
     def _supported_file_types(self) -> List[Any]:
         return [FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT]
 
-    def _get_file_type_error_message(self, file_type: FileType) -> str:
+    def _get_file_type_error_message(
+        self,
+        file_type: FileType | None,
+    ) -> str:
         supported_file_types = ", ".join([str(type) for type in self._supported_file_types()])
-        return f"File type {file_type} is not supported. Supported file types are {supported_file_types}"
+        return f"File type {file_type or 'None'!s} is not supported. Supported file types are {supported_file_types}"
 
     def _render_markdown(self, elements: List[Any]) -> str:
         return "\n\n".join((self._convert_to_markdown(el) for el in elements))
 
     def _convert_to_markdown(self, el: Dict[str, Any]) -> str:
         if dpath.get(el, "type") == "Title":
-            heading_str = "#" * (dpath.get(el, "metadata/category_depth", default=1) or 1)
+            category_depth = dpath.get(el, "metadata/category_depth", default=1) or 1
+            if not isinstance(category_depth, int):
+                category_depth = (
+                    int(category_depth) if isinstance(category_depth, (str, float)) else 1
+                )
+            heading_str = "#" * category_depth
             return f"{heading_str} {dpath.get(el, 'text')}"
         elif dpath.get(el, "type") == "ListItem":
             return f"- {dpath.get(el, 'text')}"

{airbyte_cdk-6.7.0rc1.dist-info → airbyte_cdk-6.7.0rc2.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.7.0rc1
+Version: 6.7.0rc2
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
 Keywords: airbyte,connector-development-kit,cdk
 Author: Airbyte
 Author-email: contact@airbyte.io
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.10,<3.13
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -15,7 +15,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Provides-Extra: file-based
@@ -42,7 +41,7 @@ Requires-Dist: jsonschema (>=3.2.0,<3.3.0)
 Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
 Requires-Dist: langchain_core (==0.1.42)
 Requires-Dist: markdown ; extra == "file-based"
-Requires-Dist: nltk (==3.8.1)
+Requires-Dist: nltk (==3.9.1)
 Requires-Dist: numpy (<2)
 Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
 Requires-Dist: orjson (>=3.10.7,<4.0.0)

{airbyte_cdk-6.7.0rc1.dist-info → airbyte_cdk-6.7.0rc2.dist-info}/RECORD

@@ -95,7 +95,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=LYEZ
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=-V5UddGm69UKEB6o_O1EIES9kfY8FV_X4Ji8w1yOuSA,981
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=KwTd0oagnZI4tARxnJZlQiDHn1IXqS7dbnRT0rKRAj8,6626
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=QgIfSVPHx_MMUCgbQdm-NMpUlp_cpk0OQhoRDFtkrxE,4040
-airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=LjOyya1Eh3x3NOO_0MIbiev57OniJUyKkQ3dheT16kI,12896
+airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=SR5s8gbLmuQwCd4mgiNyk1tttocMC1bJbkA582TzbYA,12918
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iNsF3jWCaZAmJYArmDQg0MJgZikk6frh3IfhcMBR_Qc,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
@@ -202,7 +202,7 @@ airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=HyGRihJxcb_lEs
 airbyte_cdk/sources/file_based/file_types/file_type_parser.py,sha256=JgpH21PrbRqwK92BJklZWvh2TndA6xZ-eP1LPMo44oQ,2832
 airbyte_cdk/sources/file_based/file_types/jsonl_parser.py,sha256=k1ri7TtwrN8oYZpCl1bNNeAQmwBbwLjmOmIz8-tKflY,5897
 airbyte_cdk/sources/file_based/file_types/parquet_parser.py,sha256=0B4RYehU4z4dys3Tu-O98B0Uw7JO_LzStRwmNxKh6Xk,10486
-airbyte_cdk/sources/file_based/file_types/unstructured_parser.py,sha256=oR0XdsMLpOh9kXzkVuqZbIfxzsREeWYCWpWY2vlyVHk,17171
+airbyte_cdk/sources/file_based/file_types/unstructured_parser.py,sha256=3dlhzKq4Vuc-E01uG5TLPAn0E8Be2SrBNukeP22RNFM,18601
 airbyte_cdk/sources/file_based/remote_file.py,sha256=yqRz93vPe8PBXLIMJ5W5u2JRlZRhg6sBrAjn3pPjJ8A,315
 airbyte_cdk/sources/file_based/schema_helpers.py,sha256=Cf8FH1bDFP0qCDDfEYir_WjP4exXUnikz8hZ40y1Ek0,9601
 airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=sEVnRhZ8x9f7PNjo6lewxid9z0PI8eSj7gSoFC3MH1Y,527
@@ -327,7 +327,7 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EemcgcQlI8-LPYOPlYv4Qkdjyho79XVLWaUHF5X
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=LVc9KbtMeV_z99jWo0Ou8u4l6eBJ0BWNhxj4zrrGKRs,763
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=89TQdFuYZ1NJgmFpqLzY_T_T_64TpJYmVqs119Bp43g,6164
-airbyte_cdk-6.7.0rc1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.7.0rc1.dist-info/METADATA,sha256=CCsHOmSWEHYUT7Yd4UNppLGoDi6EcBvFr9fw5o0A9Dc,13350
-airbyte_cdk-6.7.0rc1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.7.0rc1.dist-info/RECORD,,
+airbyte_cdk-6.7.0rc2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.7.0rc2.dist-info/METADATA,sha256=0dmVuMrkaui3uUXZp0tg1CfHzY58AHUPJGAGQhk4VRw,13300
+airbyte_cdk-6.7.0rc2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.7.0rc2.dist-info/RECORD,,