airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/sources/file_based/file_types/avro_parser.py

@@ -3,13 +3,17 @@
 #
 
 import logging
-from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
+from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, cast
 
 import fastavro
+
 from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
@@ -60,19 +64,27 @@ class AvroParser(FileTypeParser):
             raise ValueError(f"Expected ParquetFormat, got {avro_format}")
 
         with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
-            avro_reader = fastavro.reader(fp)
+            avro_reader = fastavro.reader(fp)  # type: ignore [arg-type]
             avro_schema = avro_reader.writer_schema
-        if not avro_schema["type"] == "record":
-            unsupported_type = avro_schema["type"]
-            raise ValueError(f"Only record based avro files are supported. Found {unsupported_type}")
+        if not avro_schema["type"] == "record":  # type: ignore [index, call-overload]
+            unsupported_type = avro_schema["type"]  # type: ignore [index, call-overload]
+            raise ValueError(
+                f"Only record based avro files are supported. Found {unsupported_type}"
+            )
         json_schema = {
-            field["name"]: AvroParser._convert_avro_type_to_json(avro_format, field["name"], field["type"])
-            for field in avro_schema["fields"]
+            field["name"]: AvroParser._convert_avro_type_to_json(  # type: ignore [index]
+                avro_format,
+                field["name"],  # type: ignore [index]
+                field["type"],  # type: ignore [index]
+            )
+            for field in avro_schema["fields"]  # type: ignore [index, call-overload]
         }
         return json_schema
 
     @classmethod
-    def _convert_avro_type_to_json(cls, avro_format: AvroFormat, field_name: str, avro_field: str) -> Mapping[str, Any]:
+    def _convert_avro_type_to_json(
+        cls, avro_format: AvroFormat, field_name: str, avro_field: str
+    ) -> Mapping[str, Any]:
         if isinstance(avro_field, str) and avro_field in AVRO_TYPE_TO_JSON_TYPE:
             # Legacy behavior to retain backwards compatibility. Long term we should always represent doubles as strings
             if avro_field == "double" and not avro_format.double_as_string:
@@ -83,17 +95,28 @@ class AvroParser(FileTypeParser):
                 return {
                     "type": "object",
                     "properties": {
-                        object_field["name"]: AvroParser._convert_avro_type_to_json(avro_format, object_field["name"], object_field["type"])
+                        object_field["name"]: AvroParser._convert_avro_type_to_json(
+                            avro_format, object_field["name"], object_field["type"]
+                        )
                         for object_field in avro_field["fields"]
                     },
                 }
             elif avro_field["type"] == "array":
                 if "items" not in avro_field:
-                    raise ValueError(f"{field_name} array type does not have a required field items")
-                return {"type": "array", "items": AvroParser._convert_avro_type_to_json(avro_format, "", avro_field["items"])}
+                    raise ValueError(
+                        f"{field_name} array type does not have a required field items"
+                    )
+                return {
+                    "type": "array",
+                    "items": AvroParser._convert_avro_type_to_json(
+                        avro_format, "", avro_field["items"]
+                    ),
+                }
             elif avro_field["type"] == "enum":
                 if "symbols" not in avro_field:
-                    raise ValueError(f"{field_name} enum type does not have a required field symbols")
+                    raise ValueError(
+                        f"{field_name} enum type does not have a required field symbols"
+                    )
                 if "name" not in avro_field:
                     raise ValueError(f"{field_name} enum type does not have a required field name")
                 return {"type": "string", "enum": avro_field["symbols"]}
@@ -102,7 +125,9 @@ class AvroParser(FileTypeParser):
                     raise ValueError(f"{field_name} map type does not have a required field values")
                 return {
                     "type": "object",
-                    "additionalProperties": AvroParser._convert_avro_type_to_json(avro_format, "", avro_field["values"]),
+                    "additionalProperties": AvroParser._convert_avro_type_to_json(
+                        avro_format, "", avro_field["values"]
+                    ),
                 }
             elif avro_field["type"] == "fixed" and avro_field.get("logicalType") != "duration":
                 if "size" not in avro_field:
@@ -115,18 +140,27 @@ class AvroParser(FileTypeParser):
                 }
             elif avro_field.get("logicalType") == "decimal":
                 if "precision" not in avro_field:
-                    raise ValueError(f"{field_name} decimal type does not have a required field precision")
+                    raise ValueError(
+                        f"{field_name} decimal type does not have a required field precision"
+                    )
                 if "scale" not in avro_field:
-                    raise ValueError(f"{field_name} decimal type does not have a required field scale")
+                    raise ValueError(
+                        f"{field_name} decimal type does not have a required field scale"
+                    )
                 max_whole_number_range = avro_field["precision"] - avro_field["scale"]
                 decimal_range = avro_field["scale"]
 
                 # This regex looks like a mess, but it is validation for at least one whole number and optional fractional numbers
                 # For example: ^-?\d{1,5}(?:\.\d{1,3})?$ would accept 12345.123 and 123456.12345 would  be rejected
-                return {"type": "string", "pattern": f"^-?\\d{{{1,max_whole_number_range}}}(?:\\.\\d{1,decimal_range})?$"}
+                return {
+                    "type": "string",
+                    "pattern": f"^-?\\d{{{1,max_whole_number_range}}}(?:\\.\\d{1,decimal_range})?$",
+                }
             elif "logicalType" in avro_field:
                 if avro_field["logicalType"] not in AVRO_LOGICAL_TYPE_TO_JSON:
-                    raise ValueError(f"{avro_field['logical_type']} is not a valid Avro logical type")
+                    raise ValueError(
+                        f"{avro_field['logicalType']} is not a valid Avro logical type"
+                    )
                 return AVRO_LOGICAL_TYPE_TO_JSON[avro_field["logicalType"]]
             else:
                 raise ValueError(f"Unsupported avro type: {avro_field}")
@@ -148,25 +182,38 @@ class AvroParser(FileTypeParser):
         line_no = 0
         try:
             with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
-                avro_reader = fastavro.reader(fp)
+                avro_reader = fastavro.reader(fp)  # type: ignore [arg-type]
                 schema = avro_reader.writer_schema
-                schema_field_name_to_type = {field["name"]: field["type"] for field in schema["fields"]}
+                schema_field_name_to_type = {
+                    field["name"]: cast(dict[str, Any], field["type"])  # type: ignore [index]
+                    for field in schema["fields"]  # type: ignore [index, call-overload]  # If schema is not dict, it is not subscriptable by strings
+                }
                 for record in avro_reader:
                     line_no += 1
                     yield {
-                        record_field: self._to_output_value(avro_format, schema_field_name_to_type[record_field], record[record_field])
+                        record_field: self._to_output_value(
+                            avro_format,
+                            schema_field_name_to_type[record_field],  # type: ignore [index] # Any not subscriptable
+                            record[record_field],  # type: ignore [index] # Any not subscriptable
+                        )
                         for record_field, record_value in schema_field_name_to_type.items()
                     }
         except Exception as exc:
-            raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no) from exc
+            raise RecordParseError(
+                FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no
+            ) from exc
 
     @property
     def file_read_mode(self) -> FileReadMode:
         return FileReadMode.READ_BINARY
 
     @staticmethod
-    def _to_output_value(avro_format: AvroFormat, record_type: Mapping[str, Any], record_value: Any) -> Any:
-        if not isinstance(record_type, Mapping):
+    def _to_output_value(
+        avro_format: AvroFormat, record_type: Mapping[str, Any], record_value: Any
+    ) -> Any:
+        if isinstance(record_value, bytes):
+            return record_value.decode()
+        elif not isinstance(record_type, Mapping):
             if record_type == "double" and avro_format.double_as_string:
                 return str(record_value)
             return record_value
@@ -174,6 +221,10 @@ class AvroParser(FileTypeParser):
             return str(record_value)
         elif record_type.get("logicalType") == "date":
             return record_value.isoformat()
+        elif record_type.get("logicalType") == "timestamp-millis":
+            return record_value.isoformat(sep="T", timespec="milliseconds")
+        elif record_type.get("logicalType") == "timestamp-micros":
+            return record_value.isoformat(sep="T", timespec="microseconds")
         elif record_type.get("logicalType") == "local-timestamp-millis":
             return record_value.isoformat(sep="T", timespec="milliseconds")
         elif record_type.get("logicalType") == "local-timestamp-micros":

airbyte_cdk/sources/file_based/file_types/csv_parser.py

@@ -12,11 +12,21 @@ from io import IOBase
 from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Set, Tuple
 from uuid import uuid4
 
+import orjson
+
 from airbyte_cdk.models import FailureType
-from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, CsvHeaderAutogenerated, CsvHeaderUserProvided, InferenceType
+from airbyte_cdk.sources.file_based.config.csv_format import (
+    CsvFormat,
+    CsvHeaderAutogenerated,
+    CsvHeaderUserProvided,
+    InferenceType,
+)
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.schema_helpers import TYPE_PYTHON_MAPPING, SchemaType
@@ -52,7 +62,12 @@ class _CsvReader:
             quoting=csv.QUOTE_MINIMAL,
         )
         with stream_reader.open_file(file, file_read_mode, config_format.encoding, logger) as fp:
-            headers = self._get_headers(fp, config_format, dialect_name)
+            try:
+                headers = self._get_headers(fp, config_format, dialect_name)
+            except UnicodeError:
+                raise AirbyteTracedException(
+                    message=f"{FileBasedSourceError.ENCODING_ERROR.value} Expected encoding: {config_format.encoding}",
+                )
 
             rows_to_skip = (
                 config_format.skip_rows_before_header
@@ -70,13 +85,27 @@ class _CsvReader:
                     # The row was not properly parsed if any of the values are None. This will most likely occur if there are more columns
                     # than headers or more headers dans columns
                     if None in row:
-                        raise RecordParseError(
-                            FileBasedSourceError.ERROR_PARSING_RECORD_MISMATCHED_COLUMNS,
-                            filename=file.uri,
-                            lineno=lineno,
-                        )
+                        if config_format.ignore_errors_on_fields_mismatch:
+                            logger.error(
+                                f"Skipping record in line {lineno} of file {file.uri}; invalid CSV row with missing column."
+                            )
+                        else:
+                            raise RecordParseError(
+                                FileBasedSourceError.ERROR_PARSING_RECORD_MISMATCHED_COLUMNS,
+                                filename=file.uri,
+                                lineno=lineno,
+                            )
                     if None in row.values():
-                        raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD_MISMATCHED_ROWS, filename=file.uri, lineno=lineno)
+                        if config_format.ignore_errors_on_fields_mismatch:
+                            logger.error(
+                                f"Skipping record in line {lineno} of file {file.uri}; invalid CSV row with extra column."
+                            )
+                        else:
+                            raise RecordParseError(
+                                FileBasedSourceError.ERROR_PARSING_RECORD_MISMATCHED_ROWS,
+                                filename=file.uri,
+                                lineno=lineno,
+                            )
                     yield row
             finally:
                 # due to RecordParseError or GeneratorExit
@@ -88,10 +117,12 @@ class _CsvReader:
         """
         # Note that this method assumes the dialect has already been registered if we're parsing the headers
         if isinstance(config_format.header_definition, CsvHeaderUserProvided):
-            return config_format.header_definition.column_names  # type: ignore  # should be CsvHeaderUserProvided given the type
+            return config_format.header_definition.column_names
 
         if isinstance(config_format.header_definition, CsvHeaderAutogenerated):
-            self._skip_rows(fp, config_format.skip_rows_before_header + config_format.skip_rows_after_header)
+            self._skip_rows(
+                fp, config_format.skip_rows_before_header + config_format.skip_rows_after_header
+            )
             headers = self._auto_generate_headers(fp, dialect_name)
         else:
             # Then read the header
@@ -123,7 +154,11 @@ class _CsvReader:
 class CsvParser(FileTypeParser):
     _MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE = 1_000_000
 
-    def __init__(self, csv_reader: Optional[_CsvReader] = None):
+    def __init__(self, csv_reader: Optional[_CsvReader] = None, csv_field_max_bytes: int = 2**31):
+        # Increase the maximum length of data that can be parsed in a single CSV field. The default is 128k, which is typically sufficient
+        # but given the use of Airbyte in loading a large variety of data it is best to allow for a larger maximum field size to avoid
+        # skipping data on load. https://stackoverflow.com/questions/15063936/csv-error-field-larger-than-field-limit-131072
+        csv.field_size_limit(csv_field_max_bytes)
         self._csv_reader = csv_reader if csv_reader else _CsvReader()
 
     def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]:
@@ -147,11 +182,15 @@ class CsvParser(FileTypeParser):
         #  sources will likely require one. Rather than modify the interface now we can wait until the real use case
         config_format = _extract_format(config)
         type_inferrer_by_field: Dict[str, _TypeInferrer] = defaultdict(
-            lambda: _JsonTypeInferrer(config_format.true_values, config_format.false_values, config_format.null_values)
+            lambda: _JsonTypeInferrer(
+                config_format.true_values, config_format.false_values, config_format.null_values
+            )
             if config_format.inference_type != InferenceType.NONE
             else _DisabledTypeInferrer()
         )
-        data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode)
+        data_generator = self._csv_reader.read_data(
+            config, file, stream_reader, logger, self.file_read_mode
+        )
         read_bytes = 0
         for row in data_generator:
             for header, value in row.items():
@@ -169,7 +208,10 @@ class CsvParser(FileTypeParser):
                 f"Else, please contact Airbyte.",
                 failure_type=FailureType.config_error,
             )
-        schema = {header.strip(): {"type": type_inferred.infer()} for header, type_inferred in type_inferrer_by_field.items()}
+        schema = {
+            header.strip(): {"type": type_inferred.infer()}
+            for header, type_inferred in type_inferrer_by_field.items()
+        }
         data_generator.close()
         return schema
 
@@ -185,19 +227,30 @@ class CsvParser(FileTypeParser):
         try:
             config_format = _extract_format(config)
             if discovered_schema:
-                property_types = {col: prop["type"] for col, prop in discovered_schema["properties"].items()}  # type: ignore # discovered_schema["properties"] is known to be a mapping
+                property_types = {
+                    col: prop["type"] for col, prop in discovered_schema["properties"].items()
+                }
                 deduped_property_types = CsvParser._pre_propcess_property_types(property_types)
             else:
                 deduped_property_types = {}
-            cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger, config.schemaless)
-            data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode)
+            cast_fn = CsvParser._get_cast_function(
+                deduped_property_types, config_format, logger, config.schemaless
+            )
+            data_generator = self._csv_reader.read_data(
+                config, file, stream_reader, logger, self.file_read_mode
+            )
             for row in data_generator:
                 line_no += 1
                 yield CsvParser._to_nullable(
-                    cast_fn(row), deduped_property_types, config_format.null_values, config_format.strings_can_be_null
+                    cast_fn(row),
+                    deduped_property_types,
+                    config_format.null_values,
+                    config_format.strings_can_be_null,
                 )
         except RecordParseError as parse_err:
-            raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no) from parse_err
+            raise RecordParseError(
+                FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no
+            ) from parse_err
         finally:
             data_generator.close()
 
@@ -207,27 +260,47 @@ class CsvParser(FileTypeParser):
 
     @staticmethod
     def _get_cast_function(
-        deduped_property_types: Mapping[str, str], config_format: CsvFormat, logger: logging.Logger, schemaless: bool
+        deduped_property_types: Mapping[str, str],
+        config_format: CsvFormat,
+        logger: logging.Logger,
+        schemaless: bool,
     ) -> Callable[[Mapping[str, str]], Mapping[str, str]]:
         # Only cast values if the schema is provided
         if deduped_property_types and not schemaless:
-            return partial(CsvParser._cast_types, deduped_property_types=deduped_property_types, config_format=config_format, logger=logger)
+            return partial(
+                CsvParser._cast_types,
+                deduped_property_types=deduped_property_types,
+                config_format=config_format,
+                logger=logger,
+            )
         else:
             # If no schema is provided, yield the rows as they are
             return _no_cast
 
     @staticmethod
     def _to_nullable(
-        row: Mapping[str, str], deduped_property_types: Mapping[str, str], null_values: Set[str], strings_can_be_null: bool
+        row: Mapping[str, str],
+        deduped_property_types: Mapping[str, str],
+        null_values: Set[str],
+        strings_can_be_null: bool,
     ) -> Dict[str, Optional[str]]:
         nullable = {
-            k: None if CsvParser._value_is_none(v, deduped_property_types.get(k), null_values, strings_can_be_null) else v
+            k: None
+            if CsvParser._value_is_none(
+                v, deduped_property_types.get(k), null_values, strings_can_be_null
+            )
+            else v
             for k, v in row.items()
         }
         return nullable
 
     @staticmethod
-    def _value_is_none(value: Any, deduped_property_type: Optional[str], null_values: Set[str], strings_can_be_null: bool) -> bool:
+    def _value_is_none(
+        value: Any,
+        deduped_property_type: Optional[str],
+        null_values: Set[str],
+        strings_can_be_null: bool,
+    ) -> bool:
         return value in null_values and (strings_can_be_null or deduped_property_type != "string")
 
     @staticmethod
@@ -262,7 +335,10 @@ class CsvParser(FileTypeParser):
 
     @staticmethod
     def _cast_types(
-        row: Dict[str, str], deduped_property_types: Dict[str, str], config_format: CsvFormat, logger: logging.Logger
+        row: Dict[str, str],
+        deduped_property_types: Mapping[str, str],
+        config_format: CsvFormat,
+        logger: logging.Logger,
     ) -> Dict[str, Any]:
         """
         Casts the values in the input 'row' dictionary according to the types defined in the JSON schema.
@@ -287,20 +363,22 @@ class CsvParser(FileTypeParser):
                     else:
                         warnings.append(_format_warning(key, value, prop_type))
 
-                elif python_type == bool:
+                elif python_type is bool:
                     try:
-                        cast_value = _value_to_bool(value, config_format.true_values, config_format.false_values)
+                        cast_value = _value_to_bool(
+                            value, config_format.true_values, config_format.false_values
+                        )
                     except ValueError:
                         warnings.append(_format_warning(key, value, prop_type))
 
-                elif python_type == dict:
+                elif python_type is dict:
                     try:
                         # we don't re-use _value_to_object here because we type the column as object as long as there is only one object
-                        cast_value = json.loads(value)
-                    except json.JSONDecodeError:
+                        cast_value = orjson.loads(value)
+                    except orjson.JSONDecodeError:
                         warnings.append(_format_warning(key, value, prop_type))
 
-                elif python_type == list:
+                elif python_type is list:
                     try:
                         cast_value = _value_to_list(value)
                     except (ValueError, json.JSONDecodeError):
@@ -346,7 +424,9 @@ class _JsonTypeInferrer(_TypeInferrer):
     _NUMBER_TYPE = "number"
     _STRING_TYPE = "string"
 
-    def __init__(self, boolean_trues: Set[str], boolean_falses: Set[str], null_values: Set[str]) -> None:
+    def __init__(
+        self, boolean_trues: Set[str], boolean_falses: Set[str], null_values: Set[str]
+    ) -> None:
         self._boolean_trues = boolean_trues
         self._boolean_falses = boolean_falses
         self._null_values = null_values
@@ -357,7 +437,9 @@ class _JsonTypeInferrer(_TypeInferrer):
 
     def infer(self) -> str:
         types_by_value = {value: self._infer_type(value) for value in self._values}
-        types_excluding_null_values = [types for types in types_by_value.values() if self._NULL_TYPE not in types]
+        types_excluding_null_values = [
+            types for types in types_by_value.values() if self._NULL_TYPE not in types
+        ]
         if not types_excluding_null_values:
             # this is highly unusual but we will consider the column as a string
             return self._STRING_TYPE

airbyte_cdk/sources/file_based/file_types/excel_parser.py

@@ -0,0 +1,196 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from io import IOBase
+from pathlib import Path
+from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
+
+import orjson
+import pandas as pd
+from numpy import datetime64, issubdtype
+from numpy import dtype as dtype_
+from pydantic.v1 import BaseModel
+
+from airbyte_cdk.sources.file_based.config.file_based_stream_config import (
+    ExcelFormat,
+    FileBasedStreamConfig,
+)
+from airbyte_cdk.sources.file_based.exceptions import (
+    ConfigValidationError,
+    FileBasedSourceError,
+    RecordParseError,
+)
+from airbyte_cdk.sources.file_based.file_based_stream_reader import (
+    AbstractFileBasedStreamReader,
+    FileReadMode,
+)
+from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.schema_helpers import SchemaType
+
+
+class ExcelParser(FileTypeParser):
+    ENCODING = None
+
+    def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]:
+        """
+        ExcelParser does not require config checks, implicit pydantic validation is enough.
+        """
+        return True, None
+
+    async def infer_schema(
+        self,
+        config: FileBasedStreamConfig,
+        file: RemoteFile,
+        stream_reader: AbstractFileBasedStreamReader,
+        logger: logging.Logger,
+    ) -> SchemaType:
+        """
+        Infers the schema of the Excel file by examining its contents.
+
+        Args:
+            config (FileBasedStreamConfig): Configuration for the file-based stream.
+            file (RemoteFile): The remote file to be read.
+            stream_reader (AbstractFileBasedStreamReader): Reader to read the file.
+            logger (logging.Logger): Logger for logging information and errors.
+
+        Returns:
+            SchemaType: Inferred schema of the Excel file.
+        """
+
+        # Validate the format of the config
+        self.validate_format(config.format, logger)
+
+        fields: Dict[str, str] = {}
+
+        with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
+            df = self.open_and_parse_file(fp)
+            for column, df_type in df.dtypes.items():
+                # Choose the broadest data type if the column's data type differs in dataframes
+                prev_frame_column_type = fields.get(column)  # type: ignore [call-overload]
+                fields[column] = self.dtype_to_json_type(  # type: ignore [index]
+                    prev_frame_column_type,
+                    df_type,
+                )
+
+        schema = {
+            field: (
+                {"type": "string", "format": "date-time"}
+                if fields[field] == "date-time"
+                else {"type": fields[field]}
+            )
+            for field in fields
+        }
+        return schema
+
+    def parse_records(
+        self,
+        config: FileBasedStreamConfig,
+        file: RemoteFile,
+        stream_reader: AbstractFileBasedStreamReader,
+        logger: logging.Logger,
+        discovered_schema: Optional[Mapping[str, SchemaType]] = None,
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Parses records from an Excel file based on the provided configuration.
+
+        Args:
+            config (FileBasedStreamConfig): Configuration for the file-based stream.
+            file (RemoteFile): The remote file to be read.
+            stream_reader (AbstractFileBasedStreamReader): Reader to read the file.
+            logger (logging.Logger): Logger for logging information and errors.
+            discovered_schema (Optional[Mapping[str, SchemaType]]): Discovered schema for validation.
+
+        Yields:
+            Iterable[Dict[str, Any]]: Parsed records from the Excel file.
+        """
+
+        # Validate the format of the config
+        self.validate_format(config.format, logger)
+
+        try:
+            # Open and parse the file using the stream reader
+            with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp:
+                df = self.open_and_parse_file(fp)
+                # Yield records as dictionaries
+                # DataFrame.to_dict() method returns datetime values in pandas.Timestamp values, which are not serializable by orjson
+                # DataFrame.to_json() returns string with datetime values serialized to iso8601 with microseconds to align with pydantic behavior
+                # see PR description: https://github.com/airbytehq/airbyte/pull/44444/
+                yield from orjson.loads(
+                    df.to_json(orient="records", date_format="iso", date_unit="us")
+                )
+
+        except Exception as exc:
+            # Raise a RecordParseError if any exception occurs during parsing
+            raise RecordParseError(
+                FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri
+            ) from exc
+
+    @property
+    def file_read_mode(self) -> FileReadMode:
+        """
+        Returns the file read mode for the Excel file.
+
+        Returns:
+            FileReadMode: The file read mode (binary).
+        """
+        return FileReadMode.READ_BINARY
+
+    @staticmethod
+    def dtype_to_json_type(
+        current_type: Optional[str],
+        dtype: dtype_,  # type: ignore [type-arg]
+    ) -> str:
+        """
+        Convert Pandas DataFrame types to Airbyte Types.
+
+        Args:
+            current_type (Optional[str]): One of the previous types based on earlier dataframes.
+            dtype: Pandas DataFrame type.
+
+        Returns:
+            str: Corresponding Airbyte Type.
+        """
+        number_types = ("int64", "float64")
+        if current_type == "string":
+            # Previous column values were of the string type, no need to look further.
+            return current_type
+        if dtype is object:
+            return "string"
+        if dtype in number_types and (not current_type or current_type == "number"):
+            return "number"
+        if dtype == "bool" and (not current_type or current_type == "boolean"):
+            return "boolean"
+        if issubdtype(dtype, datetime64):
+            return "date-time"
+        return "string"
+
+    @staticmethod
+    def validate_format(excel_format: BaseModel, logger: logging.Logger) -> None:
+        """
+        Validates if the given format is of type ExcelFormat.
+
+        Args:
+            excel_format (Any): The format to be validated.
+
+        Raises:
+            ConfigValidationError: If the format is not ExcelFormat.
+        """
+        if not isinstance(excel_format, ExcelFormat):
+            logger.info(f"Expected ExcelFormat, got {excel_format}")
+            raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR)
+
+    @staticmethod
+    def open_and_parse_file(fp: Union[IOBase, str, Path]) -> pd.DataFrame:
+        """
+        Opens and parses the Excel file.
+
+        Args:
+            fp: File pointer to the Excel file.
+
+        Returns:
+            pd.DataFrame: Parsed data from the Excel file.
+        """
+        return pd.ExcelFile(fp, engine="calamine").parse()  # type: ignore [arg-type, call-overload, no-any-return]