Flowfile 0.3.4.1__py3-none-any.whl → 0.3.6__py3-none-any.whl

flowfile/web/static/assets/dropDownGeneric-60f56a8a.js

@@ -1,72 +0,0 @@
-import { C as ColumnSelector } from "./dropDown-52790b15.js";
-import { d as defineComponent, r as ref, m as watch, c as openBlock, e as createElementBlock, t as toDisplayString, i as createCommentVNode, p as createBaseVNode, f as createVNode, _ as _export_sfc } from "./index-e235a8bc.js";
-const _hoisted_1 = {
-  key: 0,
-  class: "label"
-};
-const _hoisted_2 = { class: "select-wrapper" };
-const _sfc_main = /* @__PURE__ */ defineComponent({
-  __name: "dropDownGeneric",
-  props: {
-    modelValue: {
-      type: String,
-      default: "NewField"
-    },
-    optionList: {
-      type: Array,
-      required: true
-    },
-    title: {
-      type: String,
-      default: ""
-    },
-    allowOther: {
-      type: Boolean,
-      default: true
-    },
-    placeholder: {
-      type: String,
-      default: "Select an option"
-    },
-    isLoading: {
-      type: Boolean,
-      default: false
-    }
-  },
-  emits: ["update:modelValue", "change"],
-  setup(__props, { emit: __emit }) {
-    const props = __props;
-    const emit = __emit;
-    const localSelectedValue = ref(props.modelValue);
-    watch(
-      () => props.modelValue,
-      (newVal) => {
-        localSelectedValue.value = newVal;
-      }
-    );
-    watch(localSelectedValue, (newVal) => {
-      emit("update:modelValue", newVal);
-      emit("change", newVal);
-    });
-    return (_ctx, _cache) => {
-      return openBlock(), createElementBlock("div", null, [
-        __props.title !== "" ? (openBlock(), createElementBlock("p", _hoisted_1, toDisplayString(__props.title), 1)) : createCommentVNode("", true),
-        createBaseVNode("div", _hoisted_2, [
-          createVNode(ColumnSelector, {
-            modelValue: localSelectedValue.value,
-            "onUpdate:modelValue": _cache[0] || (_cache[0] = ($event) => localSelectedValue.value = $event),
-            "column-options": __props.optionList,
-            "allow-other": __props.allowOther,
-            placeholder: __props.placeholder,
-            "is-loading": __props.isLoading
-          }, null, 8, ["modelValue", "column-options", "allow-other", "placeholder", "is-loading"])
-        ])
-      ]);
-    };
-  }
-});
-const dropDownGeneric_vue_vue_type_style_index_0_scoped_f2958f57_lang = "";
-const DropDownGeneric = /* @__PURE__ */ _export_sfc(_sfc_main, [["__scopeId", "data-v-f2958f57"]]);
-export {
-  DropDownGeneric as D
-};

flowfile/web/static/assets/dropDownGeneric-895680d6.css

@@ -1,10 +0,0 @@
-
-.label[data-v-f2958f57] {
-  font-weight: bold;
-  margin-bottom: 8px;
-  color: #333;
-}
-.select-wrapper[data-v-f2958f57] {
-  width: 100%;
-  position: relative;
-}

flowfile_core/flowfile/sources/external_sources/airbyte_sources/airbyte.py

@@ -1,159 +0,0 @@
-import os
-from ast import literal_eval
-import polars as pl
-from typing import Any, Dict, Generator, List, Optional
-from flowfile_core.configs import logger
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
-from flowfile_core.flowfile.sources.external_sources.base_class import ExternalDataSource
-from flowfile_core.flowfile.sources.external_sources.airbyte_sources.models import (
-    AirbyteProperty, JsonSchema, AirbyteResponse, AirbyteSettings
-)
-
-
-class LazyAirbyteImporter:
-    """Lazy importer for airbyte module."""
-    _airbyte = None
-
-    @classmethod
-    def get_airbyte(cls):
-        if cls._airbyte is None:
-            logger.info("Importing airbyte module")
-            import airbyte as ab
-            cls._airbyte = ab
-        return cls._airbyte
-
-
-class AirbyteSource(ExternalDataSource):
-    stream: str
-    source_name: str
-    cache_store: Optional['airbyte.results.ReadResult'] = None
-    _type: str
-    is_collected: bool
-    _airbyte_response: Optional[AirbyteResponse] = None
-    _airbyte_module = None
-    _enforce_full_refresh: Optional[bool] = True
-    version: Optional[str] = None
-
-    def __init__(self, airbyte_settings: AirbyteSettings):
-        self.is_collected = False
-        self._airbyte_response = None
-        self.stream = airbyte_settings.stream
-        self.source_name = airbyte_settings.source_name
-        self._enforce_full_refresh = airbyte_settings.enforce_full_refresh
-        if hasattr(airbyte_settings, 'version'):
-            self.version = airbyte_settings.version
-
-        # Handle config
-        if airbyte_settings.config_ref and not airbyte_settings.config:
-            logger.info(f"Getting config from {airbyte_settings.config_ref}")
-            config = literal_eval(os.environ.get(airbyte_settings.config_ref))
-        else:
-            logger.info(f"Using provided config")
-            config = airbyte_settings.config
-
-        if config is None:
-            raise ValueError("Config must be provided")
-
-        self.config = config
-        self._type = 'airbyte'
-        self.read_result = None
-
-        # Only load source if fields aren't provided
-        if not airbyte_settings.fields:
-            self.load_source(airbyte_settings)
-        else:
-            logger.info('Using provided schema')
-            self.schema = [
-                FlowfileColumn.from_input(column_name=col.name, data_type=col.data_type)
-                for col in airbyte_settings.fields
-            ]
-
-    def load_source(self, airbyte_settings: AirbyteSettings):
-        logger.info(f"Loading source {self.source_name}")
-        if airbyte_settings.fields is not None and len(airbyte_settings.fields) > 0:
-            logger.info('Using provided schema')
-            self.schema = [
-                FlowfileColumn.from_input(column_name=col.name, data_type=col.data_type)
-                for col in airbyte_settings.fields
-            ]
-        else:
-            logger.info('Using airbyte schema')
-            logger.info(f"Loading source {self.source_name}")
-            _ = self.airbyte_response
-
-    @property
-    def airbyte_response(self) -> AirbyteResponse:
-        if self._airbyte_response is None:
-            # Lazy import airbyte
-            ab = LazyAirbyteImporter.get_airbyte()
-
-            source = ab.get_source(
-                name=self.source_name,
-                config=self.config,
-                streams=self.stream,
-                docker_image=True,
-                version=self.version
-            )
-
-            try:
-                source.check()
-            except Exception:
-                logger.warning('Source check failed, trying to continue')
-
-            logger.info(f'Source check passed, starting to load data for {self.stream}')
-
-            json_schema = source.get_stream_json_schema(self.stream)['properties']
-            properties = [
-                AirbyteProperty(name=name, json_schema=JsonSchema(**schema))
-                for name, schema in json_schema.items()
-            ]
-
-            logger.info(f"Loaded source {self.source_name}")
-            self._airbyte_response = AirbyteResponse(properties=properties, source=source)
-            self.schema = self.parse_schema(self._airbyte_response)
-
-        return self._airbyte_response
-
-    def get_initial_data(self):
-        return []
-
-    def get_iter(self) -> Generator[Dict[str, Any], None, None]:
-        logger.warning('Getting data in iteration, this is suboptimal')
-        data = self.data_getter()
-        for row in data:
-            yield row
-        self.is_collected = True
-
-    def get_sample(self, n: int = 10000):
-        logger.warning('Getting sample in iteration, this is suboptimal')
-        data = self.get_iter()
-        for i in range(n):
-            try:
-                yield next(data)
-            except StopIteration:
-                break
-
-    @staticmethod
-    def parse_schema(airbyte_response: AirbyteResponse) -> List[FlowfileColumn]:
-        return airbyte_response.get_flow_file_columns()
-
-    def get_df(self):
-        if self.read_result is None:
-            self.read_result = self.airbyte_response.source.read()
-
-        df = self.read_result[self.stream].to_pandas()
-        drop_cols = [c for c in df.columns if c.startswith('_airbyte')]
-        df.drop(drop_cols, axis=1, inplace=True)
-        return df
-
-    def get_pl_df(self) -> pl.DataFrame:
-        self.is_collected = True
-        return pl.from_pandas(self.get_df())
-
-    def data_getter(self) -> List[Dict]:
-        return self.get_df().to_dict(orient='records')
-
-    @classmethod
-    def create_from_frontend_input(cls, config: Any, stream_name: str, source_name: str):
-        # Implementation details to be added
-        pass

flowfile_core/flowfile/sources/external_sources/airbyte_sources/models.py

@@ -1,172 +0,0 @@
-from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union
-from pydantic import BaseModel, field_validator, ConfigDict
-import polars as pl
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.utils import cast_str_to_polars_type
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn
-from flowfile_core.schemas.input_schema import MinimalFieldInfo
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.polars_type import PlType
-from flowfile_core.configs import logger
-
-# Use TYPE_CHECKING to avoid circular imports
-if TYPE_CHECKING:
-    from airbyte import Source
-else:
-    Source = Any
-
-
-class LazyAirbyteSource:
-    """Lazy wrapper for airbyte Source class."""
-    _source_class = None
-
-    @classmethod
-    def get_source_class(cls):
-        if cls._source_class is None:
-            logger.info("Importing airbyte Source class")
-            from airbyte import Source
-            cls._source_class = Source
-        return cls._source_class
-
-
-class JsonSchema(BaseModel):
-    type: Optional[Union[str, List[str]]]
-    airbyte_type: Optional[Union[str, List[str]]] = None
-    format: Optional[str] = None
-
-    def get_pl_type(self) -> pl.DataType:
-        if self.format:
-            format_mapping = {
-                'date-time': 'datetime',
-                'date': 'date',
-                'time': 'time'
-            }
-            dtype = format_mapping.get(self.format, 'string')
-        else:
-            type_mapping = {
-                'string': 'string',
-                'boolean': 'bool',
-                'integer': 'int',
-                'number': 'float',
-                'array': 'string',
-                'object': 'string'
-            }
-            if isinstance(self.type, list) and len(self.type) >= 1:
-                _type_mappings = (type_mapping.get(t) for t in self.type)
-                dtype = next((t for t in _type_mappings if t is not None), self.type[0])
-            elif isinstance(self.type, list) and len(self.type) == 0:
-                dtype = 'string'
-            else:
-                dtype = type_mapping.get(self.type[0] if isinstance(self.type, list) else self.type, 'string')
-        return cast_str_to_polars_type(dtype)
-
-
-class AirbyteProperty(BaseModel):
-    name: str
-    json_schema: JsonSchema
-
-    def get_pl_type(self) -> PlType:
-        return PlType(column_name=self.name, pl_datatype=self.json_schema.get_pl_type())
-
-
-class AirbyteResponse(BaseModel):
-    source: Any  # Using Any to avoid direct Source import
-    properties: list[AirbyteProperty]
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    @field_validator('source')
-    @classmethod
-    def validate_source(cls, v: Any) -> Any:
-        source_class = LazyAirbyteSource.get_source_class()
-        if not isinstance(v, source_class):
-            raise ValueError(f"Source must be an instance of airbyte.Source, got {type(v)}")
-        return v
-
-    def get_flow_file_columns(self) -> List[FlowfileColumn]:
-        return [
-            FlowfileColumn.create_from_polars_type(c.get_pl_type(), col_index=i)
-            for i, c in enumerate(self.properties)
-        ]
-
-
-class GenericProperties(BaseModel):
-    type: str
-    title: Optional[str] = None
-    description: Optional[str] = None
-    order: Optional[int] = None
-    required: Optional[List[str]] = None
-    airbyte_secret: Optional[bool] = None
-    pattern: Optional[str] = None
-    pattern_descriptor: Optional[str] = None
-    format: Optional[str] = None
-    examples: Optional[List[Any]] = None
-    enum: Optional[List[str]] = None
-    minimum: Optional[float] = None
-    maximum: Optional[float] = None
-    items: Optional[Any] = None
-    properties: Optional[Dict[str, Any]] = None
-
-    @field_validator('items', 'properties')
-    @classmethod
-    def validate_nested(cls, value: Any) -> Any:
-        if isinstance(value, dict):
-            if 'type' in value:
-                return GenericProperties(**value)
-            return {k: GenericProperties(**v) if isinstance(v, dict) else v for k, v in value.items()}
-        return value
-
-
-class GenericSchema(BaseModel):
-    title: str
-    type: str
-    required: Optional[List[str]] = None
-    additionalProperties: Optional[bool] = None
-    properties: Dict[str, GenericProperties]
-
-
-class FieldProperty(BaseModel):
-    title: Optional[str] = None
-    type: str
-    key: str
-    description: Optional[str] = None
-    airbyte_secret: Optional[bool] = None
-    input_value: Optional[str] = None
-    default: Any
-
-
-class OverallFieldProperty(BaseModel):
-    title: Optional[str] = None
-    type: str
-    key: str
-    required: bool
-    properties: List[FieldProperty]
-    items: Optional[List[FieldProperty]]
-    isOpen: bool
-    description: Optional[str] = None
-    input_value: Optional[str] = None
-    airbyte_secret: Optional[bool] = None
-    default: Any
-
-
-class AirbyteConfigTemplate(BaseModel):
-    source_name: str
-    docs_url: Optional[str] = None
-    config_spec: Dict
-    available_streams: Optional[List[str]] = None
-
-
-class AirbyteSettings(BaseModel):
-    source_name: str
-    stream: str
-    config_ref: Optional[str] = None
-    config: Optional[Dict] = None
-    fields: Optional[List[MinimalFieldInfo]] = None
-    enforce_full_refresh: Optional[bool] = True
-    flowfile_flow_id: int
-    flowfile_node_id: int
-    version: Optional[str] = None
-
-
-def get_source_instance(*args, **kwargs) -> 'Source':
-    """Helper function to get a Source instance with lazy loading."""
-    source_class = LazyAirbyteSource.get_source_class()
-    return source_class(*args, **kwargs)

flowfile_core/flowfile/sources/external_sources/airbyte_sources/settings.py

@@ -1,173 +0,0 @@
-from typing import List, Dict, Optional, Any, Type
-from flowfile_core.configs import logger
-from flowfile_core.flowfile.sources.external_sources.airbyte_sources.models import AirbyteConfigTemplate, \
-    AirbyteSettings
-from flowfile_core.schemas.external_sources.airbyte_schemas import AirbyteConfig
-from flowfile_core.flowfile.connection_manager import connection_manager
-from flowfile_core.flowfile.connection_manager._connection_manager import Connection
-
-
-class LazyAirbyteImporter:
-    """Lazy importer for airbyte module."""
-    _airbyte = None
-
-    @classmethod
-    def get_airbyte(cls):
-        if cls._airbyte is None:
-            logger.info("Importing airbyte module")
-            import airbyte as ab
-            cls._airbyte = ab
-        return cls._airbyte
-
-
-class AirbyteConfigHandler:
-    _available_connectors: Optional[List[str]] = None
-    configs: Dict[str, AirbyteConfigTemplate]
-    _airbyte = None
-
-    def __init__(self):
-        self.configs = {}
-        self._available_connectors = None
-
-    @property
-    def airbyte(self):
-        """Lazy load airbyte module when needed."""
-        if self._airbyte is None:
-            self._airbyte = LazyAirbyteImporter.get_airbyte()
-        return self._airbyte
-
-    @property
-    def available_connectors(self) -> List[str]:
-        if self._available_connectors is None:
-            self._available_connectors = [
-                c.replace('source-', '')
-                for c in self.airbyte.get_available_connectors()
-                if c.startswith('source-')
-            ]
-        return self._available_connectors
-
-    @property
-    def available_configs(self) -> List[str]:
-        return list(self.configs.keys())
-
-    def get_config(self, config_name: str) -> AirbyteConfigTemplate:
-        """Get configuration for a specific source."""
-        logger.info(f"Getting config for {config_name}")
-
-        if config_name not in self.configs:
-            try:
-                source = self.airbyte.get_source(
-                    name=config_name,
-                    install_if_missing=True,
-                    docker_image=True
-                )
-                logger.info(f"Got source {config_name}")
-
-                self.configs[config_name] = AirbyteConfigTemplate(
-                    config_spec=source.config_spec,
-                    docs_url=source.docs_url,
-                    source_name=config_name
-                )
-            except Exception as e:
-                logger.error(f"Error getting config for {config_name}: {str(e)}")
-                raise
-
-        return self.configs[config_name]
-
-    def get_available_streams(self, config_name: str, config_settings: Any) -> List[str]:
-        """Get available streams for a specific configuration."""
-        if config_name not in self.configs:
-            raise ValueError(f"Config {config_name} not found")
-
-        logger.info(f'Getting available streams for {config_name}')
-
-        try:
-            source = self.airbyte.get_source(
-                name=config_name,
-                install_if_missing=True,
-                config=config_settings,
-                docker_image=True
-            )
-            streams = source.get_available_streams()
-            if len(streams) == 0 or streams is None:
-                raise ValueError(f"No streams found for {config_name}")
-            self.configs[config_name].available_streams = streams
-            return self.configs[config_name].available_streams
-
-        except Exception as e:
-            logger.error(f"Error getting streams for {config_name}: {str(e)}")
-            raise
-
-
-# Create singleton instance
-airbyte_config_handler = AirbyteConfigHandler()
-
-
-class AirbyteHandler:
-    """Handler for Airbyte configurations and connections."""
-    config: AirbyteConfig
-
-    def __init__(self, airbyte_config: AirbyteConfig):
-        self.config = airbyte_config
-
-    def set_airbyte_config(self, airbyte_config_in: AirbyteConfig) -> AirbyteConfig:
-        """Update the current configuration."""
-        self.config.mapped_config_spec = airbyte_config_in.mapped_config_spec
-        self.config.parsed_config = airbyte_config_in.parsed_config
-        return airbyte_config_in
-
-    def get_available_streams(self) -> List[str]:
-        """Get available streams for the current configuration."""
-        config_template = airbyte_config_handler.configs.get(self.config.full_source_name)
-
-        if not config_template:
-            logger.warning(
-                f"Config {self.config.source_name} not found, trying to recreate the config"
-            )
-            try:
-                _ = airbyte_config_handler.get_config(self.config.full_source_name)
-                logger.info(f"Config {self.config.source_name} recreated")
-            except Exception as e:
-                logger.error(f"Error recreating config: {str(e)}")
-                raise
-
-        return airbyte_config_handler.get_available_streams(
-            self.config.full_source_name,
-            self.config.mapped_config_spec
-        )
-
-    def save_connection(self, connection_name: str) -> None:
-        """Save the current configuration as a connection."""
-        connection = Connection(
-            group=self.config.source_name,
-            name=connection_name,
-            config_setting=self.config,
-            type='airbyte'
-        )
-
-        connection_manager.add_connection(
-            self.config.source_name,
-            connection_name=connection_name,
-            connection=connection
-        )
-
-
-def airbyte_settings_from_config(airbyte_config: AirbyteConfig, flow_id: int, node_id: int|str) -> AirbyteSettings:
-    """Create AirbyteSettings from an AirbyteConfig."""
-    if airbyte_config.config_mode == 'key_vault':
-        connection = connection_manager.get_connection(
-            connection_group=airbyte_config.source_name,
-            connection_name=airbyte_config.connection_name
-        )
-        config = connection.config_setting.mapped_config_spec
-    else:
-        config = airbyte_config.mapped_config_spec
-
-    return AirbyteSettings(
-        source_name=airbyte_config.full_source_name,
-        stream=airbyte_config.selected_stream,
-        config=config,
-        flowfile_flow_id=flow_id,
-        flowfile_node_id=node_id,
-        version=airbyte_config.version
-    )

flowfile_core/schemas/external_sources/airbyte_schemas.py

@@ -1,20 +0,0 @@
-from typing import TypeAlias, Optional, Dict, Any, Literal
-from pydantic import BaseModel, Field
-
-
-config_options: TypeAlias = Literal["in_line", "key_vault"]
-
-
-class AirbyteConfig(BaseModel):
-    source_name: str
-    selected_stream: Optional[str] = None
-    config_mode: config_options = "in_line"
-    mapped_config_spec: Optional[Dict[str, Any]] = Field(default_factory=dict)
-    parsed_config: Optional[Any] = None
-    connection_name: Optional[str] = None
-    version: Optional[str] = None
-
-    @property
-    def full_source_name(self) -> str:
-        return f"source-{self.source_name}"
-