Flowfile 0.4.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

flowfile_core/schemas/analysis_schemas/graphic_walker_schemas.py

@@ -1,4 +1,5 @@
-from typing import Any, Dict, List, Optional, Literal
+from typing import Any, Literal
+
 from pydantic import BaseModel, Field, model_validator
 
 
@@ -12,107 +13,106 @@ class Expression(BaseModel):
     expression: str  # Example attribute
 
 
-AnalyticTypeLit = Literal['measure', 'dimension']
+AnalyticTypeLit = Literal["measure", "dimension"]
 
 
 class IField(BaseModel):
     fid: str
     name: str
-    basename: Optional[str] = None
+    basename: str | None = None
     semanticType: str
     analyticType: AnalyticTypeLit
-    cmp: Optional[str] = None
-    geoRole: Optional[GeoRole] = None
-    computed: Optional[bool] = None
-    expression: Optional[str] = None
-    timeUnit: Optional[str] = None
-    path: Optional[List[str]] = None
-    offset: Optional[int] = None
-    aggName: Optional[str] = None
-    aggregated: Optional[bool] = None
-
-    @model_validator(mode='after')
+    cmp: str | None = None
+    geoRole: GeoRole | None = None
+    computed: bool | None = None
+    expression: str | None = None
+    timeUnit: str | None = None
+    path: list[str] | None = None
+    offset: int | None = None
+    aggName: str | None = None
+    aggregated: bool | None = None
+
+    @model_validator(mode="after")
     def set_default_aggname(self):
-        if self.aggName is None and self.analyticType == 'measure':
+        if self.aggName is None and self.analyticType == "measure":
             self.aggName = "sum"
         return self
 
     def model_dump_dict(self):
         d = self.model_dump(exclude_none=True)
-        d['offset'] = None
+        d["offset"] = None
         return d
 
 
 class ViewField(IField):
-    sort: Optional[str] = None
+    sort: str | None = None
 
 
 class FilterField(ViewField):
     rule: Any
-    enableAgg: Optional[bool] = False
+    enableAgg: bool | None = False
 
 
 class DraggableFieldState(BaseModel):
-    dimensions: List[ViewField]
-    measures: List[ViewField]
-    rows: List[ViewField]
-    columns: List[ViewField]
-    color: List[ViewField]
-    opacity: List[ViewField]
-    size: List[ViewField]
-    shape: List[ViewField]
-    theta: List[ViewField]
-    radius: List[ViewField]
-    longitude: List[ViewField]
-    latitude: List[ViewField]
-    geoId: List[ViewField]
-    details: List[ViewField]
-    filters: List[FilterField]
-    text: List[ViewField]
+    dimensions: list[ViewField]
+    measures: list[ViewField]
+    rows: list[ViewField]
+    columns: list[ViewField]
+    color: list[ViewField]
+    opacity: list[ViewField]
+    size: list[ViewField]
+    shape: list[ViewField]
+    theta: list[ViewField]
+    radius: list[ViewField]
+    longitude: list[ViewField]
+    latitude: list[ViewField]
+    geoId: list[ViewField]
+    details: list[ViewField]
+    filters: list[FilterField]
+    text: list[ViewField]
 
 
 class ConfigScale(BaseModel):
-    rangeMax: Optional[int]
-    rangeMin: Optional[int]
-    domainMin: Optional[int]
-    domainMax: Optional[int]
+    rangeMax: int | None
+    rangeMin: int | None
+    domainMin: int | None
+    domainMax: int | None
 
 
 class MutField(BaseModel):
     fid: str
-    key: Optional[str] = None
-    name: Optional[str] = None
-    basename: Optional[str] = None
-    disable: Optional[bool] = False
+    key: str | None = None
+    name: str | None = None
+    basename: str | None = None
+    disable: bool | None = False
     semanticType: str
     analyticType: AnalyticTypeLit
-    path: Optional[List[str]] = None
-    offset: Optional[int] = None
+    path: list[str] | None = None
+    offset: int | None = None
 
 
 class DataModel(BaseModel):
-    data: List[Dict[str, Any]]
-    fields: List[MutField]
+    data: list[dict[str, Any]]
+    fields: list[MutField]
 
 
-class IVisualConfigNew (BaseModel):
+class IVisualConfigNew(BaseModel):
     defaultAggregated: bool
-    geoms: List[str]
-    coordSystem: Optional[str]
+    geoms: list[str]
+    coordSystem: str | None
     limit: int = None
-    folds: Optional[List[str]] = []
-    timezoneDisplayOffset: Optional[int] = None
+    folds: list[str] | None = []
+    timezoneDisplayOffset: int | None = None
 
 
 class Chart(BaseModel):
     visId: str
-    name: Optional[str]
+    name: str | None
     encodings: DraggableFieldState
     config: IVisualConfigNew
 
 
-class GraphicWalkerInput (BaseModel):
+class GraphicWalkerInput(BaseModel):
     dataModel: DataModel = Field(default_factory=lambda: DataModel(data=[], fields=[]))
     is_initial: bool = True
-    specList: Optional[List[Any]] = None
-
+    specList: list[Any] | None = None

flowfile_core/schemas/cloud_storage_schemas.py

@@ -1,18 +1,20 @@
 """Cloud storage connection schemas for S3, ADLS, and other cloud providers."""
 
-from typing import Optional, Literal
-import polars as pl
 import base64
+from typing import Literal
 
-from pydantic import BaseModel, SecretStr, field_validator, Field
+import polars as pl
+from pydantic import BaseModel, SecretStr, field_validator
 
 from flowfile_core.secret_manager.secret_manager import encrypt_secret
 
 CloudStorageType = Literal["s3", "adls", "gcs"]
-AuthMethod = Literal["access_key", "iam_role", "service_principal", "managed_identity", "sas_token", "aws-cli", "env_vars"]
+AuthMethod = Literal[
+    "access_key", "iam_role", "service_principal", "managed_identity", "sas_token", "aws-cli", "env_vars"
+]
 
 
-def encrypt_for_worker(secret_value: SecretStr|None) -> str|None:
+def encrypt_for_worker(secret_value: SecretStr | None) -> str | None:
     """
     Encrypts a secret value for use in worker contexts.
     This is a placeholder function that simulates encryption.
@@ -27,31 +29,32 @@ class AuthSettingsInput(BaseModel):
     The information needed for the user to provide the details that are needed to provide how to connect to the
      Cloud provider
     """
+
     storage_type: CloudStorageType
     auth_method: AuthMethod
-    connection_name: Optional[str] = "None"  # This is the reference to the item we will fetch that contains the data
+    connection_name: str | None = "None"  # This is the reference to the item we will fetch that contains the data
 
 
 class FullCloudStorageConnectionWorkerInterface(AuthSettingsInput):
     """Internal model with decrypted secrets"""
 
     # AWS S3
-    aws_region: Optional[str] = None
-    aws_access_key_id: Optional[str] = None
-    aws_secret_access_key: Optional[str] = None
-    aws_role_arn: Optional[str] = None
-    aws_allow_unsafe_html: Optional[bool] = None
-    aws_session_token: Optional[str] = None
+    aws_region: str | None = None
+    aws_access_key_id: str | None = None
+    aws_secret_access_key: str | None = None
+    aws_role_arn: str | None = None
+    aws_allow_unsafe_html: bool | None = None
+    aws_session_token: str | None = None
 
     # Azure ADLS
-    azure_account_name: Optional[str] = None
-    azure_account_key: Optional[str] = None
-    azure_tenant_id: Optional[str] = None
-    azure_client_id: Optional[str] = None
-    azure_client_secret: Optional[str] = None
+    azure_account_name: str | None = None
+    azure_account_key: str | None = None
+    azure_tenant_id: str | None = None
+    azure_client_id: str | None = None
+    azure_client_secret: str | None = None
 
     # Common
-    endpoint_url: Optional[str] = None
+    endpoint_url: str | None = None
     verify_ssl: bool = True
 
 
@@ -59,22 +62,22 @@ class FullCloudStorageConnection(AuthSettingsInput):
     """Internal model with decrypted secrets"""
 
     # AWS S3
-    aws_region: Optional[str] = None
-    aws_access_key_id: Optional[str] = None
-    aws_secret_access_key: Optional[SecretStr] = None
-    aws_role_arn: Optional[str] = None
-    aws_allow_unsafe_html: Optional[bool] = None
-    aws_session_token: Optional[SecretStr] = None
+    aws_region: str | None = None
+    aws_access_key_id: str | None = None
+    aws_secret_access_key: SecretStr | None = None
+    aws_role_arn: str | None = None
+    aws_allow_unsafe_html: bool | None = None
+    aws_session_token: SecretStr | None = None
 
     # Azure ADLS
-    azure_account_name: Optional[str] = None
-    azure_account_key: Optional[SecretStr] = None
-    azure_tenant_id: Optional[str] = None
-    azure_client_id: Optional[str] = None
-    azure_client_secret: Optional[SecretStr] = None
+    azure_account_name: str | None = None
+    azure_account_key: SecretStr | None = None
+    azure_tenant_id: str | None = None
+    azure_client_id: str | None = None
+    azure_client_secret: SecretStr | None = None
 
     # Common
-    endpoint_url: Optional[str] = None
+    endpoint_url: str | None = None
     verify_ssl: bool = True
 
     def get_worker_interface(self) -> "FullCloudStorageConnectionWorkerInterface":
@@ -97,7 +100,7 @@ class FullCloudStorageConnection(AuthSettingsInput):
             azure_client_id=self.azure_client_id,
             azure_client_secret=encrypt_for_worker(self.azure_client_secret),
             endpoint_url=self.endpoint_url,
-            verify_ssl=self.verify_ssl
+            verify_ssl=self.verify_ssl,
         )
 
 
@@ -105,14 +108,14 @@ class FullCloudStorageConnectionInterface(AuthSettingsInput):
     """API response model - no secrets exposed"""
 
     # Public fields only
-    aws_allow_unsafe_html: Optional[bool] = None
-    aws_region: Optional[str] = None
-    aws_access_key_id: Optional[str] = None
-    aws_role_arn: Optional[str] = None
-    azure_account_name: Optional[str] = None
-    azure_tenant_id: Optional[str] = None
-    azure_client_id: Optional[str] = None
-    endpoint_url: Optional[str] = None
+    aws_allow_unsafe_html: bool | None = None
+    aws_region: str | None = None
+    aws_access_key_id: str | None = None
+    aws_role_arn: str | None = None
+    azure_account_name: str | None = None
+    azure_tenant_id: str | None = None
+    azure_client_id: str | None = None
+    endpoint_url: str | None = None
     verify_ssl: bool = True
 
 
@@ -120,7 +123,7 @@ class CloudStorageSettings(BaseModel):
     """Settings for cloud storage nodes in the visual designer"""
 
     auth_mode: AuthMethod = "auto"
-    connection_name: Optional[str] = None  # Required only for 'reference' mode
+    connection_name: str | None = None  # Required only for 'reference' mode
     resource_path: str  # s3://bucket/path/to/file.csv
 
     @field_validator("auth_mode", mode="after")
@@ -136,12 +139,10 @@ class CloudStorageReadSettings(CloudStorageSettings):
 
     scan_mode: Literal["single_file", "directory"] = "single_file"
     file_format: Literal["csv", "parquet", "json", "delta", "iceberg"] = "parquet"
-    # CSV specific options
-    csv_has_header: Optional[bool] = True
-    csv_delimiter: Optional[str] = ","
-    csv_encoding: Optional[str] = "utf8"
-    # Deltalake specific settings
-    delta_version: Optional[int] = None
+    csv_has_header: bool | None = True
+    csv_delimiter: str | None = ","
+    csv_encoding: str | None = "utf8"
+    delta_version: int | None = None
 
 
 class CloudStorageReadSettingsInternal(BaseModel):
@@ -151,6 +152,7 @@ class CloudStorageReadSettingsInternal(BaseModel):
 
 class WriteSettingsWorkerInterface(BaseModel):
     """Settings for writing to cloud storage"""
+
     resource_path: str  # s3://bucket/path/to/file.csv
 
     write_mode: Literal["overwrite", "append"] = "overwrite"
@@ -164,6 +166,7 @@ class WriteSettingsWorkerInterface(BaseModel):
 
 class CloudStorageWriteSettings(CloudStorageSettings, WriteSettingsWorkerInterface):
     """Settings for writing to cloud storage"""
+
     pass
 
     def get_write_setting_worker_interface(self) -> WriteSettingsWorkerInterface:
@@ -176,7 +179,7 @@ class CloudStorageWriteSettings(CloudStorageSettings, WriteSettingsWorkerInterfa
             file_format=self.file_format,
             parquet_compression=self.parquet_compression,
             csv_delimiter=self.csv_delimiter,
-            csv_encoding=self.csv_encoding
+            csv_encoding=self.csv_encoding,
         )
 
 
@@ -187,6 +190,7 @@ class CloudStorageWriteSettingsInternal(BaseModel):
 
 class CloudStorageWriteSettingsWorkerInterface(BaseModel):
     """Settings for writing to cloud storage in worker context"""
+
     operation: str
     write_settings: WriteSettingsWorkerInterface
     connection: FullCloudStorageConnectionWorkerInterface
@@ -195,12 +199,12 @@ class CloudStorageWriteSettingsWorkerInterface(BaseModel):
 
 
 def get_cloud_storage_write_settings_worker_interface(
-        write_settings: CloudStorageWriteSettings,
-        connection: FullCloudStorageConnection,
-        lf: pl.LazyFrame,
-        flowfile_flow_id: int = 1,
-        flowfile_node_id: int | str = -1,
-        ) -> CloudStorageWriteSettingsWorkerInterface:
+    write_settings: CloudStorageWriteSettings,
+    connection: FullCloudStorageConnection,
+    lf: pl.LazyFrame,
+    flowfile_flow_id: int = 1,
+    flowfile_node_id: int | str = -1,
+) -> CloudStorageWriteSettingsWorkerInterface:
     """
     Convert to a worker interface model with hashed secrets.
     """
@@ -211,5 +215,5 @@ def get_cloud_storage_write_settings_worker_interface(
         write_settings=write_settings.get_write_setting_worker_interface(),
         connection=connection.get_worker_interface(),
         flowfile_flow_id=flowfile_flow_id,  # Default value, can be overridden
-        flowfile_node_id=flowfile_node_id  # Default value, can be overridden
-    )
+        flowfile_node_id=flowfile_node_id,  # Default value, can be overridden
+    )