omnata-plugin-runtime 0.10.22a276__tar.gz → 0.12.2a340__tar.gz

omnata_plugin_runtime-0.12.2a340/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.4
+Name: omnata-plugin-runtime
+Version: 0.12.2a340
+Summary: Classes and common runtime components for building and running Omnata Plugins
+License-File: LICENSE
+Author: James Weakley
+Author-email: james.weakley@omnata.com
+Requires-Python: >=3.10,<=3.13
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: annotated-types (<=0.6.0)
+Requires-Dist: certifi (<=2025.1.31)
+Requires-Dist: cffi (<=2.0.0)
+Requires-Dist: charset-normalizer (<=3.4.4)
+Requires-Dist: cryptography (<=46.0.3)
+Requires-Dist: filelock (<=3.20.0)
+Requires-Dist: idna (<=3.11)
+Requires-Dist: jinja2 (>=3.1.2,<=3.1.6)
+Requires-Dist: markupsafe (<=3.0.2)
+Requires-Dist: numpy (<=2.3.5)
+Requires-Dist: opentelemetry-api (<=1.38.0)
+Requires-Dist: packaging (<=25.0)
+Requires-Dist: pandas (<=2.3.3)
+Requires-Dist: platformdirs (<=4.5.0)
+Requires-Dist: protobuf (<=6.33.0)
+Requires-Dist: pyarrow (<=21.0.0)
+Requires-Dist: pycparser (<=2.23)
+Requires-Dist: pydantic (>=2,<=2.12.4)
+Requires-Dist: pydantic-core (<=2.41.5)
+Requires-Dist: pyjwt (<=2.10.1)
+Requires-Dist: pyopenssl (<=225.3.0)
+Requires-Dist: pytz (<=2025.2)
+Requires-Dist: pyyaml (<=6.0.3)
+Requires-Dist: requests (>=2,<=2.32.5)
+Requires-Dist: setuptools (<=80.9.0)
+Requires-Dist: snowflake-connector-python (>=3,<4)
+Requires-Dist: snowflake-snowpark-python (>=1.20.0,<=1.43.0)
+Requires-Dist: snowflake-telemetry-python (<=0.5.0)
+Requires-Dist: tenacity (>=8,<9)
+Requires-Dist: tomlkit (<=0.13.3)
+Requires-Dist: urllib3 (<=2.5.0)
+Requires-Dist: wheel (<=0.45.1)
+Requires-Dist: wrapt (<=2.0.1)
+Description-Content-Type: text/markdown
+
+# omnata-plugin-runtime
+This package is a runtime dependency for [Omnata Plugins](https://docs.omnata.com/omnata-product-documentation/omnata-sync-for-snowflake/plugins).
+
+It contains data classes, interfaces and application logic used to perform plugin operations.
+
+For instructions on creating plugins, visit our [docs site](https://docs.omnata.com/omnata-product-documentation/omnata-sync-for-snowflake/plugins/creating-plugins).
+
+

omnata_plugin_runtime-0.12.2a340/pyproject.toml

@@ -0,0 +1,57 @@
+[tool.poetry]
+name = "omnata-plugin-runtime"
+version = "0.12.2a340"
+description = "Classes and common runtime components for building and running Omnata Plugins"
+authors = ["James Weakley <james.weakley@omnata.com>"]
+readme = "README.md"
+packages = [{include = "omnata_plugin_runtime", from = "src"}]
+
+[tool.poetry.dependencies]
+python = ">=3.10, <=3.13"
+snowflake-snowpark-python = ">=1.20.0,<=1.43.0" # latest version available on Snowflake Anaconda, but allow pinning to 1.20.0 for to_pandas_batches workaround
+snowflake-connector-python = "^3, <=4.1.0" # latest version available on Snowflake Anaconda
+cryptography = "<=46.0.3"
+annotated-types	= "<=0.6.0"
+pycparser = "<=2.23"
+filelock = "<=3.20.0"
+pydantic-core = "<=2.41.5"
+# had to relax some of these thanks to snowcli pinning newer versions
+certifi	= "<=2025.1.31" # latest version available on Snowflake Anaconda which doesn't break the Snowflake connector
+charset-normalizer = "<=3.4.4" # latest version available on Snowflake Anaconda
+idna = "<=3.11" # latest version available on Snowflake Anaconda
+jinja2 = ">=3.1.2,<=3.1.6" # 3.1.6 was latest version available on Snowflake Anaconda
+markupsafe = "<=3.0.2" # latest version available on Snowflake Anaconda
+numpy = "<=2.3.5" # latest version available on Snowflake Anaconda
+packaging = "<=25.0" # latest version available on Snowflake Anaconda
+pandas = "<=2.3.3" # latest version available on Snowflake Anaconda
+platformdirs = "<=4.5.0" # latest version available on Snowflake Anaconda
+pydantic = "^2, <=2.12.4" # latest version available on Snowflake Anaconda
+pyjwt = "<=2.10.1" # latest version available on Snowflake Anaconda
+pyopenssl = "<=225.3.0" # latest version available on Snowflake Anaconda
+pytz = "<=2025.2" # latest version available on Snowflake Anaconda
+requests = "^2, <=2.32.5" # latest version available on Snowflake Anaconda
+setuptools = "<=80.9.0" # latest version available on Snowflake Anaconda
+tomlkit = "<=0.13.3" # latest version available on Snowflake Anaconda
+tenacity = "^8, <=9.1.2" # latest version available on Snowflake Anaconda
+urllib3 = "<=2.5.0" # latest version available on Snowflake Anaconda
+wheel = "<=0.45.1" # latest version available on Snowflake Anaconda
+pyyaml = "<=6.0.3" # latest version available on Snowflake Anaconda
+cffi = "<=2.0.0" # latest version available on Snowflake Anaconda
+pyarrow = "<=21.0.0" # latest version available on Snowflake Anaconda
+wrapt = "<=2.0.1" # latest version available on Snowflake Anaconda
+opentelemetry-api = "<=1.38.0" # latest version available on Snowflake Anaconda
+snowflake-telemetry-python = "<=0.5.0" # latest version available on Snowflake Anaconda
+protobuf = "<=6.33.0" # latest version available on Snowflake Anaconda
+
+[tool.poetry.dev-dependencies]
+pytest = "^6.2.4"
+deepdiff = "^6"
+requests-mock = ">=1.9.3"
+
+[tool.pytest.ini_options]
+addopts = ["--import-mode=importlib"]
+testpaths = ["tests"]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

{omnata_plugin_runtime-0.10.22a276 → omnata_plugin_runtime-0.12.2a340}/src/omnata_plugin_runtime/api.py

@@ -108,9 +108,9 @@ class OutboundSyncRequestPayload(BaseModel):
     Encapsulates the payload that is sent to the plugin when it is invoked to perform an outbound sync.
     """
 
-    sync_id: int  # only used by log handler
+    sync_id: int
     sync_branch_name: str = 'main'
-    sync_branch_id: Optional[int]  # only used by log handler
+    sync_branch_id: Optional[int]
     connection_id: int  # only used by log handler
     run_id: int  # used by log handler and for reporting back run status updates
     source_app_name: str  # the name of the app which is invoking this plugin
@@ -139,8 +139,8 @@ class InboundSyncRequestPayload(BaseModel):
     Encapsulates the payload that is sent to the plugin when it is invoked to perform an inbound sync.
     """
 
-    sync_id: int  # only used by log handler
-    sync_branch_name: str = 'main' # only used by rate limit updater
+    sync_id: int
+    sync_branch_name: str = 'main'
     sync_branch_id: Optional[int] = None  # only used by log handler
     connection_id: int  # only used by log handler
     run_id: int  # used by log handler and for reporting back run status updates

{omnata_plugin_runtime-0.10.22a276 → omnata_plugin_runtime-0.12.2a340}/src/omnata_plugin_runtime/configuration.py

@@ -614,6 +614,10 @@ class StreamConfiguration(SubscriptableBaseModel):
         None,
         description="Marks the stream as requiring another stream to be selected"
     )
+    mandatory: bool = Field(
+        False,
+        description="Marks the stream as mandatory, meaning it cannot be excluded from the sync configuration"
+    )
 
 
 class StoredConfigurationValue(SubscriptableBaseModel):
@@ -689,6 +693,9 @@ class ConnectionConfigurationParameters(SubscriptableBaseModel):
     _snowflake: Optional[Any] = PrivateAttr(  # or use Any to annotate the type and use Field to initialize
         default=None
     )
+    _sync_request: Optional[Any] = PrivateAttr(  # Reference to SyncRequest for worker thread access
+        default=None
+    )
 
     @model_validator(mode='after')
     def validate_ngrok_tunnel_settings(self) -> Self:
@@ -735,6 +742,23 @@ class ConnectionConfigurationParameters(SubscriptableBaseModel):
         """
         if parameter_name=='access_token' and self.access_token_secret_name is not None:
             import _snowflake # pylint: disable=import-error, import-outside-toplevel # type: ignore
+            from .threading_utils import is_managed_worker_thread
+            
+            # Check if we're in a worker thread using the explicit flag
+            # This is more reliable than checking thread names
+            if is_managed_worker_thread() and self._sync_request is not None:
+                logger.debug(f"Worker thread requesting access_token via secrets service")
+                try:
+                    secrets = self._sync_request.request_secrets_from_main_thread(
+                        self.access_token_secret_name, None
+                    )
+                    if 'access_token' in secrets:
+                        return secrets['access_token']
+                except Exception as e:
+                    logger.error(f"Error requesting access_token from main thread: {e}")
+                    raise
+            
+            # Otherwise, call _snowflake directly (main thread)
             return StoredConfigurationValue(
                 value=_snowflake.get_oauth_access_token(self.access_token_secret_name)
             )
@@ -1001,10 +1025,34 @@ StoredFieldMappings.model_rebuild()
 OutboundSyncConfigurationParameters.model_rebuild()
 
 @tracer.start_as_current_span("get_secrets")
-def get_secrets(oauth_secret_name: Optional[str], other_secrets_name: Optional[str]
+def get_secrets(oauth_secret_name: Optional[str], other_secrets_name: Optional[str],
+                sync_request: Optional[Any] = None
 ) -> Dict[str, StoredConfigurationValue]:
+    """
+    Get secrets from Snowflake. This function can be called from the main thread or worker threads.
+    When called from worker threads (e.g., within @managed_inbound_processing), it will automatically
+    route the request through the secrets service to avoid threading issues with _snowflake.get_oauth_access_token.
+    
+    :param oauth_secret_name: The name of the OAuth secret to retrieve
+    :param other_secrets_name: The name of other secrets to retrieve  
+    :param sync_request: Optional SyncRequest instance for worker threads. If not provided, will attempt to detect.
+    :return: Dictionary of StoredConfigurationValue objects
+    """
+    from .threading_utils import is_managed_worker_thread
     connection_secrets = {}
     import _snowflake # pylint: disable=import-error, import-outside-toplevel # type: ignore
+    
+    # Check if we're in a worker thread using the explicit flag
+    # This is more reliable than checking thread names
+    if is_managed_worker_thread() and sync_request is not None:
+        logger.debug(f"Worker thread requesting secrets via secrets service")
+        try:
+            return sync_request.request_secrets_from_main_thread(oauth_secret_name, other_secrets_name)
+        except Exception as e:
+            logger.error(f"Error requesting secrets from main thread: {e}")
+            raise
+    
+    # Otherwise, call _snowflake functions directly (main thread)
     if oauth_secret_name is not None:
         connection_secrets["access_token"] = StoredConfigurationValue(
             value=_snowflake.get_oauth_access_token(oauth_secret_name)