polyapi-python 0.3.8.dev7__tar.gz → 0.3.8.dev8__tar.gz

{polyapi_python-0.3.8.dev7 → polyapi_python-0.3.8.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polyapi-python
-Version: 0.3.8.dev7
+Version: 0.3.8.dev8
 Summary: The Python Client for PolyAPI, the IPaaS by Developers for Developers
 Author-email: Dan Fellin <dan@polyapi.io>
 License: MIT License

{polyapi_python-0.3.8.dev7 → polyapi_python-0.3.8.dev8}/polyapi/generate.py

@@ -2,6 +2,8 @@ import json
 import requests
 import os
 import shutil
+import logging
+import tempfile
 from typing import Any, List, Optional, Tuple, cast
 
 from .auth import render_auth_function
@@ -426,48 +428,124 @@ def add_function_file(
     function_name: str,
     spec: SpecificationDto,
 ):
-    # first lets add the import to the __init__
-    init_the_init(full_path)
+    """
+    Atomically add a function file to prevent partial corruption during generation failures.
+    
+    This function generates all content first, then writes files atomically using temporary files
+    to ensure that either the entire operation succeeds or no changes are made to the filesystem.
+    """
+    try:
+        # first lets add the import to the __init__
+        init_the_init(full_path)
 
-    func_str, func_type_defs = render_spec(spec)
+        func_str, func_type_defs = render_spec(spec)
 
-    if func_str:
-        # add function to init
-        init_path = os.path.join(full_path, "__init__.py")
-        with open(init_path, "a") as f:
-            f.write(f"\n\nfrom . import {to_func_namespace(function_name)}\n\n{func_str}")
+        if not func_str:
+            # If render_spec failed and returned empty string, don't create any files
+            raise Exception("Function rendering failed - empty function string returned")
 
-        # add type_defs to underscore file
-        file_path = os.path.join(full_path, f"{to_func_namespace(function_name)}.py")
-        with open(file_path, "w") as f:
-            f.write(func_type_defs)
+        # Prepare all content first before writing any files
+        func_namespace = to_func_namespace(function_name)
+        init_path = os.path.join(full_path, "__init__.py")
+        func_file_path = os.path.join(full_path, f"{func_namespace}.py")
+        
+        # Read current __init__.py content if it exists
+        init_content = ""
+        if os.path.exists(init_path):
+            with open(init_path, "r") as f:
+                init_content = f.read()
+        
+        # Prepare new content to append to __init__.py
+        new_init_content = init_content + f"\n\nfrom . import {func_namespace}\n\n{func_str}"
+        
+        # Use temporary files for atomic writes
+        # Write to __init__.py atomically
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_init:
+            temp_init.write(new_init_content)
+            temp_init_path = temp_init.name
+        
+        # Write to function file atomically  
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_func:
+            temp_func.write(func_type_defs)
+            temp_func_path = temp_func.name
+        
+        # Atomic operations: move temp files to final locations
+        shutil.move(temp_init_path, init_path)
+        shutil.move(temp_func_path, func_file_path)
+        
+    except Exception as e:
+        # Clean up any temporary files that might have been created
+        try:
+            if 'temp_init_path' in locals() and os.path.exists(temp_init_path):
+                os.unlink(temp_init_path)
+            if 'temp_func_path' in locals() and os.path.exists(temp_func_path):
+                os.unlink(temp_func_path)
+        except:
+            pass  # Best effort cleanup
+        
+        # Re-raise the original exception
+        raise e
 
 
 def create_function(
     spec: SpecificationDto
 ) -> None:
+    """
+    Create a function with atomic directory and file operations.
+    
+    Tracks directory creation to enable cleanup on failure.
+    """
     full_path = os.path.dirname(os.path.abspath(__file__))
     folders = f"poly.{spec['context']}.{spec['name']}".split(".")
-    for idx, folder in enumerate(folders):
-        if idx + 1 == len(folders):
-            # special handling for final level
-            add_function_file(
-                full_path,
-                folder,
-                spec,
-            )
-        else:
-            full_path = os.path.join(full_path, folder)
-            if not os.path.exists(full_path):
-                os.makedirs(full_path)
-
-            # append to __init__.py file if nested folders
-            next = folders[idx + 1] if idx + 2 < len(folders) else ""
-            if next:
-                init_the_init(full_path)
-                add_import_to_init(full_path, next)
+    created_dirs = []  # Track directories we create for cleanup on failure
+    
+    try:
+        for idx, folder in enumerate(folders):
+            if idx + 1 == len(folders):
+                # special handling for final level
+                add_function_file(
+                    full_path,
+                    folder,
+                    spec,
+                )
+            else:
+                full_path = os.path.join(full_path, folder)
+                if not os.path.exists(full_path):
+                    os.makedirs(full_path)
+                    created_dirs.append(full_path)  # Track for cleanup
+
+                # append to __init__.py file if nested folders
+                next = folders[idx + 1] if idx + 2 < len(folders) else ""
+                if next:
+                    init_the_init(full_path)
+                    add_import_to_init(full_path, next)
+                    
+    except Exception as e:
+        # Clean up directories we created (in reverse order)
+        for dir_path in reversed(created_dirs):
+            try:
+                if os.path.exists(dir_path) and not os.listdir(dir_path):  # Only remove if empty
+                    os.rmdir(dir_path)
+            except:
+                pass  # Best effort cleanup
+        
+        # Re-raise the original exception
+        raise e
 
 
 def generate_functions(functions: List[SpecificationDto]) -> None:
+    failed_functions = []
     for func in functions:
-        create_function(func)
+        try:
+            create_function(func)
+        except Exception as e:
+            function_path = f"{func.get('context', 'unknown')}.{func.get('name', 'unknown')}"
+            function_id = func.get('id', 'unknown')
+            failed_functions.append(f"{function_path} (id: {function_id})")
+            logging.warning(f"WARNING: Failed to generate function {function_path} (id: {function_id}): {str(e)}")
+            continue
+    
+    if failed_functions:
+        logging.warning(f"WARNING: {len(failed_functions)} function(s) failed to generate:")
+        for failed_func in failed_functions:
+            logging.warning(f"  - {failed_func}")

polyapi_python-0.3.8.dev8/polyapi/poly_schemas.py

@@ -0,0 +1,186 @@
+import os
+import logging
+import tempfile
+import shutil
+from typing import Any, Dict, List, Tuple
+
+from polyapi.schema import wrapped_generate_schema_types
+from polyapi.utils import add_import_to_init, init_the_init, to_func_namespace
+
+from .typedefs import SchemaSpecDto
+
+SCHEMA_CODE_IMPORTS = """from typing_extensions import TypedDict, NotRequired
+
+__all__ = []
+
+
+"""
+
+
+FALLBACK_SPEC_TEMPLATE = """class {name}(TypedDict, total=False):
+    ''' unable to generate schema for {name}, defaulting to permissive type '''
+    pass
+"""
+
+
+def generate_schemas(specs: List[SchemaSpecDto], limit_ids: List[str] = None):
+    failed_schemas = []
+    if limit_ids:
+        for spec in specs:
+            if spec["id"] in limit_ids:
+                try:
+                    create_schema(spec)
+                except Exception as e:
+                    schema_path = f"{spec.get('context', 'unknown')}.{spec.get('name', 'unknown')}"
+                    schema_id = spec.get('id', 'unknown')
+                    failed_schemas.append(f"{schema_path} (id: {schema_id})")
+                    logging.warning(f"WARNING: Failed to generate schema {schema_path} (id: {schema_id}): {str(e)}")
+                    continue
+    else:
+        for spec in specs:
+            try:
+                create_schema(spec)
+            except Exception as e:
+                schema_path = f"{spec.get('context', 'unknown')}.{spec.get('name', 'unknown')}"
+                schema_id = spec.get('id', 'unknown')
+                failed_schemas.append(f"{schema_path} (id: {schema_id})")
+                logging.warning(f"WARNING: Failed to generate schema {schema_path} (id: {schema_id}): {str(e)}")
+                continue
+    
+    if failed_schemas:
+        logging.warning(f"WARNING: {len(failed_schemas)} schema(s) failed to generate:")
+        for failed_schema in failed_schemas:
+            logging.warning(f"  - {failed_schema}")
+
+
+def add_schema_file(
+    full_path: str,
+    schema_name: str,
+    spec: SchemaSpecDto,
+):
+    """
+    Atomically add a schema file to prevent partial corruption during generation failures.
+    
+    This function generates all content first, then writes files atomically using temporary files
+    to ensure that either the entire operation succeeds or no changes are made to the filesystem.
+    """
+    try:
+        # first lets add the import to the __init__
+        init_the_init(full_path, SCHEMA_CODE_IMPORTS)
+
+        if not spec["definition"].get("title"):
+            # very empty schemas like mews.Unit are possible
+            # add a title here to be sure they render
+            spec["definition"]["title"] = schema_name
+
+        schema_defs = render_poly_schema(spec)
+
+        if not schema_defs:
+            # If render_poly_schema failed and returned empty string, don't create any files
+            raise Exception("Schema rendering failed - empty schema content returned")
+
+        # Prepare all content first before writing any files
+        schema_namespace = to_func_namespace(schema_name)
+        init_path = os.path.join(full_path, "__init__.py")
+        schema_file_path = os.path.join(full_path, f"_{schema_namespace}.py")
+        
+        # Read current __init__.py content if it exists
+        init_content = ""
+        if os.path.exists(init_path):
+            with open(init_path, "r") as f:
+                init_content = f.read()
+        
+        # Prepare new content to append to __init__.py
+        new_init_content = init_content + f"\n\nfrom ._{schema_namespace} import {schema_name}\n__all__.append('{schema_name}')\n"
+        
+        # Use temporary files for atomic writes
+        # Write to __init__.py atomically
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_init:
+            temp_init.write(new_init_content)
+            temp_init_path = temp_init.name
+        
+        # Write to schema file atomically  
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_schema:
+            temp_schema.write(schema_defs)
+            temp_schema_path = temp_schema.name
+        
+        # Atomic operations: move temp files to final locations
+        shutil.move(temp_init_path, init_path)
+        shutil.move(temp_schema_path, schema_file_path)
+        
+    except Exception as e:
+        # Clean up any temporary files that might have been created
+        try:
+            if 'temp_init_path' in locals() and os.path.exists(temp_init_path):
+                os.unlink(temp_init_path)
+            if 'temp_schema_path' in locals() and os.path.exists(temp_schema_path):
+                os.unlink(temp_schema_path)
+        except:
+            pass  # Best effort cleanup
+        
+        # Re-raise the original exception
+        raise e
+
+
+def create_schema(
+    spec: SchemaSpecDto
+) -> None:
+    """
+    Create a schema with atomic directory and file operations.
+    
+    Tracks directory creation to enable cleanup on failure.
+    """
+    full_path = os.path.dirname(os.path.abspath(__file__))
+    folders = f"schemas.{spec['context']}.{spec['name']}".split(".")
+    created_dirs = []  # Track directories we create for cleanup on failure
+    
+    try:
+        for idx, folder in enumerate(folders):
+            if idx + 1 == len(folders):
+                # special handling for final level
+                add_schema_file(
+                    full_path,
+                    folder,
+                    spec,
+                )
+            else:
+                full_path = os.path.join(full_path, folder)
+                if not os.path.exists(full_path):
+                    os.makedirs(full_path)
+                    created_dirs.append(full_path)  # Track for cleanup
+
+                # append to __init__.py file if nested folders
+                next = folders[idx + 1] if idx + 2 < len(folders) else ""
+                if next:
+                    init_the_init(full_path, SCHEMA_CODE_IMPORTS)
+                    add_import_to_init(full_path, next)
+                    
+    except Exception as e:
+        # Clean up directories we created (in reverse order)
+        for dir_path in reversed(created_dirs):
+            try:
+                if os.path.exists(dir_path) and not os.listdir(dir_path):  # Only remove if empty
+                    os.rmdir(dir_path)
+            except:
+                pass  # Best effort cleanup
+        
+        # Re-raise the original exception
+        raise e
+
+
+def add_schema_to_init(full_path: str, spec: SchemaSpecDto):
+    init_the_init(full_path, code_imports="")
+    init_path = os.path.join(full_path, "__init__.py")
+    with open(init_path, "a") as f:
+        f.write(render_poly_schema(spec) + "\n\n")
+
+
+def render_poly_schema(spec: SchemaSpecDto) -> str:
+    definition = spec["definition"]
+    if not definition.get("type"):
+        definition["type"] = "object"
+    root, schema_types = wrapped_generate_schema_types(
+        definition, root=spec["name"], fallback_type=Dict
+    )
+    return schema_types
+    # return FALLBACK_SPEC_TEMPLATE.format(name=spec["name"])

{polyapi_python-0.3.8.dev7 → polyapi_python-0.3.8.dev8}/polyapi/variables.py

@@ -1,4 +1,7 @@
 import os
+import logging
+import tempfile
+import shutil
 from typing import List
 
 from polyapi.schema import map_primitive_types
@@ -70,8 +73,21 @@ class {variable_name}:{get_method}
 
 
 def generate_variables(variables: List[VariableSpecDto]):
+    failed_variables = []
     for variable in variables:
-        create_variable(variable)
+        try:
+            create_variable(variable)
+        except Exception as e:
+            variable_path = f"{variable.get('context', 'unknown')}.{variable.get('name', 'unknown')}"
+            variable_id = variable.get('id', 'unknown')
+            failed_variables.append(f"{variable_path} (id: {variable_id})")
+            logging.warning(f"WARNING: Failed to generate variable {variable_path} (id: {variable_id}): {str(e)}")
+            continue
+    
+    if failed_variables:
+        logging.warning(f"WARNING: {len(failed_variables)} variable(s) failed to generate:")
+        for failed_var in failed_variables:
+            logging.warning(f"  - {failed_var}")
 
 
 def render_variable(variable: VariableSpecDto):
@@ -116,26 +132,84 @@ def _get_variable_type(type_spec: PropertyType) -> str:
 
 
 def create_variable(variable: VariableSpecDto) -> None:
+    """
+    Create a variable with atomic directory and file operations.
+    
+    Tracks directory creation to enable cleanup on failure.
+    """
     folders = ["vari"]
     if variable["context"]:
         folders += variable["context"].split(".")
 
     # build up the full_path by adding all the folders
     full_path = os.path.join(os.path.dirname(os.path.abspath(__file__)))
-
-    for idx, folder in enumerate(folders):
-        full_path = os.path.join(full_path, folder)
-        if not os.path.exists(full_path):
-            os.makedirs(full_path)
-        next = folders[idx + 1] if idx + 1 < len(folders) else None
-        if next:
-            add_import_to_init(full_path, next)
-
-    add_variable_to_init(full_path, variable)
+    created_dirs = []  # Track directories we create for cleanup on failure
+
+    try:
+        for idx, folder in enumerate(folders):
+            full_path = os.path.join(full_path, folder)
+            if not os.path.exists(full_path):
+                os.makedirs(full_path)
+                created_dirs.append(full_path)  # Track for cleanup
+            next = folders[idx + 1] if idx + 1 < len(folders) else None
+            if next:
+                add_import_to_init(full_path, next)
+
+        add_variable_to_init(full_path, variable)
+        
+    except Exception as e:
+        # Clean up directories we created (in reverse order)
+        for dir_path in reversed(created_dirs):
+            try:
+                if os.path.exists(dir_path) and not os.listdir(dir_path):  # Only remove if empty
+                    os.rmdir(dir_path)
+            except:
+                pass  # Best effort cleanup
+        
+        # Re-raise the original exception
+        raise e
 
 
 def add_variable_to_init(full_path: str, variable: VariableSpecDto):
-    init_the_init(full_path)
-    init_path = os.path.join(full_path, "__init__.py")
-    with open(init_path, "a") as f:
-        f.write(render_variable(variable) + "\n\n")
+    """
+    Atomically add a variable to __init__.py to prevent partial corruption during generation failures.
+    
+    This function generates all content first, then writes the file atomically using temporary files
+    to ensure that either the entire operation succeeds or no changes are made to the filesystem.
+    """
+    try:
+        init_the_init(full_path)
+        init_path = os.path.join(full_path, "__init__.py")
+        
+        # Generate variable content first
+        variable_content = render_variable(variable)
+        if not variable_content:
+            raise Exception("Variable rendering failed - empty content returned")
+        
+        # Read current __init__.py content if it exists
+        init_content = ""
+        if os.path.exists(init_path):
+            with open(init_path, "r") as f:
+                init_content = f.read()
+        
+        # Prepare new content to append
+        new_init_content = init_content + variable_content + "\n\n"
+        
+        # Write to temporary file first, then atomic move
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=full_path, suffix=".tmp") as temp_file:
+            temp_file.write(new_init_content)
+            temp_file_path = temp_file.name
+        
+        # Atomic operation: move temp file to final location
+        shutil.move(temp_file_path, init_path)
+        
+    except Exception as e:
+        # Clean up temporary file if it exists
+        try:
+            if 'temp_file_path' in locals() and os.path.exists(temp_file_path):
+                os.unlink(temp_file_path)
+        except:
+            pass  # Best effort cleanup
+        
+        # Re-raise the original exception
+        raise e

{polyapi_python-0.3.8.dev7 → polyapi_python-0.3.8.dev8}/polyapi/webhook.py

@@ -2,6 +2,7 @@ import asyncio
 import socketio  # type: ignore
 from socketio.exceptions import ConnectionError  # type: ignore
 import uuid
+import logging
 from typing import Any, Dict, List, Tuple
 
 from polyapi.config import get_api_key_and_url
@@ -121,22 +122,27 @@ def render_webhook_handle(
     arguments: List[PropertySpecification],
     return_type: Dict[str, Any],
 ) -> Tuple[str, str]:
-    function_args, function_args_def = parse_arguments(function_name, arguments)
-
-    if "WebhookEventType" in function_args:
-        # let's add the function name import!
-        function_args = function_args.replace("WebhookEventType", f"{to_func_namespace(function_name)}.WebhookEventType")
-
-    func_str = WEBHOOK_TEMPLATE.format(
-        description=function_description,
-        client_id=uuid.uuid4().hex,
-        function_id=function_id,
-        function_name=function_name,
-        function_args=function_args,
-        function_path=poly_full_path(function_context, function_name),
-    )
-    func_defs = WEBHOOK_DEFS_TEMPLATE.format(function_args_def=function_args_def)
-    return func_str, func_defs
+    try:
+        function_args, function_args_def = parse_arguments(function_name, arguments)
+
+        if "WebhookEventType" in function_args:
+            # let's add the function name import!
+            function_args = function_args.replace("WebhookEventType", f"{to_func_namespace(function_name)}.WebhookEventType")
+
+        func_str = WEBHOOK_TEMPLATE.format(
+            description=function_description,
+            client_id=uuid.uuid4().hex,
+            function_id=function_id,
+            function_name=function_name,
+            function_args=function_args,
+            function_path=poly_full_path(function_context, function_name),
+        )
+        func_defs = WEBHOOK_DEFS_TEMPLATE.format(function_args_def=function_args_def)
+        return func_str, func_defs
+    except Exception as e:
+        logging.warning(f"Failed to render webhook handle {function_context}.{function_name} (id: {function_id}): {str(e)}")
+        # Return empty strings to indicate generation failure - this will be caught by generate_functions error handling
+        return "", ""
 
 
 def start(*args):

{polyapi_python-0.3.8.dev7 → polyapi_python-0.3.8.dev8}/polyapi_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polyapi-python
-Version: 0.3.8.dev7
+Version: 0.3.8.dev8
 Summary: The Python Client for PolyAPI, the IPaaS by Developers for Developers
 Author-email: Dan Fellin <dan@polyapi.io>
 License: MIT License

{polyapi_python-0.3.8.dev7 → polyapi_python-0.3.8.dev8}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2", "wheel"]
 
 [project]
 name = "polyapi-python"
-version = "0.3.8.dev7"
+version = "0.3.8.dev8"
 description = "The Python Client for PolyAPI, the IPaaS by Developers for Developers"
 authors = [{ name = "Dan Fellin", email = "dan@polyapi.io" }]
 dependencies = [