analytic-workspace-client 1.29.2__tar.gz → 1.31.0__tar.gz

{analytic_workspace_client-1.29.2 → analytic_workspace_client-1.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: analytic_workspace_client
-Version: 1.29.2
+Version: 1.31.0
 Summary: Библиотека для подключения к Analytic Workspace
 Home-page: https://analyticworkspace.ru/
 Author: Analytic Workspace
@@ -10,7 +10,7 @@ Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8,<4
+Requires-Python: >=3.10,<4
 Description-Content-Type: text/markdown
 Provides-Extra: dev
 Provides-Extra: ml

{analytic_workspace_client-1.29.2 → analytic_workspace_client-1.31.0}/setup.py

@@ -10,7 +10,7 @@ long_description = (here / "README.md").read_text(encoding="utf-8")
 
 setup(
     name='analytic_workspace_client',
-    version='1.29.2',
+    version='1.31.0',
 
     description='Библиотека для подключения к Analytic Workspace',
     long_description=long_description,
@@ -27,7 +27,7 @@ setup(
         'Operating System :: OS Independent'
     ],
 
-    python_requires='>=3.8,<4',
+    python_requires='>=3.10,<4',
 
     package_dir={'': 'src'},
     packages=find_packages(where='src'),
@@ -41,7 +41,7 @@ setup(
     ],
 
     extras_require={
-        'dev': ['pyspark==3.5.0', 'pytest>=8.2,<8.3'],
+        'dev': ['pyspark==3.5.1', 'pytest>=8.2,<8.3'],
         'ml': ['mlflow==2.14.3']
     },
     

{analytic_workspace_client-1.29.2 → analytic_workspace_client-1.31.0}/src/analytic_workspace_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: analytic-workspace-client
-Version: 1.29.2
+Version: 1.31.0
 Summary: Библиотека для подключения к Analytic Workspace
 Home-page: https://analyticworkspace.ru/
 Author: Analytic Workspace
@@ -10,7 +10,7 @@ Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8,<4
+Requires-Python: >=3.10,<4
 Description-Content-Type: text/markdown
 Provides-Extra: dev
 Provides-Extra: ml

{analytic_workspace_client-1.29.2 → analytic_workspace_client-1.31.0}/src/analytic_workspace_client.egg-info/requires.txt

@@ -5,7 +5,7 @@ pydantic>=2.0
 python-dotenv<1.1,>=1.0
 
 [dev]
-pyspark==3.5.0
+pyspark==3.5.1
 pytest<8.3,>=8.2
 
 [ml]

analytic_workspace_client-1.31.0/src/aw_client/etl_blocks/__init__.py

@@ -0,0 +1,9 @@
+from .runtime import (
+    get_etl_block_schema, 
+    get_etl_block_data, 
+    get_etl_block_meta, 
+    run_block_action
+)
+from .test_data import ModelObjectTestData
+
+

{analytic_workspace_client-1.29.2 → analytic_workspace_client-1.31.0}/src/aw_client/etl_blocks/runtime.py

@@ -9,6 +9,7 @@ from aw_client.core.model_vault import Vault
 from aw_client.core.spark import build_spark_session
 from aw_client.core.bundle import NamedObjectsBundle
 from aw_client.models.model_schema import ModelObject, ModelObjectField
+from aw_etl.etl_blocks import ETLBlockActionResult
 from .application import ETLBlockApplication
 from .test_data import ModelObjectTestData
 from .tools import build_dataframe, build_model_object, build_spark_schema
@@ -98,6 +99,8 @@ def get_etl_block_schema(block_path: Path,
         model_module=model_module
     )
 
+    child_objects_data = (test_data if isinstance(test_data, list) else [test_data]) if test_data is not None else None
+
     # Определение параметров для передачи
     block_schema_kwargs = {}
     if 'params' in block_schema_parameters:
@@ -105,7 +108,14 @@ def get_etl_block_schema(block_path: Path,
     if 'app' in block_schema_parameters:
         block_schema_kwargs['app'] = app
     if 'model_object' in block_schema_parameters:
-        block_schema_kwargs['model_object'] = build_model_object(test_data[0] if isinstance(test_data, list) else test_data)
+        model_object = ModelObject(
+            name='obj',
+            model_name='obj',
+            type='etl_block',
+            fields=[],
+            childs=[build_model_object(o) for o in child_objects_data]
+        )
+        block_schema_kwargs['model_object'] = model_object
     if 'schema' in block_schema_parameters:
         block_schema_kwargs['schema'] = upstream_schemas.first()
     if 'schemas' in block_schema_parameters:
@@ -181,22 +191,146 @@ def get_etl_block_data(block_path: Path,
         model_module=model_module
     )
 
+    child_objects_data = (test_data if isinstance(test_data, list) else [test_data]) if test_data is not None else None
+
     # Определение параметров для передачи
-    block_schema_kwargs = {}
+    block_data_kwargs = {}
     if 'params' in block_data_parameters:
-        block_schema_kwargs['params'] = params
+        block_data_kwargs['params'] = params
     if 'app' in block_data_parameters:
-        block_schema_kwargs['app'] = app
+        block_data_kwargs['app'] = app
     if 'model_object' in block_data_parameters:
-        block_schema_kwargs['model_object'] = build_model_object(test_data[0] if isinstance(test_data, list) else test_data)
+        model_object = ModelObject(
+            name='obj',
+            model_name='obj',
+            type='etl_block',
+            fields=[],
+            childs=[build_model_object(o) for o in child_objects_data]
+        )
+        block_data_kwargs['model_object'] = model_object
     if 'df' in block_data_parameters:
-        block_schema_kwargs['df'] = upstream_dataframes.first()
+        block_data_kwargs['df'] = upstream_dataframes.first()
     if 'dfs' in block_data_parameters:
-        block_schema_kwargs['dfs'] = upstream_dataframes
+        block_data_kwargs['dfs'] = upstream_dataframes
     if 'upstream_dataframe' in block_data_parameters:
-        block_schema_kwargs['upstream_dataframe'] = upstream_dataframes.first()
+        block_data_kwargs['upstream_dataframe'] = upstream_dataframes.first()
     if 'upstream_dataframes' in block_data_parameters:
-        block_schema_kwargs['upstream_dataframes'] = upstream_dataframes
+        block_data_kwargs['upstream_dataframes'] = upstream_dataframes
+    
+    return block_module['block_data'](**block_data_kwargs)
     
-    return block_module['block_data'](**block_schema_kwargs)
+
+def run_block_action(
+        block_path: Path,
+        action_name: str,
+        test_data: Union[ModelObjectTestData, List[ModelObjectTestData]],
+        params: Optional[Dict] = None, 
+        run_mode: Optional[Literal['']] = None, 
+        vault: Optional[Vault] = None,
+        model_script_code: Optional[str] = None
+) -> ETLBlockActionResult:
+    """ 
+    Выполняет действие etl-блока 
+    
+    Args:
+    -------------------------------------------------------
+    block_path : Path
+        Путь к исходным текстам блока
+    """
+    block_code_path = block_path / 'block_code.py' if block_path.is_dir() else block_path
+    
+    if not block_code_path.exists():
+        raise Exception(f'Файл с исходным кодом блока не найден: {block_code_path}')
     
+    with open(block_code_path, 'rt') as f:
+        block_code = f.read()
+
+    # Компиляция кода блока
+    try:
+        block_module = ScriptCompiler().compile(source_code=block_code, mode=ScriptCompiler.MODE_ETL_BLOCK)
+    except ScriptCompiler.CannotCompile as e:
+        raise Exception(f'Ошибка компиляции исходного кода блока: {e}')
+    
+    # Проверяем, что функция для запуска действия объявлена в модуле блока
+    block_action_function_name = f'block_action_{action_name}'
+    if not block_action_function_name in block_module:
+        raise Exception(f'Функция для обработки действия {block_action_function_name} '
+                        'не найдена в исходном коде блока')
+    
+    # Компиляция кода модели
+    if model_script_code:
+        try:
+            model_module = ScriptCompiler().compile(source_code=model_script_code, mode=ScriptCompiler.MODE_ETL)
+        except ScriptCompiler.CannotCompile as e:
+            raise Exception(f'Ошибка компиляции исходного кода скрипта модели: {e}')
+    else:
+        model_module = None
+
+    spark = build_spark_session()
+
+    # Дочерние схемы
+    schemas = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        schema = build_spark_schema(td)
+        if not schemas:
+            schemas['child'] = schema
+        schemas[td.model_name] = schema
+
+    upstream_schemas = NamedObjectsBundle(schemas)
+
+    # Дочерние датафреймы
+    dataframes = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        df = build_dataframe(spark, td)
+        if not dataframes:
+            dataframes['child'] = df
+        dataframes[td.model_name] = df
+    
+    upstream_dataframes = NamedObjectsBundle(dataframes)
+
+    block_action_parameters = inspect.signature(block_module[block_action_function_name]).parameters
+
+    app = ETLBlockApplication(
+        spark_builder=build_spark_session,
+        run_mode=run_mode or 'full',
+        vault=vault or Vault(),
+        model_module=model_module
+    )
+
+    child_objects_data = (test_data if isinstance(test_data, list) else [test_data]) if test_data is not None else None
+
+    block_action_kwargs = {}
+    if 'params' in block_action_parameters:
+        block_action_kwargs['params'] = params
+    if 'app' in block_action_parameters:
+        block_action_kwargs['app'] = app
+    if 'model_object' in block_action_parameters:
+        model_object = ModelObject(
+            name='obj',
+            model_name='obj',
+            type='etl_block',
+            fields=[],
+            childs=[build_model_object(o) for o in child_objects_data]
+        )
+        block_action_kwargs['model_object'] = model_object
+    if 'schema' in block_action_parameters:
+        block_action_kwargs['schema'] = upstream_schemas.first()
+    if 'schemas' in block_action_parameters:
+        block_action_kwargs['schemas'] = upstream_schemas
+    if 'upstream_schema' in block_action_parameters:
+        block_action_kwargs['upstream_schema'] = upstream_schemas.first()
+    if 'upstream_schemas' in block_action_parameters:
+        block_action_kwargs['upstream_schemas'] = upstream_schemas
+    if 'df' in block_action_parameters:
+        block_action_kwargs['df'] = upstream_dataframes.first()
+    if 'dfs' in block_action_parameters:
+        block_action_kwargs['dfs'] = upstream_dataframes
+    if 'upstream_dataframe' in block_action_parameters:
+        block_action_kwargs['upstream_dataframe'] = upstream_dataframes.first()
+    if 'upstream_dataframes' in block_action_parameters:
+        block_action_kwargs['upstream_dataframes'] = upstream_dataframes
+    
+    return block_module[block_action_function_name](**block_action_kwargs)
+
+
+

{analytic_workspace_client-1.29.2 → analytic_workspace_client-1.31.0}/src/aw_client/models/model_schema.py

@@ -22,6 +22,8 @@ class ModelObject(BaseModel):
     sql_text: Optional[str] = None
     fields: List[ModelObjectField] = []
 
+    childs: List['ModelObject'] = []
+
     model_config = ConfigDict(
         protected_namespaces=()
     )

{analytic_workspace_client-1.29.2 → analytic_workspace_client-1.31.0}/src/aw_etl/etl_blocks.py

@@ -1,4 +1,6 @@
-from typing import Protocol, Optional
+from typing import Protocol, Optional, Dict, Any
+
+from pydantic import BaseModel
 
 try:
     from pyspark.sql import SparkSession
@@ -36,4 +38,8 @@ class ETLBlockApplication(Protocol):
     def run_mode(self) -> str:
         """ """
 
-    
+class ETLBlockActionResult(BaseModel):
+    """ 
+    Результат выполнения экшена блока
+    """
+    params_patch: Dict[str, Any]

{analytic_workspace_client-1.29.2 → analytic_workspace_client-1.31.0}/src/aw_etl/models.py

@@ -14,6 +14,10 @@ class ModelObject(Protocol):
     def fields(self) -> List[ModelObjectField]:
         """ """
 
+    @property
+    def childs(self) -> List['ModelObject']:
+        """ """
+
 
 class Vault(Protocol):
     """ """

analytic_workspace_client-1.29.2/src/aw_client/etl_blocks/__init__.py

@@ -1,4 +0,0 @@
-from .runtime import get_etl_block_schema, get_etl_block_data, get_etl_block_meta
-from .test_data import ModelObjectTestData
-
-