analytic-workspace-client 1.29.1__tar.gz → 1.29.3__tar.gz

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/PKG-INFO

@@ -1,10 +1,12 @@
 Metadata-Version: 2.1
 Name: analytic_workspace_client
-Version: 1.29.1
+Version: 1.29.3
 Summary: Библиотека для подключения к Analytic Workspace
 Home-page: https://analyticworkspace.ru/
 Author: Analytic Workspace
 Author-email: aw_help@analyticworkspace.ru
+License: UNKNOWN
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -39,3 +41,4 @@ df = session.load(model_id=123)  # df: pandas.DataFrame
 
 display(df)
 ```
+

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/setup.py

@@ -10,7 +10,7 @@ long_description = (here / "README.md").read_text(encoding="utf-8")
 
 setup(
     name='analytic_workspace_client',
-    version='1.29.1',
+    version='1.29.3',
 
     description='Библиотека для подключения к Analytic Workspace',
     long_description=long_description,
@@ -36,7 +36,7 @@ setup(
         'python-dotenv>=1.0,<1.1',
         'httpx>=0.25,<1.0',
         'pandas',
-        'pydantic>=1.10,<2.0',
+        'pydantic>=2.0',
         'colorama>=0.4,<0.5'
     ],
 

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/analytic_workspace_client.egg-info/PKG-INFO

@@ -1,10 +1,12 @@
 Metadata-Version: 2.1
 Name: analytic-workspace-client
-Version: 1.29.1
+Version: 1.29.3
 Summary: Библиотека для подключения к Analytic Workspace
 Home-page: https://analyticworkspace.ru/
 Author: Analytic Workspace
 Author-email: aw_help@analyticworkspace.ru
+License: UNKNOWN
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -39,3 +41,4 @@ df = session.load(model_id=123)  # df: pandas.DataFrame
 
 display(df)
 ```
+

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/analytic_workspace_client.egg-info/requires.txt

@@ -1,8 +1,8 @@
-python-dotenv<1.1,>=1.0
+colorama<0.5,>=0.4
 httpx<1.0,>=0.25
 pandas
-pydantic<2.0,>=1.10
-colorama<0.5,>=0.4
+pydantic>=2.0
+python-dotenv<1.1,>=1.0
 
 [dev]
 pyspark==3.5.0

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/aw_client/data_master/base.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, Iterator
 
 import abc
 from pathlib import Path
@@ -38,7 +38,7 @@ class DataMasterApi(metaclass=abc.ABCMeta):
         """ """
 
     @contextmanager
-    def get_http_client(self) -> httpx.Client:
+    def get_http_client(self) -> Iterator[httpx.Client]:
         """ """
         if not self.api_config.aw_url:
             raise DataMasterApi.MisconfiguredError('Не указан URL к Analytic Workspace')

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/aw_client/data_master/v2.py

@@ -34,7 +34,7 @@ class DataMasterV2(DataMasterApi):
                     raise AwModelForbidden(f'Доступ к модели {model_id} запрещен')
                 raise Exception(f'Ошибка запроса схемы модели HTTP {r.status_code} GET {r.url}: {r.text}')
 
-            return ModelSchema.parse_obj(r.json())
+            return ModelSchema.model_validate(r.json())
 
     def load_model_object_data(self, folder: Path, model_id: int, model_object_name: str, limit: Optional[int]):
         """ """

analytic_workspace_client-1.29.3/src/aw_client/etl_blocks/__init__.py

@@ -0,0 +1,9 @@
+from .runtime import (
+    get_etl_block_schema, 
+    get_etl_block_data, 
+    get_etl_block_meta, 
+    run_block_action
+)
+from .test_data import ModelObjectTestData
+
+

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/aw_client/etl_blocks/dto.py

@@ -3,7 +3,7 @@ from typing import Optional, Any, List, Union
 import datetime
 from enum import Enum
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class ETLBlockParamType(str, Enum):

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/aw_client/etl_blocks/runtime.py

@@ -9,6 +9,7 @@ from aw_client.core.model_vault import Vault
 from aw_client.core.spark import build_spark_session
 from aw_client.core.bundle import NamedObjectsBundle
 from aw_client.models.model_schema import ModelObject, ModelObjectField
+from aw_etl.etl_blocks import ETLBlockActionResult
 from .application import ETLBlockApplication
 from .test_data import ModelObjectTestData
 from .tools import build_dataframe, build_model_object, build_spark_schema
@@ -32,7 +33,7 @@ def get_etl_block_meta(block_path: Path) -> ETLBlockMeta:
         raise Exception(f'Файл с метаданным блока не найден: {block_meta_path}')
 
     with open(block_meta_path, 'rt') as f:
-        return ETLBlockMeta.parse_raw(f.read())
+        return ETLBlockMeta.model_validate_json(f.read())
 
 
 def get_etl_block_schema(block_path: Path, 
@@ -98,6 +99,8 @@ def get_etl_block_schema(block_path: Path,
         model_module=model_module
     )
 
+    child_objects_data = (test_data if isinstance(test_data, list) else [test_data]) if test_data is not None else None
+
     # Определение параметров для передачи
     block_schema_kwargs = {}
     if 'params' in block_schema_parameters:
@@ -105,7 +108,14 @@ def get_etl_block_schema(block_path: Path,
     if 'app' in block_schema_parameters:
         block_schema_kwargs['app'] = app
     if 'model_object' in block_schema_parameters:
-        block_schema_kwargs['model_object'] = build_model_object(test_data[0] if isinstance(test_data, list) else test_data)
+        model_object = ModelObject(
+            name='obj',
+            model_name='obj',
+            type='etl_block',
+            fields=[],
+            childs=[build_model_object(o) for o in child_objects_data]
+        )
+        block_schema_kwargs['model_object'] = model_object
     if 'schema' in block_schema_parameters:
         block_schema_kwargs['schema'] = upstream_schemas.first()
     if 'schemas' in block_schema_parameters:
@@ -181,22 +191,146 @@ def get_etl_block_data(block_path: Path,
         model_module=model_module
     )
 
+    child_objects_data = (test_data if isinstance(test_data, list) else [test_data]) if test_data is not None else None
+
     # Определение параметров для передачи
-    block_schema_kwargs = {}
+    block_data_kwargs = {}
     if 'params' in block_data_parameters:
-        block_schema_kwargs['params'] = params
+        block_data_kwargs['params'] = params
     if 'app' in block_data_parameters:
-        block_schema_kwargs['app'] = app
+        block_data_kwargs['app'] = app
     if 'model_object' in block_data_parameters:
-        block_schema_kwargs['model_object'] = build_model_object(test_data[0] if isinstance(test_data, list) else test_data)
+        model_object = ModelObject(
+            name='obj',
+            model_name='obj',
+            type='etl_block',
+            fields=[],
+            childs=[build_model_object(o) for o in child_objects_data]
+        )
+        block_data_kwargs['model_object'] = model_object
     if 'df' in block_data_parameters:
-        block_schema_kwargs['df'] = upstream_dataframes.first()
+        block_data_kwargs['df'] = upstream_dataframes.first()
     if 'dfs' in block_data_parameters:
-        block_schema_kwargs['dfs'] = upstream_dataframes
+        block_data_kwargs['dfs'] = upstream_dataframes
     if 'upstream_dataframe' in block_data_parameters:
-        block_schema_kwargs['upstream_dataframe'] = upstream_dataframes.first()
+        block_data_kwargs['upstream_dataframe'] = upstream_dataframes.first()
     if 'upstream_dataframes' in block_data_parameters:
-        block_schema_kwargs['upstream_dataframes'] = upstream_dataframes
+        block_data_kwargs['upstream_dataframes'] = upstream_dataframes
+    
+    return block_module['block_data'](**block_data_kwargs)
     
-    return block_module['block_data'](**block_schema_kwargs)
+
+def run_block_action(
+        block_path: Path,
+        action_name: str,
+        test_data: Union[ModelObjectTestData, List[ModelObjectTestData]],
+        params: Optional[Dict] = None, 
+        run_mode: Optional[Literal['']] = None, 
+        vault: Optional[Vault] = None,
+        model_script_code: Optional[str] = None
+) -> ETLBlockActionResult:
+    """ 
+    Выполняет действие etl-блока 
+    
+    Args:
+    -------------------------------------------------------
+    block_path : Path
+        Путь к исходным текстам блока
+    """
+    block_code_path = block_path / 'block_code.py' if block_path.is_dir() else block_path
+    
+    if not block_code_path.exists():
+        raise Exception(f'Файл с исходным кодом блока не найден: {block_code_path}')
     
+    with open(block_code_path, 'rt') as f:
+        block_code = f.read()
+
+    # Компиляция кода блока
+    try:
+        block_module = ScriptCompiler().compile(source_code=block_code, mode=ScriptCompiler.MODE_ETL_BLOCK)
+    except ScriptCompiler.CannotCompile as e:
+        raise Exception(f'Ошибка компиляции исходного кода блока: {e}')
+    
+    # Проверяем, что функция для запуска действия объявлена в модуле блока
+    block_action_function_name = f'block_action_{action_name}'
+    if not block_action_function_name in block_module:
+        raise Exception(f'Функция для обработки действия {block_action_function_name} '
+                        'не найдена в исходном коде блока')
+    
+    # Компиляция кода модели
+    if model_script_code:
+        try:
+            model_module = ScriptCompiler().compile(source_code=model_script_code, mode=ScriptCompiler.MODE_ETL)
+        except ScriptCompiler.CannotCompile as e:
+            raise Exception(f'Ошибка компиляции исходного кода скрипта модели: {e}')
+    else:
+        model_module = None
+
+    spark = build_spark_session()
+
+    # Дочерние схемы
+    schemas = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        schema = build_spark_schema(td)
+        if not schemas:
+            schemas['child'] = schema
+        schemas[td.model_name] = schema
+
+    upstream_schemas = NamedObjectsBundle(schemas)
+
+    # Дочерние датафреймы
+    dataframes = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        df = build_dataframe(spark, td)
+        if not dataframes:
+            dataframes['child'] = df
+        dataframes[td.model_name] = df
+    
+    upstream_dataframes = NamedObjectsBundle(dataframes)
+
+    block_action_parameters = inspect.signature(block_module[block_action_function_name]).parameters
+
+    app = ETLBlockApplication(
+        spark_builder=build_spark_session,
+        run_mode=run_mode or 'full',
+        vault=vault or Vault(),
+        model_module=model_module
+    )
+
+    child_objects_data = (test_data if isinstance(test_data, list) else [test_data]) if test_data is not None else None
+
+    block_action_kwargs = {}
+    if 'params' in block_action_parameters:
+        block_action_kwargs['params'] = params
+    if 'app' in block_action_parameters:
+        block_action_kwargs['app'] = app
+    if 'model_object' in block_action_parameters:
+        model_object = ModelObject(
+            name='obj',
+            model_name='obj',
+            type='etl_block',
+            fields=[],
+            childs=[build_model_object(o) for o in child_objects_data]
+        )
+        block_action_kwargs['model_object'] = model_object
+    if 'schema' in block_action_parameters:
+        block_action_kwargs['schema'] = upstream_schemas.first()
+    if 'schemas' in block_action_parameters:
+        block_action_kwargs['schemas'] = upstream_schemas
+    if 'upstream_schema' in block_action_parameters:
+        block_action_kwargs['upstream_schema'] = upstream_schemas.first()
+    if 'upstream_schemas' in block_action_parameters:
+        block_action_kwargs['upstream_schemas'] = upstream_schemas
+    if 'df' in block_action_parameters:
+        block_action_kwargs['df'] = upstream_dataframes.first()
+    if 'dfs' in block_action_parameters:
+        block_action_kwargs['dfs'] = upstream_dataframes
+    if 'upstream_dataframe' in block_action_parameters:
+        block_action_kwargs['upstream_dataframe'] = upstream_dataframes.first()
+    if 'upstream_dataframes' in block_action_parameters:
+        block_action_kwargs['upstream_dataframes'] = upstream_dataframes
+    
+    return block_module[block_action_function_name](**block_action_kwargs)
+
+
+

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/aw_client/model_dev/runner.py

@@ -150,7 +150,8 @@ if __name__ == '__main__':
         if cache.model_schema_path().exists() and not args.no_cache:
             print(Fore.YELLOW + f'Не удалось получить схему модели {model_id}: {e}. '
                                 f'Для запуска скрипта используется схема модели из кеша.' + Fore.RESET)
-            model_schema = ModelSchema.parse_file(cache.model_schema_path())
+            with open(cache.model_schema_path(), 'rt') as f:
+                model_schema = ModelSchema.model_validate_json(f.read())
         else:
             print(Fore.RED + f'ОШИБКА: Не удалось получить схему модели {model_id}: {e}' + Fore.RESET)
             sys.exit(1)
@@ -162,7 +163,7 @@ if __name__ == '__main__':
     cache_folder = cache.get_model_folder()
     os.makedirs(cache_folder, exist_ok=True)
     with open(cache.model_schema_path(), 'wt') as f:
-        f.write(model_schema.json(ensure_ascii=False, indent=2))
+        f.write(model_schema.model_dump_json(indent=2))
 
     # ------------------------------------------------------------------------------
     # Загрузка данных объектов модели

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/aw_client/models/model_schema.py

@@ -1,6 +1,6 @@
 from typing import Optional, List, Dict
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class ModelObjectField(BaseModel):
@@ -9,6 +9,10 @@ class ModelObjectField(BaseModel):
     model_name: str
     simple_type: str
 
+    model_config = ConfigDict(
+        protected_namespaces=()
+    )
+
 
 class ModelObject(BaseModel):
     """ """
@@ -18,6 +22,12 @@ class ModelObject(BaseModel):
     sql_text: Optional[str] = None
     fields: List[ModelObjectField] = []
 
+    childs: List['ModelObject'] = []
+
+    model_config = ConfigDict(
+        protected_namespaces=()
+    )
+
 
 class ModelSql(BaseModel):
     """

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/aw_etl/etl_blocks.py

@@ -1,4 +1,6 @@
-from typing import Protocol, Optional
+from typing import Protocol, Optional, Dict, Any
+
+from pydantic import BaseModel
 
 try:
     from pyspark.sql import SparkSession
@@ -36,4 +38,8 @@ class ETLBlockApplication(Protocol):
     def run_mode(self) -> str:
         """ """
 
-    
+class ETLBlockActionResult(BaseModel):
+    """ 
+    Результат выполнения экшена блока
+    """
+    params_patch: Dict[str, Any]

{analytic_workspace_client-1.29.1 → analytic_workspace_client-1.29.3}/src/aw_etl/models.py

@@ -14,6 +14,10 @@ class ModelObject(Protocol):
     def fields(self) -> List[ModelObjectField]:
         """ """
 
+    @property
+    def childs(self) -> List['ModelObject']:
+        """ """
+
 
 class Vault(Protocol):
     """ """

analytic_workspace_client-1.29.1/src/aw_client/etl_blocks/__init__.py

@@ -1,4 +0,0 @@
-from .runtime import get_etl_block_schema, get_etl_block_data, get_etl_block_meta
-from .test_data import ModelObjectTestData
-
-