analytic-workspace-client 1.1.1__tar.gz → 1.29.0__tar.gz

{analytic_workspace_client-1.1.1 → analytic_workspace_client-1.29.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: analytic_workspace_client
-Version: 1.1.1
+Version: 1.29.0
 Summary: Библиотека для подключения к Analytic Workspace
 Home-page: https://analyticworkspace.ru/
 Author: Analytic Workspace
@@ -11,6 +11,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8,<4
 Description-Content-Type: text/markdown
 Provides-Extra: dev
+Provides-Extra: ml
 
 # Библиотека для Analytic Workspace
 
@@ -34,7 +35,7 @@ session = Session(token=aw_token, aw_url='https://aw.example.ru')
 # Если токен доступа указан в переменной окружения AW_DATA_TOKEN, то объект сессии можно создавать 
 # без явного указания параметра token: session = Session(aw_url='https://aw.example.ru')
 
-df = session.load()  # df: pandas.DataFrame
+df = session.load(model_id=123)  # df: pandas.DataFrame
 
 display(df)
 ```

{analytic_workspace_client-1.1.1 → analytic_workspace_client-1.29.0}/README.md

@@ -20,7 +20,7 @@ session = Session(token=aw_token, aw_url='https://aw.example.ru')
 # Если токен доступа указан в переменной окружения AW_DATA_TOKEN, то объект сессии можно создавать 
 # без явного указания параметра token: session = Session(aw_url='https://aw.example.ru')
 
-df = session.load()  # df: pandas.DataFrame
+df = session.load(model_id=123)  # df: pandas.DataFrame
 
 display(df)
 ```

{analytic_workspace_client-1.1.1 → analytic_workspace_client-1.29.0}/setup.py

@@ -10,7 +10,7 @@ long_description = (here / "README.md").read_text(encoding="utf-8")
 
 setup(
     name='analytic_workspace_client',
-    version='1.1.1',
+    version='1.29.0',
 
     description='Библиотека для подключения к Analytic Workspace',
     long_description=long_description,
@@ -33,15 +33,16 @@ setup(
     packages=find_packages(where='src'),
 
     install_requires=[
-        'python-dotenv>=0.21,<1.0',
-        'httpx>=0.23,<1.0',
-        'pandas>=1.5,<2.0',
+        'python-dotenv>=1.0,<1.1',
+        'httpx>=0.25,<1.0',
+        'pandas',
         'pydantic>=1.10,<2.0',
         'colorama>=0.4,<0.5'
     ],
 
     extras_require={
-        'dev': 'pyspark==3.3.1',
+        'dev': ['pyspark==3.5.0', 'pytest>=8.2,<8.3'],
+        'ml': ['mlflow>=2.14,<2.15']
     },
     
     setup_requires=['wheel'],  

{analytic_workspace_client-1.1.1 → analytic_workspace_client-1.29.0}/src/analytic_workspace_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: analytic-workspace-client
-Version: 1.1.1
+Version: 1.29.0
 Summary: Библиотека для подключения к Analytic Workspace
 Home-page: https://analyticworkspace.ru/
 Author: Analytic Workspace
@@ -11,6 +11,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8,<4
 Description-Content-Type: text/markdown
 Provides-Extra: dev
+Provides-Extra: ml
 
 # Библиотека для Analytic Workspace
 
@@ -34,7 +35,7 @@ session = Session(token=aw_token, aw_url='https://aw.example.ru')
 # Если токен доступа указан в переменной окружения AW_DATA_TOKEN, то объект сессии можно создавать 
 # без явного указания параметра token: session = Session(aw_url='https://aw.example.ru')
 
-df = session.load()  # df: pandas.DataFrame
+df = session.load(model_id=123)  # df: pandas.DataFrame
 
 display(df)
 ```

{analytic_workspace_client-1.1.1 → analytic_workspace_client-1.29.0}/src/analytic_workspace_client.egg-info/SOURCES.txt

@@ -12,15 +12,31 @@ src/aw_client/domain.py
 src/aw_client/exceptions.py
 src/aw_client/session.py
 src/aw_client/tools.py
+src/aw_client/core/__init__.py
+src/aw_client/core/bundle.py
+src/aw_client/core/compiler.py
+src/aw_client/core/model_vault.py
+src/aw_client/core/spark.py
 src/aw_client/data_master/__init__.py
 src/aw_client/data_master/base.py
 src/aw_client/data_master/v0.py
 src/aw_client/data_master/v1.py
 src/aw_client/data_master/v2.py
+src/aw_client/etl_blocks/__init__.py
+src/aw_client/etl_blocks/application.py
+src/aw_client/etl_blocks/dto.py
+src/aw_client/etl_blocks/runtime.py
+src/aw_client/etl_blocks/services.py
+src/aw_client/etl_blocks/test_data.py
+src/aw_client/etl_blocks/tools.py
 src/aw_client/model_dev/__init__.py
 src/aw_client/model_dev/application.py
 src/aw_client/model_dev/cache.py
 src/aw_client/model_dev/runner.py
 src/aw_client/model_dev/virtual_objects.py
 src/aw_client/models/__init__.py
-src/aw_client/models/model_schema.py
+src/aw_client/models/model_schema.py
+src/aw_etl/__init__.py
+src/aw_etl/compiler.py
+src/aw_etl/etl_blocks.py
+src/aw_etl/models.py

analytic_workspace_client-1.29.0/src/analytic_workspace_client.egg-info/requires.txt

@@ -0,0 +1,12 @@
+python-dotenv<1.1,>=1.0
+httpx<1.0,>=0.25
+pandas
+pydantic<2.0,>=1.10
+colorama<0.5,>=0.4
+
+[dev]
+pyspark==3.5.0
+pytest<8.3,>=8.2
+
+[ml]
+mlflow<2.15,>=2.14

{analytic_workspace_client-1.1.1 → analytic_workspace_client-1.29.0}/src/analytic_workspace_client.egg-info/top_level.txt

@@ -1 +1,2 @@
 aw_client
+aw_etl

analytic_workspace_client-1.29.0/src/aw_client/core/bundle.py

@@ -0,0 +1,39 @@
+from typing import List, Dict, Union, Optional, Any
+
+
+class NamedObjectsBundle:
+    """ """
+    def __init__(self, objs: Dict[str, Any]):
+        self._obj_list: List[Any] = [df for _, df in objs.items()]
+        self._obj_named: Dict[str, Any] = objs
+
+    def first(self) -> Optional[Any]:
+        """ """
+        return self._obj_list[0] if self._obj_list else None
+    
+    def as_list(self) -> List[Any]:
+        return self._obj_list
+    
+    def as_named(self) -> Dict[str, Any]:
+        return self._obj_named
+    
+    def __getitem__(self, item):
+        if isinstance(item, int):
+            # если item указано целым числом, то возвращаем по индексу
+            return self._obj_list[item]
+        
+        if not item in self._obj_named:
+            raise Exception(f'Данные объекта с именем "{item}" не найдена')
+        return self._obj_named[item]
+    
+    def __iter__(self):
+        return iter(self._obj_named.values())
+
+    def __bool__(self):
+        return len(self._obj_list) > 0
+    
+    def __len__(self):
+        return len(self._obj_list)
+    
+    def items(self):
+        return self._obj_named.items()

analytic_workspace_client-1.29.0/src/aw_client/core/compiler.py

@@ -0,0 +1,165 @@
+from typing import List, Tuple, Optional
+
+import builtins
+
+
+class CompiledModule:
+    """ """
+    def __init__(self, compiled_globals: dict):
+        self.compiled_globals = compiled_globals
+
+    def __contains__(self, item):
+        return item in self.compiled_globals
+
+    def __getattr__(self, item):
+        return self.compiled_globals[item]
+
+    def __getitem__(self, item):
+        return self.compiled_globals[item]
+
+    @property
+    def is_empty(self) -> bool:
+        """ Возвращает True, если в модуле нет кода (только одни комментарии, например) """
+        return len(set(self.compiled_globals) - {'__builtins__'}) == 0
+
+
+class ScriptCompiler:
+    """ """
+
+    class CompilerException(Exception):
+        """ """
+
+    # Список исключений
+    class Misconfigured(CompilerException):
+        """ """
+
+    class ForbiddenImport(CompilerException):
+        """ Исключение, которое возникает при наличии некорректных импортов в коде """
+
+    class CannotCompile(CompilerException):
+        """ Исключение, которое вызывается при невозможности компиляции кода """
+
+    # Перечисление режимов работы компилятора
+    MODE_ETL = 1
+    MODE_VIRTUAL_OBJECT_SCHEMA = 2
+    MODE_ETL_BLOCK = 3
+
+    def __init__(self, mode: Optional[int] = None):
+        self.mode = mode
+
+    def compile(self, source_code: str, mode: int = None) -> CompiledModule:
+        """ """
+        try:
+            byte_code = compile(source_code, filename='<string>', mode='exec')
+        except SyntaxError as e:
+            raise ScriptCompiler.CannotCompile(f'Ошибка компиляции: {e.msg} (line {e.lineno}): {e.text}')
+
+        try:
+            compiled_globals = self._get_globals(mode=mode if mode is not None else self.mode)
+            exec(byte_code, compiled_globals)
+        except Exception as e:
+            raise ScriptCompiler.CompilerException(f'Ошибка компиляции: {e}')
+
+        return CompiledModule(compiled_globals=compiled_globals)
+
+    def _get_globals(self, mode: int) -> dict:
+        """ """
+        if mode == ScriptCompiler.MODE_ETL:
+            safe_names, safe_modules = self._safe_globals_for_model_etl_script()
+        elif mode == ScriptCompiler.MODE_ETL_BLOCK:
+            safe_names, safe_modules = self._safe_globals_for_etl_block()
+        elif mode == ScriptCompiler.MODE_VIRTUAL_OBJECT_SCHEMA:
+            safe_names, safe_modules = self._safe_globals_for_virtual_object_schema_script()
+        else:
+            raise ScriptCompiler.Misconfigured(f'Указан некорректный режим компиляции {self.mode}')
+
+        def safe_import(name, globals=None, locals=None, fromlist=(), level=0):
+            """ """
+            modules = name.split('.')
+            if modules[0] not in safe_modules:
+                raise ScriptCompiler.ForbiddenImport(f'Импорт модуля {modules[0]} запрещен')
+
+            return __import__(name, globals, locals, fromlist, level)
+
+        safe_builtins = {}
+        for name in safe_names:
+            safe_builtins[name] = getattr(builtins, name)
+
+        safe_builtins['__import__'] = safe_import
+
+        return {'__builtins__': safe_builtins}
+
+    @staticmethod
+    def _safe_globals_for_virtual_object_schema_script() -> Tuple[List[str], List[str]]:
+        """ """
+        safe_names = [
+            'None', 'False', 'True', 'bool', 'bytes', 'chr', 'complex', 'float',
+            'hex', 'id', 'int', 'str', 'getattr', 'setattr', 'delattr',
+        ]
+
+        safe_modules = ['pyspark', 'aw_etl']
+
+        return safe_names, safe_modules
+
+    @staticmethod
+    def _safe_globals_full() -> Tuple[List[str], List[str]]:
+        """ Возвращает полный набор разрешенных модулей """
+        safe_names = [
+            'None', 'False', 'True', 
+            'abs', 'all', 'any', 'ascii', 
+            'bin', 'bool', 'bytes', 'bytearray', 
+            'callable', 'chr', 'classmethod', 'complex', 
+            'delattr', 'dict', 'divmod', 
+            'enumerate',
+            'float', 'filter', 'format', 'frozenset',
+            'getattr',
+            'hasattr', 'hash', 'hex', 
+            'id', 'int', 'isinstance', 'issubclass', 'iter',
+            'len', 'list',
+            'map', 'max', 'min',
+            'next', 
+            'object', 'oct', 'ord',
+            'pow', 'print', 'property', 
+            'range', 'repr', 'reversed', 'round', 
+            'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super',
+            'type', 'tuple', 
+            'vars',
+            'zip', 
+            '__build_class__', '__name__',
+            
+
+            'ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', 'BufferError', 'BytesWarning',
+            'DeprecationWarning', 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError', 'FutureWarning',
+            'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
+            'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', 'NotImplementedError', 'OSError',
+            'OverflowError', 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError', 'RuntimeWarning',
+            'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'TypeError',
+            'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
+            'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning', 'ZeroDivisionError',
+        ]
+
+        safe_modules = [
+            'pyspark', 'requests', 'pandas', 'numpy', 'aw_etl', 'pyparsing', 'pydantic',
+
+            'mlflow', 'prophet', 'statmodels', 'torch', 'sklearn', 'numpy', 'catboost', 
+
+            'array', 'calendar', 'codecs', 'collections', 'copy', 'csv', 'dataclasses', 'datetime', '_strptime',
+            'decimal', 'enum', 'functools', 'hashlib', 'itertools', 'json', 'math', 'queue', 'random', 're',
+            'statistics', 'string', 'time', 'urllib', 'xml',
+            'zoneinfo', 'typing', 'uuid', 'logging',
+        ]
+
+        return safe_names, safe_modules
+
+    @staticmethod
+    def _safe_globals_for_model_etl_script() -> Tuple[List[str], List[str]]:
+        """ """
+        return ScriptCompiler._safe_globals_full()
+
+    @staticmethod
+    def  _safe_globals_for_etl_block() -> Tuple[List[str], List[str]]:
+        """ """
+        safe_names, safe_modules = ScriptCompiler._safe_globals_full()
+        safe_modules.extend(['sqlglot', 'inspect'])
+
+        return safe_names, safe_modules

analytic_workspace_client-1.29.0/src/aw_client/core/model_vault.py

@@ -0,0 +1,14 @@
+from typing import Optional, Dict, Any
+
+
+class Vault:
+    """ """
+    def __init__(self, values: Optional[dict] = None):
+        self._values = values or {}
+
+    @property
+    def values(self) -> dict:
+        return self._values
+
+    def get(self, name: str) -> Optional[Any]:
+        return self._values.get(name, None)

analytic_workspace_client-1.29.0/src/aw_client/core/spark.py

@@ -0,0 +1,15 @@
+
+try:
+    from pyspark.sql import SparkSession
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+
+def build_spark_session():
+    """ """
+    return SparkSession.builder \
+        .master('local[*]') \
+        .config('spark.driver.host', '127.0.0.1') \
+        .config('spark.ui.enabled', 'false') \
+        .getOrCreate()
+

analytic_workspace_client-1.29.0/src/aw_client/etl_blocks/__init__.py

@@ -0,0 +1,4 @@
+from .runtime import get_etl_block_schema, get_etl_block_data, get_etl_block_meta
+from .test_data import ModelObjectTestData
+
+

analytic_workspace_client-1.29.0/src/aw_client/etl_blocks/application.py

@@ -0,0 +1,49 @@
+from typing import Optional, Callable
+
+try:
+    from pyspark.sql import SparkSession
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+from aw_client.core.model_vault import Vault
+from aw_client.core.compiler import CompiledModule
+
+
+class ETLBlockApplication:
+    """ """
+    def __init__(self, 
+                 spark_builder: Callable, 
+                 run_mode: str, 
+                 vault: Vault,
+                 model_module: Optional[CompiledModule] = None):
+        self._spark_builder = spark_builder
+        self._spark = None
+        self._run_model = run_mode
+        self._model_module = model_module
+        self._vault = vault
+    
+    @property
+    def spark(self) -> SparkSession:
+        if self._spark is None:
+            self._spark = self._spark_builder()
+        return self._spark
+    
+    @property
+    def is_spark_initialized(self) -> bool:
+        return self._spark is not None
+    
+    @property
+    def model_module(self) -> Optional[CompiledModule]:
+        """ """
+        return self._model_module
+
+    @property
+    def vault(self) -> Vault:
+        """ """
+        return self._vault
+
+    @property
+    def run_mode(self) -> str:
+        """ """
+        return self.run_mode
+    

analytic_workspace_client-1.29.0/src/aw_client/etl_blocks/dto.py

@@ -0,0 +1,85 @@
+from typing import Optional, Any, List, Union
+
+import datetime
+from enum import Enum
+
+from pydantic import BaseModel
+
+
+class ETLBlockParamType(str, Enum):
+    """ """
+    STRING = 'string'
+    TEXT = 'text'
+    PASSWORD = 'password'
+    SQL_TEXT = 'sql_text'
+    NUMBER = 'number'
+    FLOAT = 'float'
+    BOOL = 'bool'
+    DATE = 'date'
+    DATETIME = 'datetime'
+    SELECT = 'select'
+    ACTION = 'action'
+
+
+class ETLBlockParamGroupType(str, Enum):
+    """ """
+    GROUP = 'group'
+
+
+class ETLBlockParamActionType(str, Enum):
+    """ """
+    ACTION = 'action'
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Метаданные ETL блока
+# ----------------------------------------------------------------------------------------------------------------------
+class ETLBlockParam(BaseModel):
+    """ Параметр ETL блока """
+    code: str
+    name: str
+    type: ETLBlockParamType
+    description: Optional[str] = None
+
+    required: bool
+    mult: bool
+    domain: Optional[Any] = None
+    extra: Optional[Any] = None
+
+
+class ETLBlockParamGroup(BaseModel):
+    """ Группа параметров ETL блока """
+    code: str
+    name: str
+
+    type: ETLBlockParamGroupType
+    description: Optional[str] = None
+    view_options: Optional[dict] = None
+    mult: bool
+    params: List[Union['ETLBlockParam', 'ETLBlockParamAction', 'ETLBlockParamAction']] = []
+    extra: Optional[Any] = None
+
+
+class ETLBlockParamAction(BaseModel):
+    code: str
+    name: str
+    type: ETLBlockParamActionType
+    description: Optional[str] = None
+    action: str
+    extra: Optional[Any] = None
+
+
+class ETLBlockMeta(BaseModel):
+    """ Метаданные ETL-блока """
+    uid: str  # уникальный идентификатор блока
+    name: str  # название блока
+    version: str  # версия блока
+    description: str  # описание блока
+    author: str  # автор блока
+    updated_at: datetime.datetime  # дата и время последнего обновления
+    params: List[Union[ETLBlockParam, ETLBlockParamAction, ETLBlockParamGroup]]
+    engine_requires: Optional[List[str]] = []
+
+    @property
+    def verbose_name(self):
+        return f'{self.name} v{self.version}'

analytic_workspace_client-1.29.0/src/aw_client/etl_blocks/runtime.py

@@ -0,0 +1,202 @@
+from typing import Dict, Optional, Any, Union, List, Literal
+from pathlib import Path
+from collections import OrderedDict
+import datetime
+import inspect
+
+from aw_client.core.compiler import ScriptCompiler
+from aw_client.core.model_vault import Vault
+from aw_client.core.spark import build_spark_session
+from aw_client.core.bundle import NamedObjectsBundle
+from aw_client.models.model_schema import ModelObject, ModelObjectField
+from .application import ETLBlockApplication
+from .test_data import ModelObjectTestData
+from .tools import build_dataframe, build_model_object, build_spark_schema
+from .dto import ETLBlockMeta
+
+
+try:
+    from pyspark.sql import SparkSession, DataFrame
+    from pyspark.sql.types import DataType, StringType, DoubleType, TimestampType, LongType, BooleanType, \
+        ByteType, ShortType, IntegerType, DecimalType, FloatType, DateType, StructType, StructField
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+
+
+def get_etl_block_meta(block_path: Path) -> ETLBlockMeta:
+    """ """
+    block_meta_path = block_path / 'block_meta.json' if block_path.is_dir() else block_path
+
+    if not block_meta_path.exists():
+        raise Exception(f'Файл с метаданным блока не найден: {block_meta_path}')
+
+    with open(block_meta_path, 'rt') as f:
+        return ETLBlockMeta.parse_raw(f.read())
+
+
+def get_etl_block_schema(block_path: Path, 
+                         test_data: Union[ModelObjectTestData, List[ModelObjectTestData]],
+                         params: Optional[Dict] = None, 
+                         run_mode: Optional[Literal['']] = None, 
+                         vault: Optional[Vault] = None,
+                         model_script_code: Optional[str] = None) -> StructType:
+    """ 
+    Args:
+
+    """
+    block_code_path = block_path / 'block_code.py' if block_path.is_dir() else block_path
+    
+    if not block_code_path.exists():
+        raise Exception(f'Файл с исходным кодом блока не найден: {block_code_path}')
+    
+    with open(block_code_path, 'rt') as f:
+        block_code = f.read()
+
+    # Компиляция кода блока
+    try:
+        block_module = ScriptCompiler().compile(source_code=block_code, mode=ScriptCompiler.MODE_ETL_BLOCK)
+    except ScriptCompiler.CannotCompile as e:
+        raise Exception(f'Ошибка компиляции исходного кода блока: {e}')
+    
+    # Компиляция кода модели
+    if model_script_code:
+        try:
+            model_module = ScriptCompiler().compile(source_code=model_script_code, mode=ScriptCompiler.MODE_ETL)
+        except ScriptCompiler.CannotCompile as e:
+            raise Exception(f'Ошибка компиляции исходного кода скрипта модели: {e}')
+    else:
+        model_module = None
+
+    spark = build_spark_session()
+
+    # Дочерние датафреймы
+    dataframes = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        df = build_dataframe(spark, td)
+        if not dataframes:
+            dataframes['child'] = df
+    
+    upstream_dataframes = NamedObjectsBundle(dataframes)
+
+    # Дочерние схемы
+    schemas = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        schema = build_spark_schema(td)
+        if not schemas:
+            schemas['child'] = schema
+        schemas[td.model_name] = schema
+
+    upstream_schemas = NamedObjectsBundle(schemas)
+    
+    block_schema_parameters = inspect.signature(block_module['block_schema']).parameters
+
+    app = ETLBlockApplication(
+        spark_builder=build_spark_session,
+        run_mode=run_mode or 'full',
+        vault=vault or Vault(),
+        model_module=model_module
+    )
+
+    # Определение параметров для передачи
+    block_schema_kwargs = {}
+    if 'params' in block_schema_parameters:
+        block_schema_kwargs['params'] = params
+    if 'app' in block_schema_parameters:
+        block_schema_kwargs['app'] = app
+    if 'model_object' in block_schema_parameters:
+        block_schema_kwargs['model_object'] = build_model_object(test_data[0] if isinstance(test_data, list) else test_data)
+    if 'schema' in block_schema_parameters:
+        block_schema_kwargs['schema'] = upstream_schemas.first()
+    if 'schemas' in block_schema_parameters:
+        block_schema_kwargs['schemas'] = upstream_schemas
+    if 'upstream_schema' in block_schema_parameters:
+        block_schema_kwargs['upstream_schema'] = upstream_schemas.first()
+    if 'upstream_schemas' in block_schema_parameters:
+        block_schema_kwargs['upstream_schemas'] = upstream_schemas
+    if 'df' in block_schema_parameters:
+        block_schema_kwargs['df'] = upstream_dataframes.first()
+    if 'dfs' in block_schema_parameters:
+        block_schema_kwargs['dfs'] = upstream_dataframes
+    if 'upstream_dataframe' in block_schema_parameters:
+        block_schema_kwargs['upstream_dataframe'] = upstream_dataframes.first()
+    if 'upstream_dataframes' in block_schema_parameters:
+        block_schema_kwargs['upstream_dataframes'] = upstream_dataframes
+    
+
+    return block_module['block_schema'](**block_schema_kwargs)
+    
+
+def get_etl_block_data(block_path: Path, 
+                       test_data: Union[ModelObjectTestData, List[ModelObjectTestData]],
+                       params: Optional[Dict] = None, 
+                       run_mode: Optional[Literal['']] = None, 
+                       vault: Optional[Vault] = None,
+                       model_script_code: Optional[str] = None) -> StructType:
+    """ 
+    Args:
+
+    """
+    block_code_path = block_path / 'block_code.py' if block_path.is_dir() else block_path
+    
+    if not block_code_path.exists():
+        raise Exception(f'Файл с исходным кодом блока не найден: {block_code_path}')
+    
+    with open(block_code_path, 'rt') as f:
+        block_code = f.read()
+
+    # Компиляция кода блока
+    try:
+        block_module = ScriptCompiler().compile(source_code=block_code, mode=ScriptCompiler.MODE_ETL_BLOCK)
+    except ScriptCompiler.CannotCompile as e:
+        raise Exception(f'Ошибка компиляции исходного кода блока: {e}')
+    
+    # Компиляция кода модели
+    if model_script_code:
+        try:
+            model_module = ScriptCompiler().compile(source_code=model_script_code, mode=ScriptCompiler.MODE_ETL)
+        except ScriptCompiler.CannotCompile as e:
+            raise Exception(f'Ошибка компиляции исходного кода скрипта модели: {e}')
+    else:
+        model_module = None
+
+    spark = build_spark_session()
+
+    # Дочерние датафреймы
+    dataframes = OrderedDict()
+    for td in (test_data if isinstance(test_data, list) else [test_data]):
+        df = build_dataframe(spark, td)
+        if not dataframes:
+            dataframes['child'] = df
+        dataframes[td.model_name] = df
+    
+    upstream_dataframes = NamedObjectsBundle(dataframes)
+
+    block_data_parameters = inspect.signature(block_module['block_data']).parameters
+
+    app = ETLBlockApplication(
+        spark_builder=build_spark_session,
+        run_mode=run_mode or 'full',
+        vault=vault or Vault(),
+        model_module=model_module
+    )
+
+    # Определение параметров для передачи
+    block_schema_kwargs = {}
+    if 'params' in block_data_parameters:
+        block_schema_kwargs['params'] = params
+    if 'app' in block_data_parameters:
+        block_schema_kwargs['app'] = app
+    if 'model_object' in block_data_parameters:
+        block_schema_kwargs['model_object'] = build_model_object(test_data[0] if isinstance(test_data, list) else test_data)
+    if 'df' in block_data_parameters:
+        block_schema_kwargs['df'] = upstream_dataframes.first()
+    if 'dfs' in block_data_parameters:
+        block_schema_kwargs['dfs'] = upstream_dataframes
+    if 'upstream_dataframe' in block_data_parameters:
+        block_schema_kwargs['upstream_dataframe'] = upstream_dataframes.first()
+    if 'upstream_dataframes' in block_data_parameters:
+        block_schema_kwargs['upstream_dataframes'] = upstream_dataframes
+    
+    return block_module['block_data'](**block_schema_kwargs)
+    

analytic_workspace_client-1.29.0/src/aw_client/etl_blocks/test_data.py

@@ -0,0 +1,17 @@
+from typing import List, TypedDict, Optional
+
+from dataclasses import dataclass, field
+
+
+class ModelObjectSchemaField(TypedDict):
+    """ """
+    model_name: str
+    simple_type: str
+
+
+@dataclass
+class ModelObjectTestData:
+    """ """
+    model_name: str
+    rows: List[dict] = field(default_factory=list)
+    schema: Optional[List[ModelObjectSchemaField]] = None

analytic_workspace_client-1.29.0/src/aw_client/etl_blocks/tools.py

@@ -0,0 +1,99 @@
+from typing import Any
+import datetime
+
+
+from aw_client.models.model_schema import ModelObject, ModelObjectField
+from .test_data import ModelObjectTestData
+
+
+
+try:
+    from pyspark.sql import SparkSession, DataFrame
+    from pyspark.sql.types import DataType, StringType, DoubleType, TimestampType, LongType, BooleanType, \
+        ByteType, ShortType, IntegerType, DecimalType, FloatType, DateType, StructType, StructField
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+
+def build_spark_schema(test_data: ModelObjectTestData) -> StructType:
+    """ """
+    if test_data.schema is not None:
+        return StructType(fields=[
+            StructField(sf['model_name'], spark_type_for_simple_type(sf['simple_type']), True) for sf in test_data.schema
+        ])
+    return StructType(fields=[
+        StructField(n, spark_type_for_python_value(v), True) for n, v in test_data.rows[0].items()
+    ])
+
+
+def build_dataframe(spark: SparkSession, test_data: ModelObjectTestData) -> DataFrame:
+    """ """
+    return spark.createDataFrame(test_data.rows, schema=build_spark_schema(test_data))
+
+
+def build_model_object(test_data: ModelObjectTestData):
+    """ """
+    if test_data.schema is not None:
+        fields = [
+            ModelObjectField(name=f['model_name'], model_name=f['model_name'], simple_type=f['simple_type']) for f in test_data.schema
+        ]
+    else:
+        fields = [
+            ModelObjectField(name=n, model_name=n, simple_type=simple_type_for_python_value(v)) for n, v in test_data.rows[0].items()
+        ]
+    return ModelObject(
+        name=test_data.model_name,
+        model_name=test_data.model_name,
+        type='table',
+        sql_text=None,
+        fields=fields
+    )
+
+
+
+
+def spark_type_for_simple_type(simple_type: str) -> DataType:
+    """ 
+    """
+    if simple_type == 'number':
+        return LongType()
+    elif simple_type == 'float':
+        return DoubleType()
+    elif simple_type == 'date':
+        return TimestampType()
+    elif simple_type == 'bool':
+        return BooleanType()
+    else:
+        return StringType()
+        
+
+def spark_type_for_python_value(value: Any) -> DataType:
+    """ 
+    Returns Spark type
+    """
+    if isinstance(value, datetime.date):
+        return DateType()
+    if isinstance(value, datetime.datetime):
+        return TimestampType()
+    if isinstance(value, int):
+        return LongType()
+    if isinstance(value, float):
+        return DoubleType()
+    if isinstance(value, bool):
+        return BooleanType()
+    return StringType()
+
+
+def simple_type_for_python_value(value: Any) -> str:
+    """ """
+    if isinstance(value, datetime.date):
+        return 'date'
+    if isinstance(value, datetime.datetime):
+        return 'date'
+    if isinstance(value, int):
+        return 'number'
+    if isinstance(value, float):
+        return 'float'
+    if isinstance(value, bool):
+        return 'bool'
+    return 'string'

{analytic_workspace_client-1.1.1 → analytic_workspace_client-1.29.0}/src/aw_client/exceptions.py

@@ -13,4 +13,8 @@ class WrongDataMasterApiVersion(DataMasterApiError):
 class AwModelForbidden(DataMasterApiError):
     """
     Доступ к модели запрещен
-    """
+    """
+
+
+class AwClientMisconfigured(Exception):
+    """ """

{analytic_workspace_client-1.1.1 → analytic_workspace_client-1.29.0}/src/aw_client/session.py

@@ -6,10 +6,14 @@ from pathlib import Path
 import httpx
 import pandas
 from contextlib import contextmanager
+from urllib.parse import urljoin
 
 from aw_client.data_master import get_nearest_api
 from aw_client.domain import APIConfig
 from aw_client.models.model_schema import ModelSchema
+from aw_client.exceptions import AwClientMisconfigured
+
+from aw_client.tools import strip_column_name_prefix
 
 
 class Session:
@@ -22,7 +26,13 @@ class Session:
     def __init__(self, token: str = None, aw_url: str = None, version: int = None):
         """ """
         self.token = (token or os.getenv("AW_DATA_TOKEN") or '').strip()
-        self.aw_url = aw_url or os.getenv('AW_URL')
+        self.aw_url = (aw_url or os.getenv('AW_URL') or '').strip()
+        if not self.aw_url:
+            raise AwClientMisconfigured(
+                'Укажите URL к AnalyticWorkspace: Session(aw_url=\'http://aw.mydomain.ru\') или установите переменную '
+                'окружения AW_URL')
+        if not self.aw_url.endswith('/'):
+            self.aw_url += '/'
         self.version = version if version is not None else None
 
     def get_aw_version(self) -> int:
@@ -41,7 +51,7 @@ class Session:
 
         return self.version
 
-    def load_model(self, model_id: int, **options) -> pandas.DataFrame:
+    def load_model(self, model_id: int, strip_prefix: bool = False, **options) -> pandas.DataFrame:
         """
         Загружает данные модели в датафрейм pandas
         """
@@ -60,7 +70,11 @@ class Session:
         if api is None:
             raise Session.Error(f'Не удалось получить API для версии AW {self.version}')
 
-        return api.load_model(model_id=model_id, **options)
+        df = api.load_model(model_id=model_id, **options)
+        if strip_prefix:
+            df = strip_column_name_prefix(df, inplace=True)
+
+        return df
 
     def model_schema(self, model_id: int) -> ModelSchema:
         """ """
@@ -114,3 +128,27 @@ class Session:
             yield client
         finally:
             client.close()
+
+    @property
+    def mlflow(self):
+        """ """
+        try:
+            import mlflow
+        except ImportError:
+            raise AwClientMisconfigured(
+                'Для использованиея MLFlow установите библиотеку с опцией ml: pip install analytic-workspace-client[ml]')
+
+        if not self.token:
+            data_master_url = urljoin(self.aw_url, 'data-master/get-token')
+            raise AwClientMisconfigured(
+                f'Не указан токен доступа к AnalyticWorkspace. Пройдите по адресу {data_master_url} для получения '
+                f'токена')
+
+        tracking_url = urljoin(self.aw_url, 'mlflow')
+
+        if mlflow.get_tracking_uri() != tracking_url:
+            mlflow.set_tracking_uri(tracking_url)
+
+        os.environ['MLFLOW_TRACKING_TOKEN'] = self.token
+
+        return mlflow

analytic_workspace_client-1.29.0/src/aw_client/tools.py

@@ -0,0 +1,36 @@
+import tempfile
+import shutil
+from pathlib import Path
+from contextlib import contextmanager
+
+import pandas as pd
+
+
+@contextmanager
+def get_temp_folder(remove_on_exit: bool = True) -> Path:
+    """ """
+    temp_folder = tempfile.mkdtemp()
+
+    try:
+        yield Path(temp_folder)
+    finally:
+        if remove_on_exit:
+            shutil.rmtree(temp_folder, ignore_errors=True)
+
+
+def strip_column_name_prefix(df: pd.DataFrame, inplace: bool = False) -> pd.DataFrame:
+    """ 
+    Вырезает общий префикс у столбцов датафрейма 
+    """
+    column_renames = {}
+    for column_name in df.columns:
+        if '__' in column_name:
+            _, column_renames[column_name] = column_name.rsplit('__', 1)
+
+    if column_renames:
+        if inplace:
+            df.rename(columns=column_renames, inplace=inplace)
+        else:
+            df.rename(columns=column_renames)
+
+    return df

analytic_workspace_client-1.29.0/src/aw_etl/compiler.py

@@ -0,0 +1,17 @@
+from typing import Protocol
+
+
+class CompiledModule(Protocol):
+    """ """
+    def __contains__(self, item):
+        """ """
+
+    def __getattr__(self, item):
+        """ """
+
+    def __getitem__(self, item):
+        """ """
+
+    @property
+    def is_empty(self) -> bool:
+        """ """

analytic_workspace_client-1.29.0/src/aw_etl/etl_blocks.py

@@ -0,0 +1,39 @@
+from typing import Protocol, Optional
+
+try:
+    from pyspark.sql import SparkSession
+except ImportError:
+    raise Exception('Для использования Spark установите библиотеку с опцией [dev]: `pip install analytic-workspace-client[dev]`')
+
+
+from aw_etl.models import Vault
+from aw_etl.compiler import CompiledModule
+
+
+class InvalidEtlBlock(Exception):
+    """ """
+
+
+class ETLBlockApplication(Protocol):
+    """ """
+    @property
+    def spark(self) -> SparkSession:
+        """ """
+
+    @property
+    def is_spark_initialized(self) -> bool:
+        """ """
+
+    @property
+    def model_module(self) -> Optional[CompiledModule]:
+        """ """
+
+    @property
+    def vault(self) -> Vault:
+        """ """
+
+    @property
+    def run_mode(self) -> str:
+        """ """
+
+    

analytic_workspace_client-1.29.0/src/aw_etl/models.py

@@ -0,0 +1,22 @@
+from typing import Protocol, List, Any
+
+
+class ModelObjectField(Protocol):
+    """ """
+    @property
+    def simple_type(self) -> str:
+        """ """
+
+
+class ModelObject(Protocol):
+    """ """
+    @property
+    def fields(self) -> List[ModelObjectField]:
+        """ """
+
+
+class Vault(Protocol):
+    """ """
+    def get(self, name: str) -> Any:
+        """ """
+

analytic_workspace_client-1.1.1/src/analytic_workspace_client.egg-info/requires.txt

@@ -1,8 +0,0 @@
-python-dotenv<1.0,>=0.21
-httpx<1.0,>=0.23
-pandas<2.0,>=1.5
-pydantic<2.0,>=1.10
-colorama<0.5,>=0.4
-
-[dev]
-pyspark==3.3.1

analytic_workspace_client-1.1.1/src/aw_client/tools.py

@@ -1,16 +0,0 @@
-import tempfile
-import shutil
-from pathlib import Path
-from contextlib import contextmanager
-
-
-@contextmanager
-def get_temp_folder(remove_on_exit: bool = True) -> Path:
-    """ """
-    temp_folder = tempfile.mkdtemp()
-
-    try:
-        yield Path(temp_folder)
-    finally:
-        if remove_on_exit:
-            shutil.rmtree(temp_folder, ignore_errors=True)